VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 92882

Last change on this file since 92882 was 92882, checked in by vboxsync, 3 years ago

ValKit/common/utils.py: Read 'comm' when 'exe' is inaccessible on linux.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 84.5 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 92882 2021-12-13 13:34:57Z vboxsync $
3# pylint: disable=too-many-lines
4
5"""
6Common Utility Functions.
7"""
8
9from __future__ import print_function;
10
11__copyright__ = \
12"""
13Copyright (C) 2012-2020 Oracle Corporation
14
15This file is part of VirtualBox Open Source Edition (OSE), as
16available from http://www.virtualbox.org. This file is free software;
17you can redistribute it and/or modify it under the terms of the GNU
18General Public License (GPL) as published by the Free Software
19Foundation, in version 2 as it comes in the "COPYING" file of the
20VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23The contents of this file may alternatively be used under the terms
24of the Common Development and Distribution License Version 1.0
25(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26VirtualBox OSE distribution, in which case the provisions of the
27CDDL are applicable instead of those of the GPL.
28
29You may elect to license modified versions of this file under the
30terms and conditions of either the GPL or the CDDL or both.
31"""
32__version__ = "$Revision: 92882 $"
33
34
35# Standard Python imports.
36import datetime;
37import os;
38import platform;
39import re;
40import stat;
41import subprocess;
42import sys;
43import time;
44import traceback;
45import unittest;
46
47if sys.platform == 'win32':
48 import ctypes;
49 import msvcrt; # pylint: disable=import-error
50 import win32api; # pylint: disable=import-error
51 import win32con; # pylint: disable=import-error
52 import win32console; # pylint: disable=import-error
53 import win32file; # pylint: disable=import-error
54 import win32process; # pylint: disable=import-error
55 import winerror; # pylint: disable=import-error
56 import pywintypes; # pylint: disable=import-error
57else:
58 import signal;
59
60# Python 3 hacks:
61if sys.version_info[0] >= 3:
62 unicode = str; # pylint: disable=redefined-builtin,invalid-name
63 xrange = range; # pylint: disable=redefined-builtin,invalid-name
64 long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67#
68# Strings.
69#
70
71def toUnicode(sString, encoding = None, errors = 'strict'):
72 """
73 A little like the python 2 unicode() function.
74 """
75 if sys.version_info[0] >= 3:
76 if isinstance(sString, bytes):
77 return str(sString, encoding if encoding else 'utf-8', errors);
78 else:
79 if not isinstance(sString, unicode):
80 return unicode(sString, encoding if encoding else 'utf-8', errors);
81 return sString;
82
83
84
85#
86# Output.
87#
88
89def printOut(sString):
90 """
91 Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92 """
93 sStreamEncoding = sys.stdout.encoding;
94 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97 print(sString);
98 else:
99 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101def printErr(sString):
102 """
103 Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104 """
105 sStreamEncoding = sys.stderr.encoding;
106 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109 print(sString, file = sys.stderr);
110 else:
111 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114#
115# Host OS and CPU.
116#
117
118def getHostOs():
119 """
120 Gets the host OS name (short).
121
122 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123 """
124 sPlatform = platform.system();
125 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126 sPlatform = sPlatform.lower();
127 elif sPlatform == 'Windows':
128 sPlatform = 'win';
129 elif sPlatform == 'SunOS':
130 sPlatform = 'solaris';
131 else:
132 raise Exception('Unsupported platform "%s"' % (sPlatform,));
133 return sPlatform;
134
135g_sHostArch = None;
136
137def getHostArch():
138 """
139 Gets the host CPU architecture.
140
141 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142 """
143 global g_sHostArch;
144 if g_sHostArch is None:
145 sArch = platform.machine();
146 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147 sArch = 'x86';
148 elif sArch in ('AMD64', 'amd64', 'x86_64'):
149 sArch = 'amd64';
150 elif sArch == 'i86pc': # SunOS
151 if platform.architecture()[0] == '64bit':
152 sArch = 'amd64';
153 else:
154 try:
155 sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156 except:
157 pass;
158 sArch = sArch.strip();
159 if sArch != 'amd64':
160 sArch = 'x86';
161 else:
162 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163 g_sHostArch = sArch;
164 return g_sHostArch;
165
166
167def getHostOsDotArch():
168 """
169 Gets the 'os.arch' for the host.
170 """
171 return '%s.%s' % (getHostOs(), getHostArch());
172
173
174def isValidOs(sOs):
175 """
176 Validates the OS name.
177 """
178 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179 'os2', 'solaris', 'win', 'os-agnostic'):
180 return True;
181 return False;
182
183
184def isValidArch(sArch):
185 """
186 Validates the CPU architecture name.
187 """
188 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190 return True;
191 return False;
192
193def isValidOsDotArch(sOsDotArch):
194 """
195 Validates the 'os.arch' string.
196 """
197
198 asParts = sOsDotArch.split('.');
199 if asParts.length() != 2:
200 return False;
201 return isValidOs(asParts[0]) \
202 and isValidArch(asParts[1]);
203
204def getHostOsVersion():
205 """
206 Returns the host OS version. This is platform.release with additional
207 distro indicator on linux.
208 """
209 sVersion = platform.release();
210 sOs = getHostOs();
211 if sOs == 'linux':
212 sDist = '';
213 try:
214 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
215 oFile = open('/etc/lsb-release');
216 for sLine in oFile:
217 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
218 if oMatch is not None:
219 sDist = oMatch.group(1).strip();
220 except:
221 pass;
222 if sDist:
223 sVersion += ' / ' + sDist;
224 else:
225 asFiles = \
226 [
227 [ '/etc/debian_version', 'Debian v'],
228 [ '/etc/gentoo-release', '' ],
229 [ '/etc/oracle-release', '' ],
230 [ '/etc/redhat-release', '' ],
231 [ '/etc/SuSE-release', '' ],
232 ];
233 for sFile, sPrefix in asFiles:
234 if os.path.isfile(sFile):
235 try:
236 oFile = open(sFile);
237 sLine = oFile.readline();
238 oFile.close();
239 except:
240 continue;
241 sLine = sLine.strip()
242 if sLine:
243 sVersion += ' / ' + sPrefix + sLine;
244 break;
245
246 elif sOs == 'solaris':
247 sVersion = platform.version();
248 if os.path.isfile('/etc/release'):
249 try:
250 oFile = open('/etc/release');
251 sLast = oFile.readlines()[-1];
252 oFile.close();
253 sLast = sLast.strip();
254 if sLast:
255 sVersion += ' (' + sLast + ')';
256 except:
257 pass;
258
259 elif sOs == 'darwin':
260 sOsxVersion = platform.mac_ver()[0];
261 codenames = {"4": "Tiger",
262 "5": "Leopard",
263 "6": "Snow Leopard",
264 "7": "Lion",
265 "8": "Mountain Lion",
266 "9": "Mavericks",
267 "10": "Yosemite",
268 "11": "El Capitan",
269 "12": "Sierra",
270 "13": "High Sierra",
271 "14": "Mojave",
272 "15": "Catalina",
273 "16": "Unknown 16",
274 "17": "Unknown 17",
275 "18": "Unknown 18",
276 "19": "Unknown 19", }
277 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
278
279 elif sOs == 'win':
280 class OSVersionInfoEx(ctypes.Structure):
281 """ OSVERSIONEX """
282 kaFields = [
283 ('dwOSVersionInfoSize', ctypes.c_ulong),
284 ('dwMajorVersion', ctypes.c_ulong),
285 ('dwMinorVersion', ctypes.c_ulong),
286 ('dwBuildNumber', ctypes.c_ulong),
287 ('dwPlatformId', ctypes.c_ulong),
288 ('szCSDVersion', ctypes.c_wchar*128),
289 ('wServicePackMajor', ctypes.c_ushort),
290 ('wServicePackMinor', ctypes.c_ushort),
291 ('wSuiteMask', ctypes.c_ushort),
292 ('wProductType', ctypes.c_byte),
293 ('wReserved', ctypes.c_byte)]
294 _fields_ = kaFields # pylint: disable=invalid-name
295
296 def __init__(self):
297 super(OSVersionInfoEx, self).__init__()
298 self.dwOSVersionInfoSize = ctypes.sizeof(self)
299
300 oOsVersion = OSVersionInfoEx()
301 rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
302 if rc == 0:
303 # Python platform.release() is not reliable for newer server releases
304 if oOsVersion.wProductType != 1:
305 if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
306 sVersion = '2016Server';
307 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
308 sVersion = '2012ServerR2';
309 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
310 sVersion = '2012Server';
311 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
312 sVersion = '2008ServerR2';
313 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
314 sVersion = '2008Server';
315 elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
316 sVersion = '2003Server';
317 sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
318 if oOsVersion.wServicePackMajor:
319 sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
320 if oOsVersion.wServicePackMinor:
321 sVersion += '.' + str(oOsVersion.wServicePackMinor)
322
323 return sVersion;
324
325def getPresentCpuCount():
326 """
327 Gets the number of CPUs present in the system.
328
329 This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
330 that we return the active count rather than the maximum count. If we don't,
331 we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
332 never will have.
333
334 @todo This is probably not exactly what we get on non-windows...
335 """
336
337 if getHostOs() == 'win':
338 fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
339 if fnGetActiveProcessorCount:
340 cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
341 if cCpus > 0:
342 return cCpus;
343
344 import multiprocessing
345 return multiprocessing.cpu_count();
346
347
348#
349# File system.
350#
351
352def openNoInherit(sFile, sMode = 'r'):
353 """
354 Wrapper around open() that tries it's best to make sure the file isn't
355 inherited by child processes.
356
357 This is a best effort thing at the moment as it doesn't synchronizes with
358 child process spawning in any way. Thus it can be subject to races in
359 multithreaded programs.
360 """
361
362 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
363 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
364 if uPythonVer >= ((3 << 16) | 4):
365 oFile = open(sFile, sMode);
366 else:
367 try:
368 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
369 except:
370 # On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
371 if getHostOs() == 'win':
372 if uPythonVer < (3 << 16):
373 offComma = sMode.find(',');
374 if offComma < 0:
375 return open(sFile, sMode + 'N');
376 return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
377
378 # Just in case.
379 return open(sFile, sMode);
380
381 oFile = open(sFile, sMode);
382 #try:
383 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
384 #except:
385 # pass;
386 return oFile;
387
388def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
389 """
390 Wrapper around open() that tries it's best to make sure the file isn't
391 inherited by child processes.
392
393 This is a best effort thing at the moment as it doesn't synchronizes with
394 child process spawning in any way. Thus it can be subject to races in
395 multithreaded programs.
396 """
397
398 if getHostOs() == 'win':
399 # Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
400 # pylint: disable=no-member,c-extension-no-member
401 fAccess = 0;
402 fDisposition = win32file.OPEN_EXISTING;
403 if 'r' in sMode or '+' in sMode:
404 fAccess |= win32file.GENERIC_READ;
405 if 'a' in sMode:
406 fAccess |= win32file.GENERIC_WRITE;
407 fDisposition = win32file.OPEN_ALWAYS;
408 elif 'w' in sMode:
409 fAccess = win32file.GENERIC_WRITE;
410 if '+' in sMode:
411 fDisposition = win32file.OPEN_ALWAYS;
412 fAccess |= win32file.GENERIC_READ;
413 else:
414 fDisposition = win32file.CREATE_ALWAYS;
415 if not fAccess:
416 fAccess |= win32file.GENERIC_READ;
417 fSharing = (win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE
418 | win32file.FILE_SHARE_DELETE);
419 hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
420 if 'a' in sMode:
421 win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
422
423 # Turn the NT handle into a CRT file descriptor.
424 hDetachedFile = hFile.Detach();
425 if fAccess == win32file.GENERIC_READ:
426 fOpen = os.O_RDONLY;
427 elif fAccess == win32file.GENERIC_WRITE:
428 fOpen = os.O_WRONLY;
429 else:
430 fOpen = os.O_RDWR;
431 # pulint: enable=no-member,c-extension-no-member
432 if 'a' in sMode:
433 fOpen |= os.O_APPEND;
434 if 'b' in sMode or 't' in sMode:
435 fOpen |= os.O_TEXT; # pylint: disable=no-member
436 fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
437
438 # Tell python to use this handle.
439 oFile = os.fdopen(fdFile, sMode);
440 else:
441 oFile = open(sFile, sMode);
442
443 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
444 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
445 if uPythonVer < ((3 << 16) | 4):
446 try:
447 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
448 except:
449 pass;
450 else:
451 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
452 return oFile;
453
454def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
455 """
456 No exceptions os.readlink wrapper.
457 """
458 try:
459 sRet = os.readlink(sPath); # pylint: disable=no-member
460 except:
461 return sXcptRet;
462 if hasattr(sRet, 'decode'):
463 sRet = sRet.decode(sEncoding, 'ignore');
464 return sRet;
465
466def readFile(sFile, sMode = 'rb'):
467 """
468 Reads the entire file.
469 """
470 oFile = open(sFile, sMode);
471 sRet = oFile.read();
472 oFile.close();
473 return sRet;
474
475def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
476 """
477 No exceptions common.readFile wrapper.
478 """
479 try:
480 sRet = readFile(sFile, sMode);
481 except:
482 sRet = sXcptRet;
483 if sEncoding is not None and hasattr(sRet, 'decode'):
484 sRet = sRet.decode(sEncoding, 'ignore');
485 return sRet;
486
487def noxcptRmDir(sDir, oXcptRet = False):
488 """
489 No exceptions os.rmdir wrapper.
490 """
491 oRet = True;
492 try:
493 os.rmdir(sDir);
494 except:
495 oRet = oXcptRet;
496 return oRet;
497
498def noxcptDeleteFile(sFile, oXcptRet = False):
499 """
500 No exceptions os.remove wrapper.
501 """
502 oRet = True;
503 try:
504 os.remove(sFile);
505 except:
506 oRet = oXcptRet;
507 return oRet;
508
509
510def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
511 # type: (string, (string, stat) -> bool) -> bool
512 """
513 Recursively walks a directory tree, calling fnCallback for each.
514
515 fnCallback takes a full path and stat object (can be None). It
516 returns a boolean value, False stops walking and returns immediately.
517
518 Returns True or False depending on fnCallback.
519 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
520 """
521 def __worker(sCurDir):
522 """ Worker for """
523 try:
524 asNames = os.listdir(sCurDir);
525 except:
526 if not fIgnoreExceptions:
527 raise;
528 return None;
529 rc = True;
530 for sName in asNames:
531 if sName not in [ '.', '..' ]:
532 sFullName = os.path.join(sCurDir, sName);
533 try: oStat = os.lstat(sFullName);
534 except: oStat = None;
535 if fnCallback(sFullName, oStat) is False:
536 return False;
537 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
538 rc = __worker(sFullName);
539 if rc is False:
540 break;
541 return rc;
542
543 # Ensure unicode path here so listdir also returns unicode on windows.
544 ## @todo figure out unicode stuff on non-windows.
545 if sys.platform == 'win32':
546 sDir = unicode(sDir);
547 return __worker(sDir);
548
549
550
551def formatFileMode(uMode):
552 # type: (int) -> string
553 """
554 Format a st_mode value 'ls -la' fasion.
555 Returns string.
556 """
557 if stat.S_ISDIR(uMode): sMode = 'd';
558 elif stat.S_ISREG(uMode): sMode = '-';
559 elif stat.S_ISLNK(uMode): sMode = 'l';
560 elif stat.S_ISFIFO(uMode): sMode = 'p';
561 elif stat.S_ISCHR(uMode): sMode = 'c';
562 elif stat.S_ISBLK(uMode): sMode = 'b';
563 elif stat.S_ISSOCK(uMode): sMode = 's';
564 else: sMode = '?';
565 ## @todo sticky bits.
566 sMode += 'r' if uMode & stat.S_IRUSR else '-';
567 sMode += 'w' if uMode & stat.S_IWUSR else '-';
568 sMode += 'x' if uMode & stat.S_IXUSR else '-';
569 sMode += 'r' if uMode & stat.S_IRGRP else '-';
570 sMode += 'w' if uMode & stat.S_IWGRP else '-';
571 sMode += 'x' if uMode & stat.S_IXGRP else '-';
572 sMode += 'r' if uMode & stat.S_IROTH else '-';
573 sMode += 'w' if uMode & stat.S_IWOTH else '-';
574 sMode += 'x' if uMode & stat.S_IXOTH else '-';
575 sMode += ' ';
576 return sMode;
577
578
579def formatFileStat(oStat):
580 # type: (stat) -> string
581 """
582 Format a stat result 'ls -la' fasion (numeric IDs).
583 Returns string.
584 """
585 return '%s %3s %4s %4s %10s %s' \
586 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
587 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
588
589## Good buffer for file operations.
590g_cbGoodBufferSize = 256*1024;
591
592## The original shutil.copyfileobj.
593g_fnOriginalShCopyFileObj = None;
594
595def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
596 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
597 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
598
599def __installShUtilHacks(shutil):
600 """ Installs the shutil buffer size hacks. """
601 global g_fnOriginalShCopyFileObj;
602 if g_fnOriginalShCopyFileObj is None:
603 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
604 shutil.copyfileobj = __myshutilcopyfileobj;
605 return True;
606
607
608def copyFileSimple(sFileSrc, sFileDst):
609 """
610 Wrapper around shutil.copyfile that simply copies the data of a regular file.
611 Raises exception on failure.
612 Return True for show.
613 """
614 import shutil;
615 __installShUtilHacks(shutil);
616 return shutil.copyfile(sFileSrc, sFileDst);
617
618
619def getDiskUsage(sPath):
620 """
621 Get free space of a partition that corresponds to specified sPath in MB.
622
623 Returns partition free space value in MB.
624 """
625 if platform.system() == 'Windows':
626 oCTypeFreeSpace = ctypes.c_ulonglong(0);
627 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
628 ctypes.pointer(oCTypeFreeSpace));
629 cbFreeSpace = oCTypeFreeSpace.value;
630 else:
631 oStats = os.statvfs(sPath); # pylint: disable=no-member
632 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
633
634 # Convert to MB
635 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
636
637 return cMbFreeSpace;
638
639
640
641#
642# SubProcess.
643#
644
645def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
646 """
647 If the "executable" is a python script, insert the python interpreter at
648 the head of the argument list so that it will work on systems which doesn't
649 support hash-bang scripts.
650 """
651
652 asArgs = dKeywordArgs.get('args');
653 if asArgs is None:
654 asArgs = aPositionalArgs[0];
655
656 if asArgs[0].endswith('.py'):
657 if sys.executable:
658 asArgs.insert(0, sys.executable);
659 else:
660 asArgs.insert(0, 'python');
661
662 # paranoia...
663 if dKeywordArgs.get('args') is not None:
664 dKeywordArgs['args'] = asArgs;
665 else:
666 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
667 return None;
668
669def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
670 """
671 Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
672 """
673 if getHostOs() == 'win':
674 if dKeywordArgs.get('creationflags', 0) == 0:
675 dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
676 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
677
678def processCall(*aPositionalArgs, **dKeywordArgs):
679 """
680 Wrapper around subprocess.call to deal with its absence in older
681 python versions.
682 Returns process exit code (see subprocess.poll).
683 """
684 assert dKeywordArgs.get('stdout') is None;
685 assert dKeywordArgs.get('stderr') is None;
686 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
687 oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
688 return oProcess.wait();
689
690def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
691 """
692 Wrapper around subprocess.check_output to deal with its absense in older
693 python versions.
694
695 Extra keywords for specifying now output is to be decoded:
696 sEncoding='utf-8
697 fIgnoreEncoding=True/False
698 """
699 sEncoding = dKeywordArgs.get('sEncoding');
700 if sEncoding is not None: del dKeywordArgs['sEncoding'];
701 else: sEncoding = 'utf-8';
702
703 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
704 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
705 else: fIgnoreEncoding = True;
706
707 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
708 oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
709
710 sOutput, _ = oProcess.communicate();
711 iExitCode = oProcess.poll();
712
713 if iExitCode != 0:
714 asArgs = dKeywordArgs.get('args');
715 if asArgs is None:
716 asArgs = aPositionalArgs[0];
717 print(sOutput);
718 raise subprocess.CalledProcessError(iExitCode, asArgs);
719
720 if hasattr(sOutput, 'decode'):
721 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
722 return sOutput;
723
724def processOutputUnchecked(*aPositionalArgs, **dKeywordArgs):
725 """
726 Similar to processOutputChecked, but returns status code and both stdout
727 and stderr results.
728
729 Extra keywords for specifying now output is to be decoded:
730 sEncoding='utf-8
731 fIgnoreEncoding=True/False
732 """
733 sEncoding = dKeywordArgs.get('sEncoding');
734 if sEncoding is not None: del dKeywordArgs['sEncoding'];
735 else: sEncoding = 'utf-8';
736
737 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
738 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
739 else: fIgnoreEncoding = True;
740
741 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
742 oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
743
744 sOutput, sError = oProcess.communicate();
745 iExitCode = oProcess.poll();
746
747 if hasattr(sOutput, 'decode'):
748 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
749 if hasattr(sError, 'decode'):
750 sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
751 return (iExitCode, sOutput, sError);
752
753g_fOldSudo = None;
754def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
755 """
756 Adds 'sudo' (or similar) to the args parameter, whereever it is.
757 """
758
759 # Are we root?
760 fIsRoot = True;
761 try:
762 fIsRoot = os.getuid() == 0; # pylint: disable=no-member
763 except:
764 pass;
765
766 # If not, prepend sudo (non-interactive, simulate initial login).
767 if fIsRoot is not True:
768 asArgs = dKeywordArgs.get('args');
769 if asArgs is None:
770 asArgs = aPositionalArgs[0];
771
772 # Detect old sudo.
773 global g_fOldSudo;
774 if g_fOldSudo is None:
775 try:
776 sVersion = str(processOutputChecked(['sudo', '-V']));
777 except:
778 sVersion = '1.7.0';
779 sVersion = sVersion.strip().split('\n')[0];
780 sVersion = sVersion.replace('Sudo version', '').strip();
781 g_fOldSudo = len(sVersion) >= 4 \
782 and sVersion[0] == '1' \
783 and sVersion[1] == '.' \
784 and sVersion[2] <= '6' \
785 and sVersion[3] == '.';
786
787 asArgs.insert(0, 'sudo');
788 if not g_fOldSudo:
789 asArgs.insert(1, '-n');
790 if fInitialEnv and not g_fOldSudo:
791 asArgs.insert(1, '-i');
792
793 # paranoia...
794 if dKeywordArgs.get('args') is not None:
795 dKeywordArgs['args'] = asArgs;
796 else:
797 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
798 return None;
799
800
801def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
802 """
803 sudo (or similar) + subprocess.call
804 """
805 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
806 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
807 return processCall(*aPositionalArgs, **dKeywordArgs);
808
809def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
810 """
811 sudo (or similar) + subprocess.check_output.
812 """
813 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
814 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
815 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
816
817def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
818 """
819 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
820 """
821 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
822 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
823 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
824
825def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
826 """
827 sudo (or similar) + processPopenSafe.
828 """
829 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
830 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
831 return processPopenSafe(*aPositionalArgs, **dKeywordArgs);
832
833
834def whichProgram(sName, sPath = None):
835 """
836 Works similar to the 'which' utility on unix.
837
838 Returns path to the given program if found.
839 Returns None if not found.
840 """
841 sHost = getHostOs();
842 sSep = ';' if sHost in [ 'win', 'os2' ] else ':';
843
844 if sPath is None:
845 if sHost == 'win':
846 sPath = os.environ.get('Path', None);
847 else:
848 sPath = os.environ.get('PATH', None);
849 if sPath is None:
850 return None;
851
852 for sDir in sPath.split(sSep):
853 if sDir.strip() != '':
854 sTest = os.path.abspath(os.path.join(sDir, sName));
855 else:
856 sTest = os.path.abspath(sName);
857 if os.path.exists(sTest):
858 return sTest;
859
860 return None;
861
862#
863# Generic process stuff.
864#
865
866def processInterrupt(uPid):
867 """
868 Sends a SIGINT or equivalent to interrupt the specified process.
869 Returns True on success, False on failure.
870
871 On Windows hosts this may not work unless the process happens to be a
872 process group leader.
873 """
874 if sys.platform == 'win32':
875 try:
876 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
877 uPid);
878 fRc = True;
879 except:
880 fRc = False;
881 else:
882 try:
883 os.kill(uPid, signal.SIGINT);
884 fRc = True;
885 except:
886 fRc = False;
887 return fRc;
888
889def sendUserSignal1(uPid):
890 """
891 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
892 (VBoxSVC) or something.
893 Returns True on success, False on failure or if not supported (win).
894
895 On Windows hosts this may not work unless the process happens to be a
896 process group leader.
897 """
898 if sys.platform == 'win32':
899 fRc = False;
900 else:
901 try:
902 os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
903 fRc = True;
904 except:
905 fRc = False;
906 return fRc;
907
908def processTerminate(uPid):
909 """
910 Terminates the process in a nice manner (SIGTERM or equivalent).
911 Returns True on success, False on failure.
912 """
913 fRc = False;
914 if sys.platform == 'win32':
915 try:
916 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
917 False, uPid);
918 except:
919 pass;
920 else:
921 try:
922 win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
923 0x40010004); # DBG_TERMINATE_PROCESS
924 fRc = True;
925 except:
926 pass;
927 hProcess.Close(); #win32api.CloseHandle(hProcess)
928 else:
929 try:
930 os.kill(uPid, signal.SIGTERM);
931 fRc = True;
932 except:
933 pass;
934 return fRc;
935
936def processKill(uPid):
937 """
938 Terminates the process with extreme prejudice (SIGKILL).
939 Returns True on success, False on failure.
940 """
941 if sys.platform == 'win32':
942 fRc = processTerminate(uPid);
943 else:
944 try:
945 os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
946 fRc = True;
947 except:
948 fRc = False;
949 return fRc;
950
951def processKillWithNameCheck(uPid, sName):
952 """
953 Like processKill(), but checks if the process name matches before killing
954 it. This is intended for killing using potentially stale pid values.
955
956 Returns True on success, False on failure.
957 """
958
959 if processCheckPidAndName(uPid, sName) is not True:
960 return False;
961 return processKill(uPid);
962
963
964def processExists(uPid):
965 """
966 Checks if the specified process exits.
967 This will only work if we can signal/open the process.
968
969 Returns True if it positively exists, False otherwise.
970 """
971 if sys.platform == 'win32':
972 fRc = False;
973 # We try open the process for waiting since this is generally only forbidden in a very few cases.
974 try:
975 hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
976 False, uPid);
977 except pywintypes.error as oXcpt: # pylint: disable=no-member
978 if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
979 fRc = True;
980 except Exception as oXcpt:
981 pass;
982 else:
983 hProcess.Close();
984 fRc = True;
985 else:
986 try:
987 os.kill(uPid, 0);
988 fRc = True;
989 except: ## @todo check error code.
990 fRc = False;
991 return fRc;
992
993def processCheckPidAndName(uPid, sName):
994 """
995 Checks if a process PID and NAME matches.
996 """
997 fRc = processExists(uPid);
998 if fRc is not True:
999 return False;
1000
1001 if sys.platform == 'win32':
1002 try:
1003 from win32com.client import GetObject; # pylint: disable=import-error
1004 oWmi = GetObject('winmgmts:');
1005 aoProcesses = oWmi.InstancesOf('Win32_Process');
1006 for oProcess in aoProcesses:
1007 if long(oProcess.Properties_("ProcessId").Value) == uPid:
1008 sCurName = oProcess.Properties_("Name").Value;
1009 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
1010 sName = sName.lower();
1011 sCurName = sCurName.lower();
1012 if os.path.basename(sName) == sName:
1013 sCurName = os.path.basename(sCurName);
1014
1015 if sCurName == sName \
1016 or sCurName + '.exe' == sName \
1017 or sCurName == sName + '.exe':
1018 fRc = True;
1019 break;
1020 except:
1021 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
1022 pass;
1023 else:
1024 if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
1025 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1026 elif sys.platform in ('sunos5',):
1027 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1028 elif sys.platform in ('darwin',):
1029 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1030 else:
1031 asPsCmd = None;
1032
1033 if asPsCmd is not None:
1034 try:
1035 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
1036 sCurName = oPs.communicate()[0];
1037 iExitCode = oPs.wait();
1038 except:
1039 #reporter.logXcpt();
1040 return False;
1041
1042 # ps fails with non-zero exit code if the pid wasn't found.
1043 if iExitCode != 0:
1044 return False;
1045 if sCurName is None:
1046 return False;
1047 sCurName = sCurName.strip();
1048 if not sCurName:
1049 return False;
1050
1051 if os.path.basename(sName) == sName:
1052 sCurName = os.path.basename(sCurName);
1053 elif os.path.basename(sCurName) == sCurName:
1054 sName = os.path.basename(sName);
1055
1056 if sCurName != sName:
1057 return False;
1058
1059 fRc = True;
1060 return fRc;
1061
1062def processGetInfo(uPid, fSudo = False):
1063 """
1064 Tries to acquire state information of the given process.
1065
1066 Returns a string with the information on success or None on failure or
1067 if the host is not supported.
1068
1069 Note that the format of the information is host system dependent and will
1070 likely differ much between different hosts.
1071 """
1072 fRc = processExists(uPid);
1073 if fRc is not True:
1074 return None;
1075
1076 sHostOs = getHostOs();
1077 if sHostOs in [ 'linux',]:
1078 sGdb = '/usr/bin/gdb';
1079 if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1080 if not os.path.isfile(sGdb): sGdb = 'gdb';
1081 aasCmd = [
1082 [ sGdb, '-batch',
1083 '-ex', 'set pagination off',
1084 '-ex', 'thread apply all bt',
1085 '-ex', 'info proc mapping',
1086 '-ex', 'info sharedlibrary',
1087 '-p', '%u' % (uPid,), ],
1088 ];
1089 elif sHostOs == 'darwin':
1090 # LLDB doesn't work in batch mode when attaching to a process, at least
1091 # with macOS Sierra (10.12). GDB might not be installed. Use the sample
1092 # tool instead with a 1 second duration and 1000ms sampling interval to
1093 # get one stack trace. For the process mappings use vmmap.
1094 aasCmd = [
1095 [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1096 [ '/usr/bin/vmmap', '%u' % (uPid,), ],
1097 ];
1098 elif sHostOs == 'solaris':
1099 aasCmd = [
1100 [ '/usr/bin/pstack', '%u' % (uPid,), ],
1101 [ '/usr/bin/pmap', '%u' % (uPid,), ],
1102 ];
1103 else:
1104 aasCmd = [];
1105
1106 sInfo = '';
1107 for asCmd in aasCmd:
1108 try:
1109 if fSudo:
1110 sThisInfo = sudoProcessOutputChecked(asCmd);
1111 else:
1112 sThisInfo = processOutputChecked(asCmd);
1113 if sThisInfo is not None:
1114 sInfo += sThisInfo;
1115 except:
1116 pass;
1117 if not sInfo:
1118 sInfo = None;
1119
1120 return sInfo;
1121
1122
1123class ProcessInfo(object):
1124 """Process info."""
1125 def __init__(self, iPid):
1126 self.iPid = iPid;
1127 self.iParentPid = None;
1128 self.sImage = None;
1129 self.sName = None;
1130 self.asArgs = None;
1131 self.sCwd = None;
1132 self.iGid = None;
1133 self.iUid = None;
1134 self.iProcGroup = None;
1135 self.iSessionId = None;
1136
1137 def loadAll(self):
1138 """Load all the info."""
1139 sOs = getHostOs();
1140 if sOs == 'linux':
1141 sProc = '/proc/%s/' % (self.iPid,);
1142 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1143 if self.sImage is None:
1144 self.sImage = noxcptReadFile(sProc + 'comm', None);
1145 if self.sImage: self.sImage = self.sImage.strip();
1146 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1147 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1148 #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1149 # sProc = '/proc/%s/' % (self.iPid,);
1150 # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1151 # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1152 else:
1153 pass;
1154 if self.sName is None and self.sImage is not None:
1155 self.sName = self.sImage;
1156
1157 def windowsGrabProcessInfo(self, oProcess):
1158 """Windows specific loadAll."""
1159 try: self.sName = oProcess.Properties_("Name").Value;
1160 except: pass;
1161 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1162 except: pass;
1163 try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1164 except: pass;
1165 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1166 except: pass;
1167 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1168 except: pass;
1169 if self.sName is None and self.sImage is not None:
1170 self.sName = self.sImage;
1171
1172 def getBaseImageName(self):
1173 """
1174 Gets the base image name if available, use the process name if not available.
1175 Returns image/process base name or None.
1176 """
1177 sRet = self.sImage if self.sName is None else self.sName;
1178 if sRet is None:
1179 self.loadAll();
1180 sRet = self.sImage if self.sName is None else self.sName;
1181 if sRet is None:
1182 if not self.asArgs:
1183 return None;
1184 sRet = self.asArgs[0];
1185 if not sRet:
1186 return None;
1187 return os.path.basename(sRet);
1188
1189 def getBaseImageNameNoExeSuff(self):
1190 """
1191 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1192 """
1193 sRet = self.getBaseImageName();
1194 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1195 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1196 sRet = sRet[:-4];
1197 return sRet;
1198
1199
1200def processListAll():
1201 """
1202 Return a list of ProcessInfo objects for all the processes in the system
1203 that the current user can see.
1204 """
1205 asProcesses = [];
1206
1207 sOs = getHostOs();
1208 if sOs == 'win':
1209 from win32com.client import GetObject; # pylint: disable=import-error
1210 oWmi = GetObject('winmgmts:');
1211 aoProcesses = oWmi.InstancesOf('Win32_Process');
1212 for oProcess in aoProcesses:
1213 try:
1214 iPid = int(oProcess.Properties_("ProcessId").Value);
1215 except:
1216 continue;
1217 oMyInfo = ProcessInfo(iPid);
1218 oMyInfo.windowsGrabProcessInfo(oProcess);
1219 asProcesses.append(oMyInfo);
1220 return asProcesses;
1221
1222 if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1223 try:
1224 asDirs = os.listdir('/proc');
1225 except:
1226 asDirs = [];
1227 for sDir in asDirs:
1228 if sDir.isdigit():
1229 asProcesses.append(ProcessInfo(int(sDir),));
1230 return asProcesses;
1231
1232 #
1233 # The other OSes parses the output from the 'ps' utility.
1234 #
1235 asPsCmd = [
1236 '/bin/ps', # 0
1237 '-A', # 1
1238 '-o', 'pid=', # 2,3
1239 '-o', 'ppid=', # 4,5
1240 '-o', 'pgid=', # 6,7
1241 '-o', 'sid=', # 8,9
1242 '-o', 'uid=', # 10,11
1243 '-o', 'gid=', # 12,13
1244 '-o', 'comm=' # 14,15
1245 ];
1246
1247 if sOs == 'darwin':
1248 assert asPsCmd[9] == 'sid=';
1249 asPsCmd[9] = 'sess=';
1250 elif sOs == 'solaris':
1251 asPsCmd[0] = '/usr/bin/ps';
1252
1253 try:
1254 sRaw = processOutputChecked(asPsCmd);
1255 except:
1256 return asProcesses;
1257
1258 for sLine in sRaw.split('\n'):
1259 sLine = sLine.lstrip();
1260 if len(sLine) < 7 or not sLine[0].isdigit():
1261 continue;
1262
1263 iField = 0;
1264 off = 0;
1265 aoFields = [None, None, None, None, None, None, None];
1266 while iField < 7:
1267 # Eat whitespace.
1268 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1269 off += 1;
1270
1271 # Final field / EOL.
1272 if iField == 6:
1273 aoFields[6] = sLine[off:];
1274 break;
1275 if off >= len(sLine):
1276 break;
1277
1278 # Generic field parsing.
1279 offStart = off;
1280 off += 1;
1281 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1282 off += 1;
1283 try:
1284 if iField != 3:
1285 aoFields[iField] = int(sLine[offStart:off]);
1286 else:
1287 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1288 except:
1289 pass;
1290 iField += 1;
1291
1292 if aoFields[0] is not None:
1293 oMyInfo = ProcessInfo(aoFields[0]);
1294 oMyInfo.iParentPid = aoFields[1];
1295 oMyInfo.iProcGroup = aoFields[2];
1296 oMyInfo.iSessionId = aoFields[3];
1297 oMyInfo.iUid = aoFields[4];
1298 oMyInfo.iGid = aoFields[5];
1299 oMyInfo.sName = aoFields[6];
1300 asProcesses.append(oMyInfo);
1301
1302 return asProcesses;
1303
1304
1305def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1306 """
1307 Looks for information regarding the demise of the given process.
1308 """
1309 sOs = getHostOs();
1310 if sOs == 'darwin':
1311 #
1312 # On darwin we look for crash and diagnostic reports.
1313 #
1314 asLogDirs = [
1315 u'/Library/Logs/DiagnosticReports/',
1316 u'/Library/Logs/CrashReporter/',
1317 u'~/Library/Logs/DiagnosticReports/',
1318 u'~/Library/Logs/CrashReporter/',
1319 ];
1320 for sDir in asLogDirs:
1321 sDir = os.path.expanduser(sDir);
1322 if not os.path.isdir(sDir):
1323 continue;
1324 try:
1325 asDirEntries = os.listdir(sDir);
1326 except:
1327 continue;
1328 for sEntry in asDirEntries:
1329 # Only interested in .crash files.
1330 _, sSuff = os.path.splitext(sEntry);
1331 if sSuff != '.crash':
1332 continue;
1333
1334 # The pid can be found at the end of the first line.
1335 sFull = os.path.join(sDir, sEntry);
1336 try:
1337 oFile = open(sFull, 'r');
1338 sFirstLine = oFile.readline();
1339 oFile.close();
1340 except:
1341 continue;
1342 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1343 continue;
1344 offPid = len(sFirstLine) - 3;
1345 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1346 offPid -= 1;
1347 try: uReportPid = int(sFirstLine[offPid:-2]);
1348 except: continue;
1349
1350 # Does the pid we found match?
1351 if uReportPid == uPid:
1352 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1353 fnCrashFile(sFull, False);
1354 elif sOs == 'win':
1355 #
1356 # Getting WER reports would be great, however we have trouble match the
1357 # PID to those as they seems not to mention it in the brief reports.
1358 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1359 # location - see the windows readme for the testbox script) and what
1360 # the MSDN article lists for now.
1361 #
1362 # It's been observed on Windows server 2012 that the dump files takes
1363 # the form: <processimage>.<decimal-pid>.dmp
1364 #
1365 asDmpDirs = [
1366 u'%SystemDrive%/CrashDumps/', # Testboxes.
1367 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1368 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1369 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1370 u'%WINDIR%/ServiceProfiles/',
1371 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1372 ];
1373 sMatchSuffix = '.%u.dmp' % (uPid,);
1374
1375 for sDir in asDmpDirs:
1376 sDir = os.path.expandvars(sDir);
1377 if not os.path.isdir(sDir):
1378 continue;
1379 try:
1380 asDirEntries = os.listdir(sDir);
1381 except:
1382 continue;
1383 for sEntry in asDirEntries:
1384 if sEntry.endswith(sMatchSuffix):
1385 sFull = os.path.join(sDir, sEntry);
1386 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1387 fnCrashFile(sFull, True);
1388
1389 else:
1390 pass; ## TODO
1391 return None;
1392
1393
1394#
1395# Time.
1396#
1397
1398#
1399# The following test case shows how time.time() only have ~ms resolution
1400# on Windows (tested W10) and why it therefore makes sense to try use
1401# performance counters.
1402#
1403# Note! We cannot use time.clock() as the timestamp must be portable across
1404# processes. See timeout testcase problem on win hosts (no logs).
1405# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1406#
1407#import sys;
1408#import time;
1409#from common import utils;
1410#
1411#atSeries = [];
1412#for i in xrange(1,160):
1413# if i == 159: time.sleep(10);
1414# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1415#
1416#tPrev = atSeries[0]
1417#for tCur in atSeries:
1418# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1419# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1420# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1421# print '';
1422# tPrev = tCur
1423#
1424#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1425#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1426#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1427
1428g_fWinUseWinPerfCounter = sys.platform == 'win32';
1429g_fpWinPerfCounterFreq = None;
1430g_oFuncwinQueryPerformanceCounter = None;
1431
1432def _winInitPerfCounter():
1433 """ Initializes the use of performance counters. """
1434 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1435
1436 uFrequency = ctypes.c_ulonglong(0);
1437 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1438 if uFrequency.value >= 1000:
1439 #print 'uFrequency = %s' % (uFrequency,);
1440 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1441 g_fpWinPerfCounterFreq = float(uFrequency.value);
1442
1443 # Check that querying the counter works too.
1444 global g_oFuncwinQueryPerformanceCounter
1445 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1446 uCurValue = ctypes.c_ulonglong(0);
1447 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1448 if uCurValue.value > 0:
1449 return True;
1450 g_fWinUseWinPerfCounter = False;
1451 return False;
1452
1453def _winFloatTime():
1454 """ Gets floating point time on windows. """
1455 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1456 uCurValue = ctypes.c_ulonglong(0);
1457 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1458 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1459 return time.time();
1460
1461def timestampNano():
1462 """
1463 Gets a nanosecond timestamp.
1464 """
1465 if g_fWinUseWinPerfCounter is True:
1466 return long(_winFloatTime() * 1000000000);
1467 return long(time.time() * 1000000000);
1468
1469def timestampMilli():
1470 """
1471 Gets a millisecond timestamp.
1472 """
1473 if g_fWinUseWinPerfCounter is True:
1474 return long(_winFloatTime() * 1000);
1475 return long(time.time() * 1000);
1476
1477def timestampSecond():
1478 """
1479 Gets a second timestamp.
1480 """
1481 if g_fWinUseWinPerfCounter is True:
1482 return long(_winFloatTime());
1483 return long(time.time());
1484
1485def secondsSinceUnixEpoch():
1486 """
1487 Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1488 """
1489 ## ASSUMES This returns unix epoch time on all systems we care about...
1490 return time.time();
1491
1492def getTimePrefix():
1493 """
1494 Returns a timestamp prefix, typically used for logging. UTC.
1495 """
1496 try:
1497 oNow = datetime.datetime.utcnow();
1498 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1499 except:
1500 sTs = 'getTimePrefix-exception';
1501 return sTs;
1502
1503def getTimePrefixAndIsoTimestamp():
1504 """
1505 Returns current UTC as log prefix and iso timestamp.
1506 """
1507 try:
1508 oNow = datetime.datetime.utcnow();
1509 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1510 sTsIso = formatIsoTimestamp(oNow);
1511 except:
1512 sTsPrf = sTsIso = 'getTimePrefix-exception';
1513 return (sTsPrf, sTsIso);
1514
1515class UtcTzInfo(datetime.tzinfo):
1516 """UTC TZ Info Class"""
1517 def utcoffset(self, _):
1518 return datetime.timedelta(0);
1519 def tzname(self, _):
1520 return "UTC";
1521 def dst(self, _):
1522 return datetime.timedelta(0);
1523
1524class GenTzInfo(datetime.tzinfo):
1525 """Generic TZ Info Class"""
1526 def __init__(self, offInMin):
1527 datetime.tzinfo.__init__(self);
1528 self.offInMin = offInMin;
1529 def utcoffset(self, _):
1530 return datetime.timedelta(minutes = self.offInMin);
1531 def tzname(self, _):
1532 if self.offInMin >= 0:
1533 return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1534 return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1535 def dst(self, _):
1536 return datetime.timedelta(0);
1537
1538def formatIsoTimestamp(oNow):
1539 """Formats the datetime object as an ISO timestamp."""
1540 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1541 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1542 return sTs;
1543
1544def getIsoTimestamp():
1545 """Returns the current UTC timestamp as a string."""
1546 return formatIsoTimestamp(datetime.datetime.utcnow());
1547
1548def formatShortIsoTimestamp(oNow):
1549 """Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1550 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1551 return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1552
1553def getShortIsoTimestamp():
1554 """Returns the current UTC timestamp as a string, but w/o microseconds."""
1555 return formatShortIsoTimestamp(datetime.datetime.utcnow());
1556
1557def convertDateTimeToZulu(oDateTime):
1558 """ Converts oDateTime to zulu time if it has timezone info. """
1559 if oDateTime.tzinfo is not None:
1560 oDateTime = oDateTime.astimezone(UtcTzInfo());
1561 else:
1562 oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1563 return oDateTime;
1564
1565def parseIsoTimestamp(sTs):
1566 """
1567 Parses a typical ISO timestamp, returing a datetime object, reasonably
1568 forgiving, but will throw weird indexing/conversion errors if the input
1569 is malformed.
1570 """
1571 # YYYY-MM-DD
1572 iYear = int(sTs[0:4]);
1573 assert(sTs[4] == '-');
1574 iMonth = int(sTs[5:7]);
1575 assert(sTs[7] == '-');
1576 iDay = int(sTs[8:10]);
1577
1578 # Skip separator
1579 sTime = sTs[10:];
1580 while sTime[0] in 'Tt \t\n\r':
1581 sTime = sTime[1:];
1582
1583 # HH:MM[:SS]
1584 iHour = int(sTime[0:2]);
1585 assert(sTime[2] == ':');
1586 iMin = int(sTime[3:5]);
1587 if sTime[5] == ':':
1588 iSec = int(sTime[6:8]);
1589
1590 # Fraction?
1591 offTime = 8;
1592 iMicroseconds = 0;
1593 if offTime < len(sTime) and sTime[offTime] in '.,':
1594 offTime += 1;
1595 cchFraction = 0;
1596 while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1597 cchFraction += 1;
1598 if cchFraction > 0:
1599 iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1600 offTime += cchFraction;
1601 while cchFraction < 6:
1602 iMicroseconds *= 10;
1603 cchFraction += 1;
1604 while cchFraction > 6:
1605 iMicroseconds = iMicroseconds // 10;
1606 cchFraction -= 1;
1607
1608 else:
1609 iSec = 0;
1610 iMicroseconds = 0;
1611 offTime = 5;
1612
1613 # Naive?
1614 if offTime >= len(sTime):
1615 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1616
1617 # Zulu?
1618 if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1619 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1620
1621 # Some kind of offset afterwards, and strptime is useless. sigh.
1622 if sTime[offTime] in '+-':
1623 chSign = sTime[offTime];
1624 offTime += 1;
1625 cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1626 offTime += 2;
1627 if offTime < len(sTime) and sTime[offTime] in ':':
1628 offTime += 1;
1629 if offTime + 2 <= len(sTime):
1630 cMinTz += int(sTime[offTime : (offTime + 2)]);
1631 offTime += 2;
1632 assert offTime == len(sTime);
1633 if chSign == '-':
1634 cMinTz = -cMinTz;
1635 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1636 assert False, sTs;
1637 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1638
1639def normalizeIsoTimestampToZulu(sTs):
1640 """
1641 Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1642 + convertDateTimeToZulu + formatIsoTimestamp).
1643 Returns ISO tiemstamp string.
1644 """
1645 return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1646
1647def getLocalHourOfWeek():
1648 """ Local hour of week (0 based). """
1649 oNow = datetime.datetime.now();
1650 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1651
1652
1653def formatIntervalSeconds(cSeconds):
1654 """ Format a seconds interval into a nice 01h 00m 22s string """
1655 # Two simple special cases.
1656 if cSeconds < 60:
1657 return '%ss' % (cSeconds,);
1658 if cSeconds < 3600:
1659 cMins = cSeconds // 60;
1660 cSecs = cSeconds % 60;
1661 if cSecs == 0:
1662 return '%sm' % (cMins,);
1663 return '%sm %ss' % (cMins, cSecs,);
1664
1665 # Generic and a bit slower.
1666 cDays = cSeconds // 86400;
1667 cSeconds %= 86400;
1668 cHours = cSeconds // 3600;
1669 cSeconds %= 3600;
1670 cMins = cSeconds // 60;
1671 cSecs = cSeconds % 60;
1672 sRet = '';
1673 if cDays > 0:
1674 sRet = '%sd ' % (cDays,);
1675 if cHours > 0:
1676 sRet += '%sh ' % (cHours,);
1677 if cMins > 0:
1678 sRet += '%sm ' % (cMins,);
1679 if cSecs > 0:
1680 sRet += '%ss ' % (cSecs,);
1681 assert sRet; assert sRet[-1] == ' ';
1682 return sRet[:-1];
1683
1684def formatIntervalSeconds2(oSeconds):
1685 """
1686 Flexible input version of formatIntervalSeconds for use in WUI forms where
1687 data is usually already string form.
1688 """
1689 if isinstance(oSeconds, (int, long)):
1690 return formatIntervalSeconds(oSeconds);
1691 if not isString(oSeconds):
1692 try:
1693 lSeconds = long(oSeconds);
1694 except:
1695 pass;
1696 else:
1697 if lSeconds >= 0:
1698 return formatIntervalSeconds2(lSeconds);
1699 return oSeconds;
1700
1701def parseIntervalSeconds(sString):
1702 """
1703 Reverse of formatIntervalSeconds.
1704
1705 Returns (cSeconds, sError), where sError is None on success.
1706 """
1707
1708 # We might given non-strings, just return them without any fuss.
1709 if not isString(sString):
1710 if isinstance(sString, (int, long)) or sString is None:
1711 return (sString, None);
1712 ## @todo time/date objects?
1713 return (int(sString), None);
1714
1715 # Strip it and make sure it's not empty.
1716 sString = sString.strip();
1717 if not sString:
1718 return (0, 'Empty interval string.');
1719
1720 #
1721 # Split up the input into a list of 'valueN, unitN, ...'.
1722 #
1723 # Don't want to spend too much time trying to make re.split do exactly what
1724 # I need here, so please forgive the extra pass I'm making here.
1725 #
1726 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1727 asParts = [];
1728 for sPart in asRawParts:
1729 sPart = sPart.strip();
1730 if sPart:
1731 asParts.append(sPart);
1732 if not asParts:
1733 return (0, 'Empty interval string or something?');
1734
1735 #
1736 # Process them one or two at the time.
1737 #
1738 cSeconds = 0;
1739 asErrors = [];
1740 i = 0;
1741 while i < len(asParts):
1742 sNumber = asParts[i];
1743 i += 1;
1744 if sNumber.isdigit():
1745 iNumber = int(sNumber);
1746
1747 sUnit = 's';
1748 if i < len(asParts) and not asParts[i].isdigit():
1749 sUnit = asParts[i];
1750 i += 1;
1751
1752 sUnitLower = sUnit.lower();
1753 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1754 pass;
1755 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1756 iNumber *= 60;
1757 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1758 iNumber *= 3600;
1759 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1760 iNumber *= 86400;
1761 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1762 iNumber *= 7 * 86400;
1763 else:
1764 asErrors.append('Unknown unit "%s".' % (sUnit,));
1765 cSeconds += iNumber;
1766 else:
1767 asErrors.append('Bad number "%s".' % (sNumber,));
1768 return (cSeconds, None if not asErrors else ' '.join(asErrors));
1769
1770def formatIntervalHours(cHours):
1771 """ Format a hours interval into a nice 1w 2d 1h string. """
1772 # Simple special cases.
1773 if cHours < 24:
1774 return '%sh' % (cHours,);
1775
1776 # Generic and a bit slower.
1777 cWeeks = cHours / (7 * 24);
1778 cHours %= 7 * 24;
1779 cDays = cHours / 24;
1780 cHours %= 24;
1781 sRet = '';
1782 if cWeeks > 0:
1783 sRet = '%sw ' % (cWeeks,);
1784 if cDays > 0:
1785 sRet = '%sd ' % (cDays,);
1786 if cHours > 0:
1787 sRet += '%sh ' % (cHours,);
1788 assert sRet; assert sRet[-1] == ' ';
1789 return sRet[:-1];
1790
1791def parseIntervalHours(sString):
1792 """
1793 Reverse of formatIntervalHours.
1794
1795 Returns (cHours, sError), where sError is None on success.
1796 """
1797
1798 # We might given non-strings, just return them without any fuss.
1799 if not isString(sString):
1800 if isinstance(sString, (int, long)) or sString is None:
1801 return (sString, None);
1802 ## @todo time/date objects?
1803 return (int(sString), None);
1804
1805 # Strip it and make sure it's not empty.
1806 sString = sString.strip();
1807 if not sString:
1808 return (0, 'Empty interval string.');
1809
1810 #
1811 # Split up the input into a list of 'valueN, unitN, ...'.
1812 #
1813 # Don't want to spend too much time trying to make re.split do exactly what
1814 # I need here, so please forgive the extra pass I'm making here.
1815 #
1816 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1817 asParts = [];
1818 for sPart in asRawParts:
1819 sPart = sPart.strip();
1820 if sPart:
1821 asParts.append(sPart);
1822 if not asParts:
1823 return (0, 'Empty interval string or something?');
1824
1825 #
1826 # Process them one or two at the time.
1827 #
1828 cHours = 0;
1829 asErrors = [];
1830 i = 0;
1831 while i < len(asParts):
1832 sNumber = asParts[i];
1833 i += 1;
1834 if sNumber.isdigit():
1835 iNumber = int(sNumber);
1836
1837 sUnit = 'h';
1838 if i < len(asParts) and not asParts[i].isdigit():
1839 sUnit = asParts[i];
1840 i += 1;
1841
1842 sUnitLower = sUnit.lower();
1843 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1844 pass;
1845 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1846 iNumber *= 24;
1847 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1848 iNumber *= 7 * 24;
1849 else:
1850 asErrors.append('Unknown unit "%s".' % (sUnit,));
1851 cHours += iNumber;
1852 else:
1853 asErrors.append('Bad number "%s".' % (sNumber,));
1854 return (cHours, None if not asErrors else ' '.join(asErrors));
1855
1856
1857#
1858# Introspection.
1859#
1860
1861def getCallerName(oFrame=None, iFrame=2):
1862 """
1863 Returns the name of the caller's caller.
1864 """
1865 if oFrame is None:
1866 try:
1867 raise Exception();
1868 except:
1869 oFrame = sys.exc_info()[2].tb_frame.f_back;
1870 while iFrame > 1:
1871 if oFrame is not None:
1872 oFrame = oFrame.f_back;
1873 iFrame = iFrame - 1;
1874 if oFrame is not None:
1875 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1876 return sName;
1877 return "unknown";
1878
1879
1880def getXcptInfo(cFrames = 1):
1881 """
1882 Gets text detailing the exception. (Good for logging.)
1883 Returns list of info strings.
1884 """
1885
1886 #
1887 # Try get exception info.
1888 #
1889 try:
1890 oType, oValue, oTraceback = sys.exc_info();
1891 except:
1892 oType = oValue = oTraceback = None;
1893 if oType is not None:
1894
1895 #
1896 # Try format the info
1897 #
1898 asRet = [];
1899 try:
1900 try:
1901 asRet = asRet + traceback.format_exception_only(oType, oValue);
1902 asTraceBack = traceback.format_tb(oTraceback);
1903 if cFrames is not None and cFrames <= 1:
1904 asRet.append(asTraceBack[-1]);
1905 else:
1906 asRet.append('Traceback:')
1907 for iFrame in range(min(cFrames, len(asTraceBack))):
1908 asRet.append(asTraceBack[-iFrame - 1]);
1909 asRet.append('Stack:')
1910 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1911 except:
1912 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1913
1914 if not asRet:
1915 asRet.append('No exception info...');
1916 except:
1917 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1918 else:
1919 asRet = ['Couldn\'t find exception traceback.'];
1920 return asRet;
1921
1922
1923def getObjectTypeName(oObject):
1924 """
1925 Get the type name of the given object.
1926 """
1927 if oObject is None:
1928 return 'None';
1929
1930 # Get the type object.
1931 try:
1932 oType = type(oObject);
1933 except:
1934 return 'type-throws-exception';
1935
1936 # Python 2.x only: Handle old-style object wrappers.
1937 if sys.version_info[0] < 3:
1938 try:
1939 from types import InstanceType; # pylint: disable=no-name-in-module
1940 if oType == InstanceType:
1941 oType = oObject.__class__;
1942 except:
1943 pass;
1944
1945 # Get the name.
1946 try:
1947 return oType.__name__;
1948 except:
1949 return '__type__-throws-exception';
1950
1951
1952def chmodPlusX(sFile):
1953 """
1954 Makes the specified file or directory executable.
1955 Returns success indicator, no exceptions.
1956
1957 Note! Symbolic links are followed and the target will be changed.
1958 """
1959 try:
1960 oStat = os.stat(sFile);
1961 except:
1962 return False;
1963 try:
1964 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1965 except:
1966 return False;
1967 return True;
1968
1969
1970#
1971# TestSuite stuff.
1972#
1973
1974def isRunningFromCheckout(cScriptDepth = 1):
1975 """
1976 Checks if we're running from the SVN checkout or not.
1977 """
1978
1979 try:
1980 sFile = __file__;
1981 cScriptDepth = 1;
1982 except:
1983 sFile = sys.argv[0];
1984
1985 sDir = os.path.abspath(sFile);
1986 while cScriptDepth >= 0:
1987 sDir = os.path.dirname(sDir);
1988 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1989 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1990 return True;
1991 cScriptDepth -= 1;
1992
1993 return False;
1994
1995
1996#
1997# Bourne shell argument fun.
1998#
1999
2000
2001def argsSplit(sCmdLine):
2002 """
2003 Given a bourne shell command line invocation, split it up into arguments
2004 assuming IFS is space.
2005 Returns None on syntax error.
2006 """
2007 ## @todo bourne shell argument parsing!
2008 return sCmdLine.split(' ');
2009
2010def argsGetFirst(sCmdLine):
2011 """
2012 Given a bourne shell command line invocation, get return the first argument
2013 assuming IFS is space.
2014 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
2015 """
2016 asArgs = argsSplit(sCmdLine);
2017 if not asArgs:
2018 return None;
2019
2020 return asArgs[0];
2021
2022#
2023# String helpers.
2024#
2025
2026def stricmp(sFirst, sSecond):
2027 """
2028 Compares to strings in an case insensitive fashion.
2029
2030 Python doesn't seem to have any way of doing the correctly, so this is just
2031 an approximation using lower.
2032 """
2033 if sFirst == sSecond:
2034 return 0;
2035 sLower1 = sFirst.lower();
2036 sLower2 = sSecond.lower();
2037 if sLower1 == sLower2:
2038 return 0;
2039 if sLower1 < sLower2:
2040 return -1;
2041 return 1;
2042
2043
2044def versionCompare(sVer1, sVer2):
2045 """
2046 Compares to version strings in a fashion similar to RTStrVersionCompare.
2047 """
2048
2049 ## @todo implement me!!
2050
2051 if sVer1 == sVer2:
2052 return 0;
2053 if sVer1 < sVer2:
2054 return -1;
2055 return 1;
2056
2057
2058def formatNumber(lNum, sThousandSep = ' '):
2059 """
2060 Formats a decimal number with pretty separators.
2061 """
2062 sNum = str(lNum);
2063 sRet = sNum[-3:];
2064 off = len(sNum) - 3;
2065 while off > 0:
2066 off -= 3;
2067 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2068 return sRet;
2069
2070
2071def formatNumberNbsp(lNum):
2072 """
2073 Formats a decimal number with pretty separators.
2074 """
2075 sRet = formatNumber(lNum);
2076 return unicode(sRet).replace(' ', u'\u00a0');
2077
2078
2079def isString(oString):
2080 """
2081 Checks if the object is a string object, hiding difference between python 2 and 3.
2082
2083 Returns True if it's a string of some kind.
2084 Returns False if not.
2085 """
2086 if sys.version_info[0] >= 3:
2087 return isinstance(oString, str);
2088 return isinstance(oString, basestring); # pylint: disable=undefined-variable
2089
2090
2091def hasNonAsciiCharacters(sText):
2092 """
2093 Returns True is specified string has non-ASCII characters, False if ASCII only.
2094 """
2095 if isString(sText):
2096 for ch in sText:
2097 if ord(ch) >= 128:
2098 return True;
2099 else:
2100 # Probably byte array or some such thing.
2101 for ch in sText:
2102 if ch >= 128 or ch < 0:
2103 return True;
2104 return False;
2105
2106
2107#
2108# Unpacking.
2109#
2110
2111def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2112 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2113 """
2114 Worker for unpackFile that deals with ZIP files, same function signature.
2115 """
2116 import zipfile
2117 if fnError is None:
2118 fnError = fnLog;
2119
2120 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2121
2122 # Open it.
2123 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2124 except Exception as oXcpt:
2125 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2126 return None;
2127
2128 # Extract all members.
2129 asMembers = [];
2130 try:
2131 for sMember in oZipFile.namelist():
2132 if fnFilter is None or fnFilter(sMember) is not False:
2133 if sMember.endswith('/'):
2134 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2135 else:
2136 oZipFile.extract(sMember, sDstDir);
2137 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2138 except Exception as oXcpt:
2139 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2140 asMembers = None;
2141
2142 # close it.
2143 try: oZipFile.close();
2144 except Exception as oXcpt:
2145 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2146 asMembers = None;
2147
2148 return asMembers;
2149
2150
2151## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2152g_fTarCopyFileObjOverriddend = False;
2153
2154def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2155 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2156 _ = bufsize;
2157 if length is None:
2158 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2159 elif length > 0:
2160 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2161 for _ in xrange(cFull):
2162 abBuffer = src.read(g_cbGoodBufferSize);
2163 dst.write(abBuffer);
2164 if len(abBuffer) != g_cbGoodBufferSize:
2165 raise exception('unexpected end of source file');
2166 if cbRemainder > 0:
2167 abBuffer = src.read(cbRemainder);
2168 dst.write(abBuffer);
2169 if len(abBuffer) != cbRemainder:
2170 raise exception('unexpected end of source file');
2171
2172
2173def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2174 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2175 """
2176 Worker for unpackFile that deals with tarballs, same function signature.
2177 """
2178 import shutil;
2179 import tarfile;
2180 if fnError is None:
2181 fnError = fnLog;
2182
2183 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2184
2185 #
2186 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2187 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2188 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2189 #
2190 if True is True: # pylint: disable=comparison-with-itself
2191 __installShUtilHacks(shutil);
2192 global g_fTarCopyFileObjOverriddend;
2193 if g_fTarCopyFileObjOverriddend is False:
2194 g_fTarCopyFileObjOverriddend = True;
2195 #if sys.hexversion < 0x03060000:
2196 tarfile.copyfileobj = __mytarfilecopyfileobj;
2197
2198 #
2199 # Open it.
2200 #
2201 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
2202 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2203 #
2204 try:
2205 if sys.hexversion >= 0x03060000:
2206 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2207 else:
2208 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
2209 except Exception as oXcpt:
2210 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2211 return None;
2212
2213 # Extract all members.
2214 asMembers = [];
2215 try:
2216 for oTarInfo in oTarFile:
2217 try:
2218 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2219 if oTarInfo.islnk():
2220 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2221 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
2222 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2223 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2224 sParentDir = os.path.dirname(sLinkFile);
2225 try: os.unlink(sLinkFile);
2226 except: pass;
2227 if sParentDir and not os.path.exists(sParentDir):
2228 os.makedirs(sParentDir);
2229 try: os.link(sLinkTarget, sLinkFile);
2230 except: shutil.copy2(sLinkTarget, sLinkFile);
2231 else:
2232 if oTarInfo.isdir():
2233 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2234 oTarInfo.mode |= 0x1c0; # (octal: 0700)
2235 oTarFile.extract(oTarInfo, sDstDir);
2236 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2237 except Exception as oXcpt:
2238 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2239 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2240 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2241 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2242 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2243 asMembers = None;
2244 break;
2245 except Exception as oXcpt:
2246 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2247 asMembers = None;
2248
2249 #
2250 # Finally, close it.
2251 #
2252 try: oTarFile.close();
2253 except Exception as oXcpt:
2254 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2255 asMembers = None;
2256
2257 return asMembers;
2258
2259
2260def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2261 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2262 """
2263 Unpacks the given file if it has a know archive extension, otherwise do
2264 nothing.
2265
2266 fnLog & fnError both take a string parameter.
2267
2268 fnFilter takes a member name (string) and returns True if it's included
2269 and False if excluded.
2270
2271 Returns list of the extracted files (full path) on success.
2272 Returns empty list if not a supported archive format.
2273 Returns None on failure. Raises no exceptions.
2274 """
2275 sBaseNameLower = os.path.basename(sArchive).lower();
2276
2277 #
2278 # Zip file?
2279 #
2280 if sBaseNameLower.endswith('.zip'):
2281 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2282
2283 #
2284 # Tarball?
2285 #
2286 if sBaseNameLower.endswith('.tar') \
2287 or sBaseNameLower.endswith('.tar.gz') \
2288 or sBaseNameLower.endswith('.tgz') \
2289 or sBaseNameLower.endswith('.tar.bz2'):
2290 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2291
2292 #
2293 # Cannot classify it from the name, so just return that to the caller.
2294 #
2295 fnLog('Not unpacking "%s".' % (sArchive,));
2296 return [];
2297
2298
2299#
2300# Misc.
2301#
2302def areBytesEqual(oLeft, oRight):
2303 """
2304 Compares two byte arrays, strings or whatnot.
2305
2306 returns true / false accordingly.
2307 """
2308
2309 # If both are None, consider them equal (bogus?):
2310 if oLeft is None and oRight is None:
2311 return True;
2312
2313 # If just one is None, they can't match:
2314 if oLeft is None or oRight is None:
2315 return False;
2316
2317 # If both have the same type, use the compare operator of the class:
2318 if type(oLeft) is type(oRight):
2319 #print('same type: %s' % (oLeft == oRight,));
2320 return oLeft == oRight;
2321
2322 # On the offchance that they're both strings, but of different types.
2323 if isString(oLeft) and isString(oRight):
2324 #print('string compare: %s' % (oLeft == oRight,));
2325 return oLeft == oRight;
2326
2327 #
2328 # See if byte/buffer stuff that can be compared directory. If not convert
2329 # strings to bytes.
2330 #
2331 # Note! For 2.x, we must convert both sides to the buffer type or the
2332 # comparison may fail despite it working okay in test cases.
2333 #
2334 if sys.version_info[0] >= 3:
2335 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2336 return oLeft == oRight;
2337
2338 if isString(oLeft):
2339 try: oLeft = bytes(oLeft, 'utf-8');
2340 except: pass;
2341 if isString(oRight):
2342 try: oRight = bytes(oRight, 'utf-8');
2343 except: pass;
2344 else:
2345 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2346 if isinstance(oLeft, bytearray):
2347 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2348 else:
2349 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2350 #print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2351 return oLeft == oRight;
2352
2353 if isString(oLeft):
2354 try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2355 except: pass;
2356 if isString(oRight):
2357 try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2358 except: pass;
2359
2360 # Check if we now have the same type for both:
2361 if type(oLeft) is type(oRight):
2362 #print('same type now: %s' % (oLeft == oRight,));
2363 return oLeft == oRight;
2364
2365 # Check if we now have buffer/memoryview vs bytes/bytesarray again.
2366 if sys.version_info[0] >= 3:
2367 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2368 return oLeft == oRight;
2369 else:
2370 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2371 if isinstance(oLeft, bytearray):
2372 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2373 else:
2374 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2375 #print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2376 return oLeft == oRight;
2377
2378 # Do item by item comparison:
2379 if len(oLeft) != len(oRight):
2380 #print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2381 return False;
2382 i = len(oLeft);
2383 while i > 0:
2384 i = i - 1;
2385
2386 iElmLeft = oLeft[i];
2387 if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2388 iElmLeft = ord(iElmLeft);
2389
2390 iElmRight = oRight[i];
2391 if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2392 iElmRight = ord(iElmRight);
2393
2394 if iElmLeft != iElmRight:
2395 #print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2396 return False;
2397 return True;
2398
2399
2400#
2401# Unit testing.
2402#
2403
2404# pylint: disable=missing-docstring
2405# pylint: disable=undefined-variable
2406class BuildCategoryDataTestCase(unittest.TestCase):
2407 def testIntervalSeconds(self):
2408 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2409 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2410 self.assertEqual(parseIntervalSeconds('123'), (123, None));
2411 self.assertEqual(parseIntervalSeconds(123), (123, None));
2412 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2413 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2414 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2415 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2416 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2417 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2418 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2419
2420 def testZuluNormalization(self):
2421 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2422 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2423 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2424 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2425 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2426
2427 def testHasNonAsciiChars(self):
2428 self.assertEqual(hasNonAsciiCharacters(''), False);
2429 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
2430 self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2431 self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2432 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2433 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2434 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2435 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2436 self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2437 self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2438
2439 def testAreBytesEqual(self):
2440 self.assertEqual(areBytesEqual(None, None), True);
2441 self.assertEqual(areBytesEqual(None, ''), False);
2442 self.assertEqual(areBytesEqual('', ''), True);
2443 self.assertEqual(areBytesEqual('1', '1'), True);
2444 self.assertEqual(areBytesEqual('12345', '1234'), False);
2445 self.assertEqual(areBytesEqual('1234', '1234'), True);
2446 self.assertEqual(areBytesEqual('1234', b'1234'), True);
2447 self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2448 self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2449 self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2450 self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2451 self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2452 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2453 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2454 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2455 if sys.version_info[0] >= 3:
2456 pass;
2457 else:
2458 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2459 bytearray([0x31,0x32,0x33,0x34])), True);
2460 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2461 bytearray([0x99,0x32,0x32,0x34])), False);
2462 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2463 buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2464 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2465 buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2466 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2467 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2468 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2469
2470if __name__ == '__main__':
2471 unittest.main();
2472 # not reached.
2473
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette