VirtualBox

Changeset 97266 in vbox for trunk


Ignore:
Timestamp:
Oct 23, 2022 12:06:26 AM (2 years ago)
Author:
vboxsync
Message:

ValKit/analysis: Wrote a new analyzer/comparison tool, tst-a1.py didn't work any more and even after fixing bitrot it didn't provide very useful output.

Location:
trunk
Files:
4 edited
1 copied

Legend:

Unmodified
Added
Removed
  • trunk/include/iprt/test.h

    r96407 r97266  
    490490 *                rtTestUnitName returns) for mapping.  Testmanager must be
    491491 *                updated.
     492 *              - Add it to Value.kdBestByUnit in ValidationKit/analysis/reader.py.
    492493 */
    493494typedef enum RTTESTUNIT
  • trunk/src/VBox/ValidationKit/analysis/analyze.py

    r97233 r97266  
    44
    55"""
    6 Analyzer Experiment  1.
     6Analyzer CLI.
    77"""
    88
     
    5151
    5252# Validation Kit imports.
    53 from testanalysis import reader    ## @todo fix testanalysis/__init__.py.
    54 from testanalysis import reporting
    55 from testanalysis import diff
     53from analysis import reader    ## @todo fix testanalysis/__init__.py.
     54from analysis import reporting
     55#from analysis import diff
    5656
    5757
    5858def usage():
    5959    """ Display usage """
    60     print('usage: %s [options] <testresults.xml> [baseline.xml]' % (sys.argv[0]));
     60    print('usage: %s [options] <test1.xml/txt> <test2.xml/txt> [..] [-- <baseline1.file> [baseline2.file] [..]]'
     61          % (sys.argv[0]));
    6162    print('')
    6263    print('options:')
     
    6465    return 1;
    6566
     67class ResultCollection(object):
     68    """
     69    One or more test runs that should be merged before comparison.
     70    """
     71
     72    def __init__(self, sName):
     73        self.sName       = sName;
     74        self.aoTestTrees = []   # type: [Test]
     75        self.asTestFiles = []   # type: [str] - runs parallel to aoTestTrees
     76        self.oDistilled  = None # type: Test
     77
     78    def append(self, sFilename):
     79        """
     80        Loads sFilename and appends the result.
     81        Returns True on success, False on failure.
     82        """
     83        oTestTree = reader.parseTestResult(sFilename);
     84        if oTestTree:
     85            self.aoTestTrees.append(oTestTree);
     86            self.asTestFiles.append(sFilename);
     87            return True;
     88        return False;
     89
     90    def isEmpty(self):
     91        """ Checks if the result is empty. """
     92        return len(self.aoTestTrees) == 0;
     93
     94    def filterTests(self, asFilters):
     95        """
     96        Filters all the test trees using asFilters.
     97        """
     98        for oTestTree in self.aoTestTrees:
     99            oTestTree.filterTests(asFilters);
     100        return self;
     101
     102    def distill(self, sMethod, fDropLoners = False):
     103        """
     104        Distills the set of test results into a single one by the given method.
     105
     106        Valid sMethod values:
     107            - 'best': Pick the best result for each test and value among all the test runs.
     108            - 'avg':  Calculate the average value among all the test runs.
     109
     110        When fDropLoners is True, tests and values that only appear in a single test run
     111        will be discarded.  When False (the default), the lone result will be used.
     112        """
     113        assert sMethod in ['best', 'avg'];
     114        assert not self.oDistilled;
     115
     116        # If empty, nothing to do.
     117        if self.isEmpty():
     118            return None;
     119
     120        # If there is only a single tree, make a deep copy of it.
     121        if len(self.aoTestTrees) == 1:
     122            oDistilled = self.aoTestTrees[0].clone();
     123        else:
     124
     125            # Since we don't know if the test runs are all from the same test, we create
     126            # dummy root tests for each run and use these are the start for the distillation.
     127            aoDummyInputTests = [];
     128            for oRun in self.aoTestTrees:
     129                oDummy = reader.Test();
     130                oDummy.aoChildren = [oRun,];
     131                aoDummyInputTests.append(oDummy);
     132
     133            # Similarly, we end up with a "dummy" root test for the result.
     134            oDistilled = reader.Test();
     135            oDistilled.distill(aoDummyInputTests, sMethod, fDropLoners);
     136
     137            # We can drop this if there is only a single child, i.e. if all runs are for
     138            # the same test.
     139            if len(oDistilled.aoChildren) == 1:
     140                oDistilled = oDistilled.aoChildren[0];
     141
     142        self.oDistilled = oDistilled;
     143        return oDistilled;
     144
     145
     146
     147
     148
    66149def main(asArgs):
    67     """ C styl main(). """
     150    """ C style main(). """
     151    #
    68152    # Parse arguments
    69     sTestFile = None;
    70     sBaseFile = None;
    71     asFilters = [];
     153    #
     154    oCurCollection      = ResultCollection('#0');
     155    aoCollections       = [ oCurCollection, ];
     156    iBaseline           = 0;
     157    sDistillationMethod = 'best';
     158    fBrief              = True;
     159    cPctPrecision       = 2;
     160    asFilters           = [];
     161
    72162    iArg = 1;
    73163    while iArg < len(asArgs):
     164        #print("dbg: iArg=%s '%s'" % (iArg, asArgs[iArg],));
     165        if asArgs[iArg].startswith('--help'):
     166            return usage();
    74167        if asArgs[iArg] == '--filter':
    75168            iArg += 1;
    76169            asFilters.append(asArgs[iArg]);
    77         elif asArgs[iArg].startswith('--help'):
    78             return usage();
    79         elif asArgs[iArg].startswith('--'):
    80             print('syntax error: unknown option "%s"' % (asArgs[iArg]));
    81             return usage();
    82         elif sTestFile is None:
    83             sTestFile = asArgs[iArg];
    84         elif sBaseFile is None:
    85             sBaseFile = asArgs[iArg];
     170        elif asArgs[iArg] == '--best':
     171            sDistillationMethod = 'best';
     172        elif asArgs[iArg] in ('--avg', '--average'):
     173            sDistillationMethod = 'avg';
     174        elif asArgs[iArg] == '--brief':
     175            fBrief = True;
     176        elif asArgs[iArg] == '--verbose':
     177            fBrief = False;
     178        elif asArgs[iArg] in ('--pct', '--pct-precision'):
     179            iArg += 1;
     180            cPctPrecision = int(asArgs[iArg]);
     181        elif asArgs[iArg] in ('--base', '--baseline'):
     182            iArg += 1;
     183            iBaseline = int(asArgs[iArg]);
     184        # '--' starts a new collection.  If current one is empty, drop it.
     185        elif asArgs[iArg] == '--':
     186            print("dbg: new collection");
     187            #if oCurCollection.isEmpty():
     188            #    del aoCollections[-1];
     189            oCurCollection = ResultCollection("#%s" % (len(aoCollections),));
     190            aoCollections.append(oCurCollection);
     191        # Name the current result collection.
     192        elif asArgs[iArg] == '--name':
     193            iArg += 1;
     194            oCurCollection.sName = asArgs[iArg];
     195        # Read in a file and add it to the current data set.
    86196        else:
    87             print('syntax error: too many file names: %s' % (asArgs[iArg]))
    88             return usage();
     197            if not oCurCollection.append(asArgs[iArg]):
     198                return 1;
    89199        iArg += 1;
    90200
    91     # Down to business
    92     oTestTree = reader.parseTestResult(sTestFile);
    93     if oTestTree is None:
     201    #
     202    # Post argument parsing processing.
     203    #
     204
     205    # Drop the last collection if empty.
     206    if oCurCollection.isEmpty():
     207        del aoCollections[-1];
     208    if not aoCollections:
     209        print("error: No input files given!");
    94210        return 1;
    95     oTestTree = oTestTree.filterTests(asFilters)
    96 
    97     if sBaseFile is not None:
    98         oBaseline = reader.parseTestResult(sBaseFile);
    99         if oBaseline is None:
    100             return 1;
    101         oTestTree = diff.baselineDiff(oTestTree, oBaseline);
    102         if oTestTree is None:
    103             return 1;
    104 
    105     reporting.produceTextReport(oTestTree);
     211
     212    # Check the baseline value and mark the column as such.
     213    if iBaseline < 0 or iBaseline > len(aoCollections):
     214        print("error: specified baseline is out of range: %s, valid range 0 <= baseline < %s"
     215              % (iBaseline, len(aoCollections),));
     216        return 1;
     217    aoCollections[iBaseline].sName += ' (baseline)';
     218
     219    #
     220    # Apply filtering before distilling each collection into a single result
     221    # tree and comparing them to the first one.
     222    #
     223    if asFilters:
     224        for oCollection in aoCollections:
     225            oCollection.filterTests(asFilters);
     226
     227    for oCollection in aoCollections:
     228        oCollection.distill(sDistillationMethod);
     229
     230    #
     231    # Produce the report.
     232    #
     233    oTable = reporting.RunTable(iBaseline, fBrief, cPctPrecision);
     234    oTable.populateFromRuns([oCollection.oDistilled for oCollection in aoCollections],
     235                            [oCollection.sName      for oCollection in aoCollections]);
     236    print('\n'.join(oTable.formatAsText()));
    106237    return 0;
    107238
  • trunk/src/VBox/ValidationKit/analysis/reader.py

    r96407 r97266  
    4444
    4545# Standard python imports.
    46 import os
    47 import traceback
     46import datetime;
     47import os;
     48import sys;
     49import traceback;
     50
     51# Only the main script needs to modify the path.
     52try:    __file__;
     53except: __file__ = sys.argv[0];
     54g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)));
     55sys.path.append(g_ksValidationKitDir);
     56
     57# ValidationKit imports.
     58from common import utils;
     59
     60# Python 3 hacks:
     61if sys.version_info[0] >= 3:
     62    long = int;     # pylint: disable=redefined-builtin,invalid-name
    4863
    4964# pylint: disable=missing-docstring
    5065
     66
    5167class Value(object):
    5268    """
    5369    Represents a value.  Usually this is benchmark result or parameter.
    5470    """
    55     def __init__(self, oTest, hsAttrs):
     71
     72    kdBestByUnit = {
     73        "%":                +1, # Difficult to say what's best really.
     74        "bytes":            +1, # Difficult to say what's best really.
     75        "bytes/s":          +2,
     76        "KB":               +1,
     77        "KB/s":             +2,
     78        "MB":               +1,
     79        "MB/s":             +2,
     80        "packets":          +2,
     81        "packets/s":        +2,
     82        "frames":           +2,
     83        "frames/s":         +2,
     84        "occurrences":      +1, # Difficult to say what's best really.
     85        "occurrences/s":    +2,
     86        "roundtrips":       +2,
     87        "calls":            +1, # Difficult to say what's best really.
     88        "calls/s":          +2,
     89        "s":                -2,
     90        "ms":               -2,
     91        "ns":               -2,
     92        "ns/call":          -2,
     93        "ns/frame":         -2,
     94        "ns/occurrence":    -2,
     95        "ns/packet":        -2,
     96        "ns/roundtrip":     -2,
     97        "ins":              +2,
     98        "ins/sec":          -1,
     99        "":                 +1, # Difficult to say what's best really.
     100        "pp1k":             -2,
     101        "pp10k":            -2,
     102        "ppm":              -2,
     103        "ppb":              -2,
     104        "ticks":            -1, # Difficult to say what's best really.
     105        "ticks/call":       -2,
     106        "ticks/occ":        -2,
     107        "pages":            +1, # Difficult to say what's best really.
     108        "pages/s":          +2,
     109        "ticks/page":       -2,
     110        "ns/page":          -2,
     111        "ps":               -1, # Difficult to say what's best really.
     112        "ps/call":          -2,
     113        "ps/frame":         -2,
     114        "ps/occurrence":    -2,
     115        "ps/packet":        -2,
     116        "ps/roundtrip":     -2,
     117        "ps/page":          -2,
     118    };
     119
     120    def __init__(self, oTest, sName = None, sUnit = None, sTimestamp = None, lValue = None):
    56121        self.oTest      = oTest;
    57         self.sName      = hsAttrs['name'];
    58         self.sUnit      = hsAttrs['unit'];
    59         self.sTimestamp = hsAttrs['timestamp'];
    60         self.sValue     = '';
     122        self.sName      = sName;
     123        self.sUnit      = sUnit;
     124        self.sTimestamp = sTimestamp;
     125        self.lValue     = self.valueToInteger(lValue);
     126        assert self.lValue is None or isinstance(self.lValue, (int, long)), "lValue=%s %s" % (self.lValue, type(self.lValue),);
     127
     128        # Members set by processing.
    61129        self.sDiff      = None;
    62130
    63     # parsing
    64 
    65     def addData(self, sData):
    66         self.sValue    += sData;
     131    def clone(self, oParentTest):
     132        """
     133        Clones the value.
     134        """
     135        return Value(oParentTest, self.sName, self.sUnit, self.sTimestamp, self.lValue);
     136
     137
     138    @staticmethod
     139    def valueToInteger(sValue):
     140        """
     141        Returns integer (long) represention of lValue.
     142        Returns None if it cannot be converted to integer.
     143
     144        Raises an exception if sValue isn't an integer.
     145        """
     146        if sValue is None or isinstance(sValue, (int, long)):
     147            return sValue;
     148        sValue = sValue.strip();
     149        if not sValue:
     150            return None;
     151        return long(sValue);
     152
     153    # Manipluation
     154
     155    def distill(self, aoValues, sMethod):
     156        """
     157        Distills the value of the object from values from multiple test runs.
     158        """
     159        if not aoValues:
     160            return self;
     161
     162        # Everything except the value comes from the first run.
     163        self.sName      = aoValues[0].sName;
     164        self.sTimestamp = aoValues[0].sTimestamp;
     165        self.sUnit      = aoValues[0].sUnit;
     166
     167        # Find the value to use according to sMethod.
     168        if len(aoValues) == 1:
     169            self.lValue = aoValues[0].lValue;
     170        else:
     171            alValuesXcptInvalid = [oValue.lValue for oValue in aoValues if oValue.lValue is not None];
     172            if not alValuesXcptInvalid:
     173                # No integer result, so just pick the first value whatever it is.
     174                self.lValue = aoValues[0].lValue;
     175
     176            elif sMethod == 'best':
     177                # Pick the best result out of the whole bunch.
     178                if self.kdBestByUnit[self.sUnit] >= 0:
     179                    self.lValue = max(alValuesXcptInvalid);
     180                else:
     181                    self.lValue = min(alValuesXcptInvalid);
     182
     183            elif sMethod == 'avg':
     184                # Calculate the average.
     185                self.lValue = (sum(alValuesXcptInvalid) + len(alValuesXcptInvalid) // 2) // len(alValuesXcptInvalid);
     186
     187            else:
     188                assert False;
     189                self.lValue = aoValues[0].lValue;
     190
     191        return self;
     192
    67193
    68194    # debug
    69195
    70196    def printValue(self, cIndent):
    71         print('%sValue: name=%s timestamp=%s unit=%s value="%s"'
    72               % (''.ljust(cIndent*2), self.sName, self.sTimestamp, self.sUnit, self.sValue));
     197        print('%sValue: name=%s timestamp=%s unit=%s value=%s'
     198              % (''.ljust(cIndent*2), self.sName, self.sTimestamp, self.sUnit, self.lValue));
    73199
    74200
     
    77203    Nested test result.
    78204    """
    79     def __init__(self, oParent, hsAttrs):
    80         self.aoChildren     = [];
     205    def __init__(self, oParent = None, hsAttrs = None):
     206        self.aoChildren     = [] # type: list(Test)
    81207        self.aoValues       = [];
    82208        self.oParent        = oParent;
    83         self.sName          = hsAttrs['name'];
    84         self.sStartTS       = hsAttrs['timestamp'];
     209        self.sName          = hsAttrs['name']      if hsAttrs else None;
     210        self.sStartTS       = hsAttrs['timestamp'] if hsAttrs else None;
    85211        self.sEndTS         = None;
    86212        self.sStatus        = None;
    87213        self.cErrors        = -1;
     214
     215        # Members set by processing.
    88216        self.sStatusDiff    = None;
    89217        self.cErrorsDiff    = None;
     218
     219    def clone(self, oParent = None):
     220        """
     221        Returns a deep copy.
     222        """
     223        oClone = Test(oParent, {'name': self.sName, 'timestamp': self.sStartTS});
     224
     225        for oChild in self.aoChildren:
     226            oClone.aoChildren.append(oChild.clone(oClone));
     227
     228        for oValue in self.aoValues:
     229            oClone.aoValues.append(oValue.clone(oClone));
     230
     231        oClone.sEndTS  = self.sEndTS;
     232        oClone.sStatus = self.sStatus;
     233        oClone.cErrors = self.cErrors;
     234        return oClone;
    90235
    91236    # parsing
     
    95240        return oChild;
    96241
    97     def addValue(self, hsAttrs):
    98         oValue = hsAttrs['value'];
     242    def addValue(self, oValue):
    99243        self.aoValues.append(oValue);
    100244        return oValue;
    101245
    102     def markPassed(self, hsAttrs):
    103         try:    self.sEndTS = hsAttrs['timestamp'];
    104         except: pass;
     246    def __markCompleted(self, sTimestamp):
     247        """ Sets sEndTS if not already done. """
     248        if not self.sEndTS:
     249            self.sEndTS = sTimestamp;
     250
     251    def markPassed(self, sTimestamp):
     252        self.__markCompleted(sTimestamp);
    105253        self.sStatus = 'passed';
    106254        self.cErrors = 0;
    107255
    108     def markSkipped(self, hsAttrs):
    109         try:    self.sEndTS = hsAttrs['timestamp'];
    110         except: pass;
     256    def markSkipped(self, sTimestamp):
     257        self.__markCompleted(sTimestamp);
    111258        self.sStatus = 'skipped';
    112259        self.cErrors = 0;
    113260
    114     def markFailed(self, hsAttrs):
    115         try:    self.sEndTS = hsAttrs['timestamp'];
    116         except: pass;
     261    def markFailed(self, sTimestamp, cErrors):
     262        self.__markCompleted(sTimestamp);
    117263        self.sStatus = 'failed';
    118         self.cErrors = int(hsAttrs['errors']);
    119 
    120     def markEnd(self, hsAttrs):
    121         try:    self.sEndTS = hsAttrs['timestamp'];
    122         except: pass;
     264        self.cErrors = cErrors;
     265
     266    def markEnd(self, sTimestamp, cErrors):
     267        self.__markCompleted(sTimestamp);
    123268        if self.sStatus is None:
    124             self.sStatus = 'end';
     269            self.sStatus = 'failed' if cErrors != 0 else 'end';
    125270            self.cErrors = 0;
    126271
     
    194339        return self;
    195340
     341    @staticmethod
     342    def calcDurationStatic(sStartTS, sEndTS):
     343        """
     344        Returns None the start timestamp is absent or invalid.
     345        Returns datetime.timedelta otherwise.
     346        """
     347        if not sStartTS:
     348            return None;
     349        try:
     350            oStart = utils.parseIsoTimestamp(sStartTS);
     351        except:
     352            return None;
     353
     354        if not sEndTS:
     355            return datetime.timedelta.max;
     356        try:
     357            oEnd   = utils.parseIsoTimestamp(sEndTS);
     358        except:
     359            return datetime.timedelta.max;
     360
     361        return oEnd - oStart;
     362
     363    def calcDuration(self):
     364        """
     365        Returns the duration as a datetime.timedelta object or None if not available.
     366        """
     367        return self.calcDurationStatic(self.sStartTS, self.sEndTS);
     368
     369    def calcDurationAsMicroseconds(self):
     370        """
     371        Returns the duration as microseconds or None if not available.
     372        """
     373        oDuration = self.calcDuration();
     374        if not oDuration:
     375            return None;
     376        return (oDuration.days * 86400 + oDuration.seconds) * 1000000 + oDuration.microseconds;
     377
     378    @staticmethod
     379    def distillTimes(aoTestRuns, sMethod, sStatus):
     380        """
     381        Destills the error counts of the tests.
     382        Returns a (sStartTS, sEndTS) pair.
     383        """
     384
     385        #
     386        # Start by assembling two list of start and end times for all runs that have a start timestamp.
     387        # Then sort out the special cases where no run has a start timestamp and only a single one has.
     388        #
     389        asStartTS = [oRun.sStartTS for oRun in aoTestRuns if oRun.sStartTS];
     390        if not asStartTS:
     391            return (None, None);
     392        asEndTS   = [oRun.sEndTS   for oRun in aoTestRuns if oRun.sStartTS]; # parallel to asStartTS, so we don't check sEndTS.
     393        if len(asStartTS) == 1:
     394            return (asStartTS[0], asEndTS[0]);
     395
     396        #
     397        # Calculate durations for all runs.
     398        #
     399        if sMethod == 'best':
     400            aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns if oRun.sStatus == sStatus];
     401            if not aoDurations or aoDurations.count(None) == len(aoDurations):
     402                aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns];
     403            if aoDurations.count(None) == len(aoDurations):
     404                return (asStartTS[0], None);
     405            oDuration = min([oDuration for oDuration in aoDurations if oDuration is not None]);
     406
     407        elif sMethod == 'avg':
     408            print("dbg: 0: sStatus=%s []=%s"
     409                  % (sStatus, [(Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS),oRun.sStatus) for oRun in aoTestRuns],));
     410            aoDurations = [Test.calcDurationStatic(oRun.sStartTS, oRun.sEndTS) for oRun in aoTestRuns if oRun.sStatus == sStatus];
     411            print("dbg: 1: aoDurations=%s" % (aoDurations,))
     412            aoDurations = [oDuration for oDuration in aoDurations if oDuration];
     413            print("dbg: 2: aoDurations=%s" % (aoDurations,))
     414            if not aoDurations:
     415                return (asStartTS[0], None);
     416            aoDurations = [oDuration for oDuration in aoDurations if oDuration < datetime.timedelta.max];
     417            print("dbg: 3: aoDurations=%s" % (aoDurations,))
     418            if not aoDurations:
     419                return (asStartTS[0], None);
     420            # sum doesn't work on timedelta, so do it manually.
     421            oDuration = aoDurations[0];
     422            for i in range(1, len(aoDurations)):
     423                oDuration += aoDurations[i];
     424            print("dbg: 5: oDuration=%s" % (aoDurations,))
     425            oDuration = oDuration / len(aoDurations);
     426            print("dbg: 6: oDuration=%s" % (aoDurations,))
     427
     428        else:
     429            assert False;
     430            return (asStartTS[0], asEndTS[0]);
     431
     432        # Check unfinished test.
     433        if oDuration >= datetime.timedelta.max:
     434            return (asStartTS[0], None);
     435
     436        # Calculate and format the end timestamp string.
     437        oStartTS = utils.parseIsoTimestamp(asStartTS[0]);
     438        oEndTS   = oStartTS + oDuration;
     439        return (asStartTS[0], utils.formatIsoTimestamp(oEndTS));
     440
     441    @staticmethod
     442    def distillStatus(aoTestRuns, sMethod):
     443        """
     444        Destills the status of the tests.
     445        Returns the status.
     446        """
     447        asStatuses = [oRun.sStatus for oRun in aoTestRuns];
     448
     449        if sMethod == 'best':
     450            for sStatus in ('passed', 'failed', 'skipped'):
     451                if sStatus in asStatuses:
     452                    return sStatus;
     453            return asStatuses[0];
     454
     455        if sMethod == 'avg':
     456            cPassed  = asStatuses.count('passed');
     457            cFailed  = asStatuses.count('failed');
     458            cSkipped = asStatuses.count('skipped');
     459            cEnd     = asStatuses.count('end');
     460            cNone    = asStatuses.count(None);
     461            if cPassed >= cFailed and cPassed >= cSkipped and cPassed >= cNone and cPassed >= cEnd:
     462                return 'passed';
     463            if cFailed >= cPassed and cFailed >= cSkipped and cFailed >= cNone and cFailed >= cEnd:
     464                return 'failed';
     465            if cSkipped >= cPassed and cSkipped >= cFailed and cSkipped >= cNone and cSkipped >= cEnd:
     466                return 'skipped';
     467            if cEnd >= cPassed and cEnd >= cFailed and cEnd >= cNone and cEnd >= cSkipped:
     468                return 'end';
     469            return None;
     470
     471        assert False;
     472        return asStatuses[0];
     473
     474    @staticmethod
     475    def distillErrors(aoTestRuns, sMethod):
     476        """
     477        Destills the error counts of the tests.
     478        Returns the status.
     479        """
     480        acErrorsXcptNeg = [oRun.cErrors for oRun in aoTestRuns if oRun.cErrors >= 0];
     481
     482        if sMethod == 'best':
     483            if acErrorsXcptNeg:
     484                return min(acErrorsXcptNeg);
     485        elif sMethod == 'avg':
     486            if acErrorsXcptNeg:
     487                return sum(acErrorsXcptNeg) // len(acErrorsXcptNeg);
     488        else:
     489            assert False;
     490        return -1;
     491
     492    def distill(self, aoTestRuns, sMethod, fDropLoners):
     493        """
     494        Distills the test runs into this test.
     495        """
     496        #
     497        # Recurse first (before we create too much state in the stack
     498        # frame) and do child tests.
     499        #
     500        # We copy the child lists of each test run so we can remove tests we've
     501        # processed from each run and thus make sure we include tests in
     502        #
     503        #
     504        aaoChildren = [list(oRun.aoChildren) for oRun in aoTestRuns];
     505
     506        # Process the tests for each run.
     507        for i, _ in enumerate(aaoChildren):
     508            # Process all tests for the current run.
     509            while len(aaoChildren[i]) > 0:
     510                oFirst = aaoChildren[i].pop(0);
     511
     512                # Build a list of sub-test runs by searching remaining runs by test name.
     513                aoSameSubTests = [oFirst,];
     514                for j in range(i + 1, len(aaoChildren)):
     515                    aoThis = aaoChildren[j];
     516                    for iThis, oThis in enumerate(aoThis):
     517                        if oThis.sName == oFirst.sName:
     518                            del aoThis[iThis];
     519                            aoSameSubTests.append(oThis);
     520                            break;
     521
     522                # Apply fDropLoners.
     523                if not fDropLoners or len(aoSameSubTests) > 1 or len(aaoChildren) == 1:
     524                    # Create an empty test and call distill on it with the subtest array, unless
     525                    # of course that the array only has one member and we can simply clone it.
     526                    if len(aoSameSubTests) == 1:
     527                        self.addChild(oFirst.clone(self));
     528                    else:
     529                        oSubTest = Test(self);
     530                        oSubTest.sName = oFirst.sName;
     531                        oSubTest.distill(aoSameSubTests, sMethod, fDropLoners);
     532                        self.addChild(oSubTest);
     533        del aaoChildren;
     534
     535        #
     536        # Do values.  Similar approch as for the sub-tests.
     537        #
     538        aaoValues = [list(oRun.aoValues) for oRun in aoTestRuns];
     539
     540        # Process the values for each run.
     541        for i,_ in enumerate(aaoValues):
     542            # Process all values for the current run.
     543            while len(aaoValues[i]) > 0:
     544                oFirst = aaoValues[i].pop(0);
     545
     546                # Build a list of values runs by searching remaining runs by value name and unit.
     547                aoSameValues = [oFirst,];
     548                for j in range(i + 1, len(aaoValues)):
     549                    aoThis = aaoValues[j];
     550                    for iThis, oThis in enumerate(aoThis):
     551                        if oThis.sName == oFirst.sName and oThis.sUnit == oFirst.sUnit:
     552                            del aoThis[iThis];
     553                            aoSameValues.append(oThis);
     554                            break;
     555
     556                # Apply fDropLoners.
     557                if not fDropLoners or len(aoSameValues) > 1 or len(aaoValues) == 1:
     558                    # Create an empty test and call distill on it with the subtest array, unless
     559                    # of course that the array only has one member and we can simply clone it.
     560                    if len(aoSameValues) == 1:
     561                        self.aoValues.append(oFirst.clone(self));
     562                    else:
     563                        oValue = Value(self);
     564                        oValue.distill(aoSameValues, sMethod);
     565                        self.aoValues.append(oValue);
     566        del aaoValues;
     567
     568        #
     569        # Distill test properties.
     570        #
     571        self.sStatus = self.distillStatus(aoTestRuns, sMethod);
     572        self.cErrors = self.distillErrors(aoTestRuns, sMethod);
     573        (self.sStartTS, self.sEndTS) = self.distillTimes(aoTestRuns, sMethod, self.sStatus);
     574        print("dbg: %s: sStartTS=%s, sEndTS=%s" % (self.sName, self.sStartTS, self.sEndTS));
     575
     576        return self;
     577
    196578
    197579class XmlLogReader(object):
     
    209591    def parse(self):
    210592        try:
    211             oFile = open(self.sXmlFile, 'r');
     593            oFile = open(self.sXmlFile, 'rb'); # pylint: disable=consider-using-with
    212594        except:
    213595            traceback.print_exc();
     
    229611
    230612    def handleElementStart(self, sName, hsAttrs):
    231         #print '%s%s: %s' % (''.ljust(self.iLevel * 2), sName, str(hsAttrs));
     613        #print('%s%s: %s' % (''.ljust(self.iLevel * 2), sName, str(hsAttrs)));
    232614        if sName in ('Test', 'SubTest',):
    233615            self.iLevel += 1;
    234616            self.oTest = self.oTest.addChild(Test(self.oTest, hsAttrs));
    235617        elif sName == 'Value':
    236             self.oValue = self.oTest.addValue(hsAttrs);
     618            self.oValue = self.oTest.addValue(Value(self.oTest, hsAttrs.get('name'), hsAttrs.get('unit'),
     619                                                    hsAttrs.get('timestamp'), hsAttrs.get('value')));
    237620        elif sName == 'End':
    238             self.oTest.markEnd(hsAttrs);
     621            self.oTest.markEnd(hsAttrs.get('timestamp'), int(hsAttrs.get('errors', '0')));
    239622        elif sName == 'Passed':
    240             self.oTest.markPassed(hsAttrs);
     623            self.oTest.markPassed(hsAttrs.get('timestamp'));
    241624        elif sName == 'Skipped':
    242             self.oTest.markSkipped(hsAttrs);
     625            self.oTest.markSkipped(hsAttrs.get('timestamp'));
    243626        elif sName == 'Failed':
    244             self.oTest.markFailed(hsAttrs);
     627            self.oTest.markFailed(hsAttrs.get('timestamp'), int(hsAttrs['errors']));
    245628        elif sName == 'Include':
    246629            self.handleInclude(hsAttrs);
    247630        else:
    248             print('Unknonwn element "%s"' % (sName,));
     631            print('Unknown element "%s"' % (sName,));
    249632
    250633    def handleElementData(self, sData):
     
    297680    oXlr = XmlLogReader(sXmlFile);
    298681    if oXlr.parse():
     682        if len(oXlr.oRoot.aoChildren) == 1 and not oXlr.oRoot.aoValues:
     683            return oXlr.oRoot.aoChildren[0];
    299684        return oXlr.oRoot;
    300685    return None;
  • trunk/src/VBox/ValidationKit/analysis/reporting.py

    r96407 r97266  
    4040SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
    4141"""
     42
    4243__version__ = "$Revision$"
    43 __all__     = ['HtmlReport', 'RstReport', 'TextReport'];
    44 
     44
     45# Standard python imports.
     46import os;
     47import sys;
     48
     49# Only the main script needs to modify the path.
     50try:    __file__;
     51except: __file__ = sys.argv[0];
     52g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)));
     53sys.path.append(g_ksValidationKitDir);
     54
     55# ValidationKit imports.
     56from common import utils;
     57
     58# Python 3 hacks:
     59if sys.version_info[0] >= 3:
     60    long = int;     # pylint: disable=redefined-builtin,invalid-name
     61
     62
     63
     64##################################################################################################################################
     65#   Old Carp                                                                                                                     #
     66##################################################################################################################################
    4567
    4668def tryAddThousandSeparators(sPotentialInterger):
     
    7294
    7395
    74 class Table(object):
    75     """
    76     A table as a header as well as data rows, thus this class.
     96class OldTable(object):
     97    """
     98    A table has a header as well as data rows, thus this class.
    7799    """
    78100    def __init__(self, oTest, fSplitDiff):
     
    134156        if len(oTable.asHeader) != len(self.asHeader):
    135157            return False;
    136         for i in range(len(self.asHeader)):
    137             if self.asHeader[i] != oTable.asHeader[i]:
     158        for i, sHdr in enumerate(self.asHeader):
     159            if sHdr != oTable.asHeader[i]:
    138160                return False;
    139161            if self.asUnits[i]  != oTable.asUnits[i]:
     
    148170    # manipulation and stuff
    149171
    150     def optimizeUnit(self):
     172    def optimizeUnits(self):
    151173        """ Turns bytes into KB, MB or GB. """
    152174        ## @todo
    153         pass;
     175        return None;
    154176
    155177    def addThousandSeparators(self):
    156178        """ Adds thousand separators to make numbers more readable. """
    157         for iRow in range(len(self.aasRows)):
    158             for iColumn in range(1, len(self.aasRows[iRow])):
    159                 asValues = self.aasRows[iRow][iColumn].split('|');
    160                 for i in range(len(asValues)):
    161                     asValues[i] = tryAddThousandSeparators(asValues[i]);
    162                 self.aasRows[iRow][iColumn] = '|'.join(asValues);
     179        for asRow in self.aasRows:
     180            for iColumn in range(1, len(asRow)):
     181                asValues = asRow[iColumn].split('|');
     182                for i, sValue in enumerate(asValues):
     183                    asValues[i] = tryAddThousandSeparators(sValue);
     184                asRow[iColumn] = '|'.join(asValues);
    163185        return True;
    164186
     
    167189        # Header is first.
    168190        acchColumns = [];
    169         for i in range(len(self.asHeader)):
     191        for i, sHdr in enumerate(self.asHeader):
    170192            cch = 1;
    171             asWords = self.asHeader[i].split();
     193            if not isinstance(sHdr, str): print("dbg: %s" % (sHdr,));
     194            asWords = sHdr.split();
    172195            for s in asWords:
    173196                if len(s) > cch:
     
    179202        # Check out all cells.
    180203        for asColumns in self.aasRows:
    181             for i in range(len(asColumns)):
    182                 if len(asColumns[i]) > acchColumns[i]:
    183                     acchColumns[i] = len(asColumns[i]);
     204            for i, sCol in enumerate(asColumns):
     205                if len(sCol) > acchColumns[i]:
     206                    acchColumns[i] = len(sCol);
    184207        return acchColumns;
    185 
    186208
    187209def tabelizeTestResults(oTest, fSplitDiff):
     
    209231                aoTables[len(aoTables) - 1].addRow(oCurTest, fSplitDiff);
    210232            else:
    211                 aoTables.append(Table(oCurTest, fSplitDiff));
     233                aoTables.append(OldTable(oCurTest, fSplitDiff));
    212234
    213235    # Pass 2 - Combine tables with the same heading.
    214236    aoTables2 = [];
    215237    for oTable in aoTables:
    216         for i in range(len(aoTables2)):
    217             if aoTables2[i].hasTheSameHeadingAsTable(oTable):
    218                 aoTables2[i].appendTable(oTable);
     238        for oTable2 in aoTables2:
     239            if oTable2.hasTheSameHeadingAsTable(oTable):
     240                oTable2.appendTable(oTable);
    219241                oTable = None;
    220242                break;
     
    224246    return aoTables2;
    225247
     248
    226249def produceHtmlReport(oTest):
    227250    """
     
    231254    return False;
    232255
     256
    233257def produceReStructuredTextReport(oTest):
    234258    """
     
    237261    print('not implemented: %s' % (oTest));
    238262    return False;
     263
    239264
    240265def produceTextReport(oTest):
     
    265290            aasHeader[0].append('');
    266291
    267         for iColumn in range(len(oTable.asHeader)):
     292        for iColumn, _ in enumerate(oTable.asHeader):
    268293            asWords = oTable.asHeader[iColumn].split();
    269294            iLine   = 0;
     
    283308        for asLine in aasHeader:
    284309            sLine = '';
    285             for i in range(len(asLine)):
     310            for i,_ in enumerate(asLine):
    286311                if i > 0: sLine += '  ';
    287312                sLine += asLine[i].center(acchColumns[i]);
     
    290315        # Units.
    291316        sLine = '';
    292         for i in range(len(oTable.asUnits)):
     317        for i,_ in enumerate(oTable.asUnits):
    293318            if i > 0: sLine += '  ';
    294319            sLine += oTable.asUnits[i].center(acchColumns[i]);
     
    311336    return None;
    312337
     338
     339
     340##################################################################################################################################
     341#   Run Table                                                                                                                    #
     342##################################################################################################################################
     343
     344def alignTextLeft(sText, cchWidth):
     345    """ Left aligns text and pads it to cchWidth characters length. """
     346    return sText + ' ' * (cchWidth - min(len(sText), cchWidth));
     347
     348
     349def alignTextRight(sText, cchWidth):
     350    """ Right aligns text and pads it to cchWidth characters length. """
     351    return ' ' * (cchWidth - min(len(sText), cchWidth)) + sText;
     352
     353
     354def alignTextCenter(sText, cchWidth):
     355    """ Pads the text equally on both sides to cchWidth characters length. """
     356    return alignTextLeft(' ' * ((cchWidth - min(len(sText), cchWidth)) // 2) + sText, cchWidth);
     357
     358
     359g_kiAlignLeft   = -1;
     360g_kiAlignRight  = 1;
     361g_kiAlignCenter = 0;
     362def alignText(sText, cchWidth, iAlignType):
     363    """
     364    General alignment method.
     365
     366    Negative iAlignType for left aligning, zero for entered, and positive for
     367    right aligning the text.
     368    """
     369    if iAlignType < 0:
     370        return alignTextLeft(sText, cchWidth);
     371    if iAlignType > 0:
     372        return alignTextRight(sText, cchWidth);
     373    return alignTextCenter(sText, cchWidth);
     374
     375
     376class TextColumnWidth(object):
     377    """
     378    Tracking the width of a column, dealing with sub-columns and such.
     379    """
     380
     381    def __init__(self):
     382        self.cch      = 0;
     383        self.dacchSub = {};
     384
     385    def update(self, oWidth, cchSubColSpacing = 1):
     386        """
     387        Updates the column width tracking with oWidth, which is either
     388        an int or an array of ints (sub columns).
     389        """
     390        if isinstance(oWidth, int):
     391            self.cch = max(self.cch, oWidth);
     392        else:
     393            cSubCols = len(oWidth);
     394            if cSubCols not in self.dacchSub:
     395                self.dacchSub[cSubCols] = list(oWidth);
     396                self.cch = max(self.cch, sum(oWidth) + cchSubColSpacing * (cSubCols - 1));
     397            else:
     398                acchSubCols = self.dacchSub[cSubCols];
     399                for iSub in range(cSubCols):
     400                    acchSubCols[iSub] = max(acchSubCols[iSub], oWidth[iSub]);
     401                self.cch = max(self.cch, sum(acchSubCols) + cchSubColSpacing * (cSubCols - 1));
     402
     403    def finalize(self):
     404        """ Finalizes sub-column sizes. """
     405        ## @todo maybe do something here, maybe not...
     406        return self;
     407
     408    def hasSubColumns(self):
     409        """ Checks if there are sub-columns for this column. """
     410        return not self.dacchSub;
     411
     412class TextWidths(object):
     413    """
     414    Tracks the column widths for text rending of the table.
     415    """
     416    def __init__(self, cchSubColSpacing = 1, ):
     417        self.cchName          = 1;
     418        self.aoColumns        = [] # type: TextColumnWidth
     419        self.cchSubColSpacing = cchSubColSpacing;
     420        self.fFinalized       = False;
     421
     422    def update(self, aoWidths):
     423        """ Updates the tracker with the returns of calcColumnWidthsForText. """
     424        if not aoWidths[0]:
     425            self.cchName = max(self.cchName, aoWidths[1]);
     426
     427            for iCol, oWidth in enumerate(aoWidths[2]):
     428                if iCol >= len(self.aoColumns):
     429                    self.aoColumns.append(TextColumnWidth());
     430                self.aoColumns[iCol].update(oWidth, self.cchSubColSpacing);
     431
     432        return self;
     433
     434    def finalize(self):
     435        """ Finalizes sub-column sizes. """
     436        for oColumnWidth in self.aoColumns:
     437            oColumnWidth.finalize();
     438        self.fFinalized = True;
     439        return self;
     440
     441    def getColumnWidth(self, iColumn, cSubs = None, iSub = None):
     442        """ Returns the width of the specified column. """
     443        if not self.fFinalized:
     444            return 0;
     445        assert iColumn < len(self.aoColumns), "iColumn=%s vs %s" % (iColumn, len(self.aoColumns),);
     446        oColumn = self.aoColumns[iColumn];
     447        if cSubs is not None:
     448            assert iSub < cSubs;
     449            if cSubs != 1:
     450                assert cSubs in oColumn.dacchSub, \
     451                       "iColumn=%s cSubs=%s iSub=%s; dacchSub=%s" % (iColumn, cSubs, iSub, oColumn.dacchSub);
     452                return oColumn.dacchSub[cSubs][iSub];
     453        return oColumn.cch;
     454
     455
     456class TextElement(object):
     457    """
     458    A text element (cell/sub-cell in a table).
     459    """
     460
     461    def __init__(self, sText = '', iAlign = g_kiAlignRight): # type: (str, int) -> None
     462        self.sText  = sText;
     463        self.iAlign = iAlign;
     464
     465
     466class RunRow(object):
     467    """
     468    Run table row.
     469    """
     470
     471    def __init__(self, iLevel, sName, iRun = 0): # type: (int, str, int) -> None
     472        self.iLevel     = iLevel;
     473        self.sName      = sName;
     474        self.iFirstRun  = iRun;
     475
     476        # Fields used while formatting (set during construction or calcColumnWidthsForText/Html).
     477        self.cColumns   = 0;                    ##< Number of columns.
     478        self.fSkip      = False                 ##< Whether or not to skip this row in the output.
     479
     480    # Format as Text:
     481
     482    def formatNameAsText(self, cchWidth): # (int) -> str
     483        """ Format the row as text. """
     484        _ = cchWidth;
     485        return  ' ' * (self.iLevel * 2) + self.sName;
     486
     487    def getColumnCountAsText(self, oTable):
     488        """
     489        Called by calcColumnWidthsForText for getting an up-to-date self.cColumns value.
     490        Override this to update cColumns after construction.
     491        """
     492        _ = oTable;
     493        return self.cColumns;
     494
     495    def formatColumnAsText(self, iColumn, oTable): # type: (int, RunTable) -> [TextElement]
     496        """ Returns an array of TextElements for the given column in this row. """
     497        _ = iColumn; _ = oTable;
     498        return [ TextElement(),];
     499
     500    def calcColumnWidthsForText(self, oTable):
     501        """
     502        Calculates the column widths for text rendering.
     503
     504        Returns a tuple consisting of the fSkip, the formatted name width, and an
     505        array of column widths.  The entries in the latter are either integer
     506        widths or arrays of subcolumn integer widths.
     507        """
     508        aoRetCols  = [];
     509        cColumns   = self.getColumnCountAsText(oTable);
     510        for iColumn in range(cColumns):
     511            aoSubColumns = self.formatColumnAsText(iColumn, oTable);
     512            if len(aoSubColumns) == 1:
     513                aoRetCols.append(len(aoSubColumns[0].sText));
     514            else:
     515                aoRetCols.append([len(oSubColumn.sText) for oSubColumn in aoSubColumns]);
     516        return (False, len(self.formatNameAsText(0)) + self.iLevel * 2, aoRetCols);
     517
     518    @staticmethod
     519    def formatDiffAsText(lNumber, lBaseline):
     520        """ Formats the difference between lNumber and lBaseline as text. """
     521        if lNumber is not None:
     522            if lBaseline is not None:
     523                if lNumber < lBaseline:
     524                    return '-' + utils.formatNumber(lBaseline - lNumber); ## @todo formatter is busted for negative nums.
     525                if lNumber > lBaseline:
     526                    return '+' + utils.formatNumber(lNumber - lBaseline);
     527                return '0';
     528        return '';
     529
     530    @staticmethod
     531    def formatDiffInPctAsText(lNumber, lBaseline, cPctPrecision):
     532        """ Formats the difference between lNumber and lBaseline in precent as text. """
     533        if lNumber is not None:
     534            if lBaseline is not None:
     535                ## @todo implement cPctPrecision
     536                if lNumber == lBaseline:
     537                    return '0.' + '0'*cPctPrecision + '%';
     538
     539                lDiff  = lNumber - lBaseline;
     540                chSign = '+';
     541                if lDiff < 0:
     542                    lDiff  = -lDiff;
     543                    chSign = '-';
     544
     545                rdPct = lDiff / float(lBaseline);
     546                #if rdPct * 100 >= 5:
     547                #    return '%s%s%%' % (chSign, utils.formatNumber(int(rdPct * 100 + 0.5)),);
     548                #if rdPct * 1000 >= 5:
     549                #    return u'%s%s\u2030'  % (chSign, int(rdPct * 1000 + 0.5),);
     550                #if rdPct * 10000 >= 5:
     551                #    return u'%s%s\u2031' % (chSign, int(rdPct * 10000 + 0.5),);
     552                #if rdPct * 1000000 >= 0.5:
     553                #    return u'%s%sppm'   % (chSign, int(rdPct * 1000000 + 0.5),);
     554
     555                if rdPct * 100 >= 100:
     556                    return '%s%s%%' % (chSign, utils.formatNumber(int(rdPct * 100 + 0.5)),);
     557                if rdPct * 10000 + 0.5 >= 1:
     558                    return '%s%.*f%%' % (chSign, cPctPrecision, rdPct * 100 + 0.005,);
     559
     560                return '~' + chSign + '0.' + '0' * cPctPrecision + '%';
     561        return '';
     562
     563
     564class RunTestRow(RunRow):
     565    """
     566    Run table test row.
     567    """
     568
     569    def __init__(self, iLevel, oTest, iRun, aoTests = None): # type: (int, reader.Test, int, [reader.Test]) -> None
     570        RunRow.__init__(self, iLevel, oTest.sName, iRun);
     571        assert oTest;
     572        self.oTest = oTest;
     573        if aoTests is None:
     574            aoTests = [None for i in range(iRun)];
     575            aoTests.append(oTest);
     576        else:
     577            aoTests= list(aoTests);
     578        self.aoTests = aoTests
     579
     580    def isSameTest(self, oTest):
     581        """ Checks if oTest belongs to this row or not. """
     582        return oTest.sName == self.oTest.sName;
     583
     584    def getBaseTest(self, oTable):
     585        """ Returns the baseline test. """
     586        oBaseTest = self.aoTests[oTable.iBaseline];
     587        if not oBaseTest:
     588            oBaseTest = self.aoTests[self.iFirstRun];
     589        return oBaseTest;
     590
     591
     592class RunTestStartRow(RunTestRow):
     593    """
     594    Run table start of test row.
     595    """
     596
     597    def __init__(self, iLevel, oTest, iRun): # type: (int, reader.Test, int) -> None
     598        RunTestRow.__init__(self, iLevel, oTest, iRun);
     599
     600class RunTestEndRow(RunTestRow):
     601    """
     602    Run table end of test row.
     603    """
     604
     605    def __init__(self, oStartRow): # type: (RunTestStartRow) -> None
     606        RunTestRow.__init__(self, oStartRow.iLevel, oStartRow.oTest, oStartRow.iFirstRun, oStartRow.aoTests);
     607        self.oStartRow = oStartRow # type: RunTestStartRow
     608
     609    def getColumnCountAsText(self, oTable):
     610        self.cColumns = len(self.aoTests);
     611        return self.cColumns;
     612
     613    def formatColumnAsText(self, iColumn, oTable):
     614        oTest = self.aoTests[iColumn];
     615        if oTest and oTest.sStatus:
     616            if oTest.cErrors > 0:
     617                return [ TextElement(oTest.sStatus, g_kiAlignCenter),
     618                         TextElement(utils.formatNumber(oTest.cErrors) + 'errors') ];
     619            return [ TextElement(oTest.sStatus, g_kiAlignCenter) ];
     620        return [ TextElement(), ];
     621
     622
     623class RunTestEndRow2(RunTestRow):
     624    """
     625    Run table 2nd end of test row, this shows the times.
     626    """
     627
     628    def __init__(self, oStartRow): # type: (RunTestStartRow) -> None
     629        RunTestRow.__init__(self, oStartRow.iLevel, oStartRow.oTest, oStartRow.iFirstRun, oStartRow.aoTests);
     630        self.oStartRow = oStartRow # type: RunTestStartRow
     631
     632    def formatNameAsText(self, cchWidth):
     633        _ = cchWidth;
     634        return '';
     635
     636    def getColumnCountAsText(self, oTable):
     637        self.cColumns = len(self.aoTests);
     638        return self.cColumns;
     639
     640    def formatColumnAsText(self, iColumn, oTable):
     641        oTest = self.aoTests[iColumn];
     642        if oTest:
     643            cUsElapsed = oTest.calcDurationAsMicroseconds();
     644            if cUsElapsed:
     645                oBaseTest = self.getBaseTest(oTable);
     646                if oTest is oBaseTest:
     647                    return [ TextElement(utils.formatNumber(cUsElapsed)), TextElement('us', g_kiAlignLeft), ];
     648                cUsElapsedBase = oBaseTest.calcDurationAsMicroseconds();
     649                aoRet = [
     650                    TextElement(utils.formatNumber(cUsElapsed)),
     651                    TextElement(self.formatDiffAsText(cUsElapsed, cUsElapsedBase)),
     652                    TextElement(self.formatDiffInPctAsText(cUsElapsed, cUsElapsedBase, oTable.cPctPrecision)),
     653                ];
     654                return aoRet[1:] if oTable.fBrief else aoRet;
     655        return [ TextElement(), ];
     656
     657class RunValueRow(RunRow):
     658    """
     659    Run table value row.
     660    """
     661
     662    def __init__(self, iLevel, oValue, iRun): # type: (int, reader.Value, int) -> None
     663        RunRow.__init__(self, iLevel, oValue.sName, iRun);
     664        self.oValue   = oValue;
     665        self.aoValues = [None for i in range(iRun)];
     666        self.aoValues.append(oValue);
     667
     668    def isSameValue(self, oValue):
     669        """ Checks if oValue belongs to this row or not. """
     670        return oValue.sName == self.oValue.sName and oValue.sUnit == self.oValue.sUnit;
     671
     672    # Formatting as Text.
     673
     674    @staticmethod
     675    def formatOneValueAsText(oValue): # type: (reader.Value) -> str
     676        """ Formats a value. """
     677        if not oValue:
     678            return "N/A";
     679        return utils.formatNumber(oValue.lValue);
     680
     681    def getBaseValue(self, oTable):
     682        """ Returns the base value instance. """
     683        oBaseValue = self.aoValues[oTable.iBaseline];
     684        if not oBaseValue:
     685            oBaseValue = self.aoValues[self.iFirstRun];
     686        return oBaseValue;
     687
     688    def getColumnCountAsText(self, oTable):
     689        self.cColumns = len(self.aoValues);
     690        return self.cColumns;
     691
     692    def formatColumnAsText(self, iColumn, oTable):
     693        oValue     = self.aoValues[iColumn];
     694        oBaseValue = self.getBaseValue(oTable);
     695        if oValue is oBaseValue:
     696            return [ TextElement(self.formatOneValueAsText(oValue)),
     697                     TextElement(oValue.sUnit, g_kiAlignLeft), ];
     698        aoRet = [
     699            TextElement(self.formatOneValueAsText(oValue)),
     700            TextElement(self.formatDiffAsText(oValue.lValue if oValue else None, oBaseValue.lValue)),
     701            TextElement(self.formatDiffInPctAsText(oValue.lValue if oValue else None, oBaseValue.lValue, oTable.cPctPrecision))
     702        ];
     703        return aoRet[1:] if oTable.fBrief else aoRet;
     704
     705
     706class RunTable(object):
     707    """
     708    Result table.
     709
     710    This contains one or more test runs as columns.
     711    """
     712
     713    def __init__(self, iBaseline = 0, fBrief = True, cPctPrecision = 2): # (int, bool, int) -> None
     714        self.asColumns      = []            # type: [str]       ## Column names.
     715        self.aoRows         = []            # type: [RunRow]    ## The table rows.
     716        self.iBaseline      = iBaseline     # type: int         ## Which column is the baseline when diffing things.
     717        self.fBrief         = fBrief        # type: bool        ## Whether to exclude the numerical values of non-baseline runs.
     718        self.cPctPrecision  = cPctPrecision # type: int         ## Number of decimal points in diff percentage value.
     719
     720    def __populateFromValues(self, aaoValueRuns, iLevel): # type: ([reader.Value]) -> None
     721        """
     722        Internal worker for __populateFromRuns()
     723
     724        This will modify the sub-lists inside aaoValueRuns, returning with the all empty.
     725        """
     726        # Same as for __populateFromRuns, only no recursion.
     727        for iValueRun, aoValuesForRun in enumerate(aaoValueRuns):
     728            while aoValuesForRun:
     729                oRow = RunValueRow(iLevel, aoValuesForRun.pop(0), iValueRun);
     730                self.aoRows.append(oRow);
     731
     732                # Pop matching values from the other runs of this test.
     733                for iOtherRun in range(iValueRun + 1, len(aaoValueRuns)):
     734                    aoValuesForOtherRun = aaoValueRuns[iOtherRun];
     735                    for iValueToPop, oOtherValue in enumerate(aoValuesForOtherRun):
     736                        if oRow.isSameValue(oOtherValue):
     737                            oRow.aoValues.append(aoValuesForOtherRun.pop(iValueToPop));
     738                            break;
     739                    if len(oRow.aoValues) <= iOtherRun:
     740                        oRow.aoValues.append(None);
     741        return self;
     742
     743    def __populateFromRuns(self, aaoTestRuns, iLevel): # type: ([reader.Test]) -> None
     744        """
     745        Internal worker for populateFromRuns()
     746
     747        This will modify the sub-lists inside aaoTestRuns, returning with the all empty.
     748        """
     749
     750        #
     751        # Currently doing depth first, so values are always at the end.
     752        # Nominally, we should inject values according to the timestamp.
     753        # However, that's too much work right now and can be done later if needed.
     754        #
     755        for iRun, aoTestForRun in enumerate(aaoTestRuns):
     756            while aoTestForRun:
     757                # Pop the next test and create a start-test row for it.
     758                oStartRow = RunTestStartRow(iLevel, aoTestForRun.pop(0), iRun);
     759                self.aoRows.append(oStartRow);
     760
     761                # Pop matching tests from the other runs.
     762                for iOtherRun in range(iRun + 1, len(aaoTestRuns)):
     763                    aoOtherTestRun = aaoTestRuns[iOtherRun];
     764                    for iTestToPop, oOtherTest in enumerate(aoOtherTestRun):
     765                        if oStartRow.isSameTest(oOtherTest):
     766                            oStartRow.aoTests.append(aoOtherTestRun.pop(iTestToPop));
     767                            break;
     768                    if len(oStartRow.aoTests) <= iOtherRun:
     769                        oStartRow.aoTests.append(None);
     770
     771                # Now recrusively do the subtests for it and then do the values.
     772                self.__populateFromRuns(  [list(oTest.aoChildren) if oTest else list() for oTest in oStartRow.aoTests], iLevel+1);
     773                self.__populateFromValues([list(oTest.aoValues)   if oTest else list() for oTest in oStartRow.aoTests], iLevel+1);
     774
     775                # Add the end-test row for it.
     776                self.aoRows.append(RunTestEndRow(oStartRow));
     777                self.aoRows.append(RunTestEndRow2(oStartRow));
     778
     779        return self;
     780
     781    def populateFromRuns(self, aoTestRuns, asRunNames = None): # type: ([reader.Test], [str]) -> RunTable
     782        """
     783        Populates the table from the series of runs.
     784
     785        The aoTestRuns and asRunNames run in parallel.  If the latter isn't
     786        given, the names will just be ordinals starting with #0 for the
     787        first column.
     788
     789        Returns self.
     790        """
     791        #
     792        # Deal with the column names first.
     793        #
     794        if asRunNames:
     795            self.asColumns = list(asRunNames);
     796        else:
     797            self.asColumns = [];
     798        iCol = len(self.asColumns);
     799        while iCol < len(aoTestRuns):
     800            self.asColumns.append('#%u%s' % (iCol, ' (baseline)' if iCol == self.iBaseline else '',));
     801
     802        #
     803        # Now flatten the test trees into a table.
     804        #
     805        self.__populateFromRuns([[oTestRun,] for oTestRun in aoTestRuns], 0);
     806        return self;
     807
     808    #
     809    # Text formatting.
     810    #
     811
     812    def formatAsText(self):
     813        """
     814        Formats the table as text.
     815
     816        Returns a string array of the output lines.
     817        """
     818
     819        #
     820        # Pass 1: Calculate column widths.
     821        #
     822        oWidths = TextWidths(1);
     823        for oRow in self.aoRows:
     824            oWidths.update(oRow.calcColumnWidthsForText(self));
     825        oWidths.finalize();
     826
     827        #
     828        # Pass 2: Generate the output strings.
     829        #
     830        # Header
     831        asRet = [
     832            alignTextCenter('Test / Value', oWidths.cchName) + ':  '
     833            + ' | '.join([alignTextCenter(sText, oWidths.getColumnWidth(iCol)) for iCol, sText in enumerate(self.asColumns)]),
     834        ];
     835        asRet.append('=' * len(asRet[0]));
     836
     837        # The table
     838        for oRow in self.aoRows:
     839            if not oRow.fSkip:
     840                sRow = oRow.formatNameAsText(oWidths.cchName);
     841                sRow = sRow + ' ' * (oWidths.cchName - min(len(sRow), oWidths.cchName)) + ': ';
     842
     843                for iColumn in range(oRow.cColumns):
     844                    aoSubCols = oRow.formatColumnAsText(iColumn, self);
     845                    sCell = '';
     846                    for iSub, oText in enumerate(aoSubCols):
     847                        cchWidth = oWidths.getColumnWidth(iColumn, len(aoSubCols), iSub);
     848                        if iSub > 0:
     849                            sCell += ' ' * oWidths.cchSubColSpacing;
     850                        sCell += alignText(oText.sText, cchWidth, oText.iAlign);
     851                    cchWidth = oWidths.getColumnWidth(iColumn);
     852                    sRow  += (' | ' if iColumn > 0 else ' ') + ' ' * (cchWidth - min(cchWidth, len(sCell))) + sCell;
     853
     854            asRet.append(sRow);
     855
     856        # Footer?
     857        if len(asRet) > 40:
     858            asRet.append(asRet[1]);
     859            asRet.append(asRet[0]);
     860
     861        return asRet;
     862
  • trunk/src/VBox/ValidationKit/analysis/tst-a1.py

    r96407 r97266  
    5151
    5252# Validation Kit imports.
    53 from testanalysis import reader    ## @todo fix testanalysis/__init__.py.
    54 from testanalysis import reporting
    55 from testanalysis import diff
     53from analysis import reader    ## @todo fix testanalysis/__init__.py.
     54from analysis import reporting
     55from analysis import diff
    5656
    5757
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette