VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/analysis/analyze.py

Last change on this file was 106061, checked in by vboxsync, 3 months ago

Copyright year updates by scm.

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 17.3 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: analyze.py 106061 2024-09-16 14:03:52Z vboxsync $
4
5"""
6Analyzer CLI.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2010-2024 Oracle and/or its affiliates.
12
13This file is part of VirtualBox base platform packages, as
14available from https://www.virtualbox.org.
15
16This program is free software; you can redistribute it and/or
17modify it under the terms of the GNU General Public License
18as published by the Free Software Foundation, in version 3 of the
19License.
20
21This program is distributed in the hope that it will be useful, but
22WITHOUT ANY WARRANTY; without even the implied warranty of
23MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24General Public License for more details.
25
26You should have received a copy of the GNU General Public License
27along with this program; if not, see <https://www.gnu.org/licenses>.
28
29The contents of this file may alternatively be used under the terms
30of the Common Development and Distribution License Version 1.0
31(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32in the VirtualBox distribution, in which case the provisions of the
33CDDL are applicable instead of those of the GPL.
34
35You may elect to license modified versions of this file under the
36terms and conditions of either the GPL or the CDDL or both.
37
38SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39"""
40__version__ = "$Revision: 106061 $"
41
42# Standard python imports.
43import re;
44import os;
45import textwrap;
46import sys;
47
48# Only the main script needs to modify the path.
49try: __file__ # pylint: disable=used-before-assignment
50except: __file__ = sys.argv[0];
51g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)));
52sys.path.append(g_ksValidationKitDir);
53
54# Validation Kit imports.
55from analysis import reader
56from analysis import reporting
57
58
59def usage():
60 """
61 Display usage.
62 """
63 # Set up the output wrapper.
64 try: cCols = os.get_terminal_size()[0] # since 3.3
65 except: cCols = 79;
66 oWrapper = textwrap.TextWrapper(width = cCols);
67
68 # Do the outputting.
69 print('Tool for comparing test results.');
70 print('');
71 oWrapper.subsequent_indent = ' ' * (len('usage: ') + 4);
72 print(oWrapper.fill('usage: analyze.py [options] [collection-1] -- [collection-2] [-- [collection3] [..]])'))
73 oWrapper.subsequent_indent = '';
74 print('');
75 print(oWrapper.fill('This tool compares two or more result collections, using one as a baseline (first by default) '
76 'and showing how the results in others differs from it.'));
77 print('');
78 print(oWrapper.fill('The results (XML file) from one or more test runs makes up a collection. A collection can be '
79 'named using the --name <name> option, or will get a sequential name automatically. The baseline '
80 'collection will have "(baseline)" appended to its name.'));
81 print('');
82 print(oWrapper.fill('A test run produces one XML file, either via the testdriver/reporter.py machinery or via the IPRT '
83 'test.cpp code. In the latter case it can be enabled and controlled via IPRT_TEST_FILE. A collection '
84 'consists of one or more of test runs (i.e. XML result files). These are combined (aka distilled) '
85 'into a single set of results before comparing them with the others. The --best and --avg options '
86 'controls how this combining is done. The need for this is mainly to try counteract some of the '
87 'instability typically found in the restuls. Just because one test run produces a better result '
88 'after a change does not necessarily mean this will always be the case and that the change was to '
89 'the better, it might just have been regular fluctuations in the test results.'));
90
91 oWrapper.initial_indent = ' ';
92 oWrapper.subsequent_indent = ' ';
93 print('');
94 print('Options governing combining (distillation):');
95 print(' --avg, --average');
96 print(oWrapper.fill('Picks the best result by calculating the average values across all the runs.'));
97 print('');
98 print(' --best');
99 print(oWrapper.fill('Picks the best result from all the runs. For values, this means making guessing what result is '
100 'better based on the unit. This may not always lead to the right choices.'));
101 print(oWrapper.initial_indent + 'Default: --best');
102
103 print('');
104 print('Options relating to collections:');
105 print(' --name <name>');
106 print(oWrapper.fill('Sets the name of the current collection. By default a collection gets a sequential number.'));
107 print('');
108 print(' --baseline <num>');
109 print(oWrapper.fill('Sets collection given by <num> (0-based) as the baseline collection.'));
110 print(oWrapper.initial_indent + 'Default: --baseline 0')
111
112 print('');
113 print('Filtering options:');
114 print(' --filter-test <substring>');
115 print(oWrapper.fill('Exclude tests not containing any of the substrings given via the --filter-test option. The '
116 'matching is done with full test name, i.e. all parent names are prepended with ", " as separator '
117 '(for example "tstIOInstr, CPUID EAX=1").'));
118 print('');
119 print(' --filter-test-out <substring>');
120 print(oWrapper.fill('Exclude tests containing the given substring. As with --filter-test, the matching is done against '
121 'the full test name.'));
122 print('');
123 print(' --filter-value <substring>');
124 print(oWrapper.fill('Exclude values not containing any of the substrings given via the --filter-value option. The '
125 'matching is done against the value name prefixed by the full test name and ": " '
126 '(for example "tstIOInstr, CPUID EAX=1: real mode, CPUID").'));
127 print('');
128 print(' --filter-value-out <substring>');
129 print(oWrapper.fill('Exclude value containing the given substring. As with --filter-value, the matching is done against '
130 'the value name prefixed by the full test name.'));
131
132 print('');
133 print(' --regex-test <expr>');
134 print(oWrapper.fill('Same as --filter-test except the substring matching is done via a regular expression.'));
135 print('');
136 print(' --regex-test-out <expr>');
137 print(oWrapper.fill('Same as --filter-test-out except the substring matching is done via a regular expression.'));
138 print('');
139 print(' --regex-value <expr>');
140 print(oWrapper.fill('Same as --filter-value except the substring matching is done via a regular expression.'));
141 print('');
142 print(' --regex-value-out <expr>');
143 print(oWrapper.fill('Same as --filter-value-out except the substring matching is done via a regular expression.'));
144 print('');
145 print(' --filter-out-empty-leaf-tests');
146 print(oWrapper.fill('Removes any leaf tests that are without any values or sub-tests. This is useful when '
147 'only considering values, especially when doing additional value filtering.'));
148
149 print('');
150 print('Analysis options:');
151 print(' --pct-same-value <float>');
152 print(oWrapper.fill('The threshold at which the percent difference between two values are considered the same '
153 'during analysis.'));
154 print(oWrapper.initial_indent + 'Default: --pct-same-value 0.10');
155
156 print('');
157 print('Output options:');
158 print(' --brief, --verbose');
159 print(oWrapper.fill('Whether to omit (--brief) the value for non-baseline runs and just get along with the difference.'));
160 print(oWrapper.initial_indent + 'Default: --brief');
161 print('');
162 print(' --pct <num>, --pct-precision <num>');
163 print(oWrapper.fill('Specifies the number of decimal place to use when formatting the difference as percent.'));
164 print(oWrapper.initial_indent + 'Default: --pct 2');
165 return 1;
166
167
168class ResultCollection(object):
169 """
170 One or more test runs that should be merged before comparison.
171 """
172
173 def __init__(self, sName):
174 self.sName = sName;
175 self.aoTestTrees = [] # type: [Test]
176 self.asTestFiles = [] # type: [str] - runs parallel to aoTestTrees
177 self.oDistilled = None # type: Test
178
179 def append(self, sFilename):
180 """
181 Loads sFilename and appends the result.
182 Returns True on success, False on failure.
183 """
184 oTestTree = reader.parseTestResult(sFilename);
185 if oTestTree:
186 self.aoTestTrees.append(oTestTree);
187 self.asTestFiles.append(sFilename);
188 return True;
189 return False;
190
191 def isEmpty(self):
192 """ Checks if the result is empty. """
193 return len(self.aoTestTrees) == 0;
194
195 def filterTests(self, asFilters):
196 """
197 Keeps all the tests in the test trees sub-string matching asFilters (str or re).
198 """
199 for oTestTree in self.aoTestTrees:
200 oTestTree.filterTests(asFilters);
201 return self;
202
203 def filterOutTests(self, asFilters):
204 """
205 Removes all the tests in the test trees sub-string matching asFilters (str or re).
206 """
207 for oTestTree in self.aoTestTrees:
208 oTestTree.filterOutTests(asFilters);
209 return self;
210
211 def filterValues(self, asFilters):
212 """
213 Keeps all the tests in the test trees sub-string matching asFilters (str or re).
214 """
215 for oTestTree in self.aoTestTrees:
216 oTestTree.filterValues(asFilters);
217 return self;
218
219 def filterOutValues(self, asFilters):
220 """
221 Removes all the tests in the test trees sub-string matching asFilters (str or re).
222 """
223 for oTestTree in self.aoTestTrees:
224 oTestTree.filterOutValues(asFilters);
225 return self;
226
227 def filterOutEmptyLeafTests(self):
228 """
229 Removes all the tests in the test trees that have neither child tests nor values.
230 """
231 for oTestTree in self.aoTestTrees:
232 oTestTree.filterOutEmptyLeafTests();
233 return self;
234
235 def distill(self, sMethod, fDropLoners = False):
236 """
237 Distills the set of test results into a single one by the given method.
238
239 Valid sMethod values:
240 - 'best': Pick the best result for each test and value among all the test runs.
241 - 'avg': Calculate the average value among all the test runs.
242
243 When fDropLoners is True, tests and values that only appear in a single test run
244 will be discarded. When False (the default), the lone result will be used.
245 """
246 assert sMethod in ['best', 'avg'];
247 assert not self.oDistilled;
248
249 # If empty, nothing to do.
250 if self.isEmpty():
251 return None;
252
253 # If there is only a single tree, make a deep copy of it.
254 if len(self.aoTestTrees) == 1:
255 oDistilled = self.aoTestTrees[0].clone();
256 else:
257
258 # Since we don't know if the test runs are all from the same test, we create
259 # dummy root tests for each run and use these are the start for the distillation.
260 aoDummyInputTests = [];
261 for oRun in self.aoTestTrees:
262 oDummy = reader.Test();
263 oDummy.aoChildren = [oRun,];
264 aoDummyInputTests.append(oDummy);
265
266 # Similarly, we end up with a "dummy" root test for the result.
267 oDistilled = reader.Test();
268 oDistilled.distill(aoDummyInputTests, sMethod, fDropLoners);
269
270 # We can drop this if there is only a single child, i.e. if all runs are for
271 # the same test.
272 if len(oDistilled.aoChildren) == 1:
273 oDistilled = oDistilled.aoChildren[0];
274
275 self.oDistilled = oDistilled;
276 return oDistilled;
277
278
279
280# matchWithValue hacks.
281g_asOptions = [];
282g_iOptInd = 1;
283g_sOptArg = '';
284
285def matchWithValue(sOption):
286 """ Matches an option with a value, placing the value in g_sOptArg if it matches. """
287 global g_asOptions, g_iOptInd, g_sOptArg;
288 sArg = g_asOptions[g_iOptInd];
289 if sArg.startswith(sOption):
290 if len(sArg) == len(sOption):
291 if g_iOptInd + 1 < len(g_asOptions):
292 g_iOptInd += 1;
293 g_sOptArg = g_asOptions[g_iOptInd];
294 return True;
295
296 print('syntax error: Option %s takes a value!' % (sOption,));
297 raise Exception('syntax error: Option %s takes a value!' % (sOption,));
298
299 if sArg[len(sOption)] in ('=', ':'):
300 g_sOptArg = sArg[len(sOption) + 1:];
301 return True;
302 return False;
303
304
305def main(asArgs):
306 """ C style main(). """
307 #
308 # Parse arguments
309 #
310 oCurCollection = ResultCollection('#0');
311 aoCollections = [ oCurCollection, ];
312 iBaseline = 0;
313 sDistillationMethod = 'best';
314 fBrief = True;
315 cPctPrecision = 2;
316 rdPctSameValue = 0.1;
317 asTestFilters = [];
318 asTestOutFilters = [];
319 asValueFilters = [];
320 asValueOutFilters = [];
321 fFilterOutEmptyLeafTest = True;
322
323 global g_asOptions, g_iOptInd, g_sOptArg;
324 g_asOptions = asArgs;
325 g_iOptInd = 1;
326 while g_iOptInd < len(asArgs):
327 sArg = asArgs[g_iOptInd];
328 g_sOptArg = '';
329 #print("dbg: g_iOptInd=%s '%s'" % (g_iOptInd, sArg,));
330
331 if sArg.startswith('--help'):
332 return usage();
333
334 if matchWithValue('--filter-test'):
335 asTestFilters.append(g_sOptArg);
336 elif matchWithValue('--filter-test-out'):
337 asTestOutFilters.append(g_sOptArg);
338 elif matchWithValue('--filter-value'):
339 asValueFilters.append(g_sOptArg);
340 elif matchWithValue('--filter-value-out'):
341 asValueOutFilters.append(g_sOptArg);
342
343 elif matchWithValue('--regex-test'):
344 asTestFilters.append(re.compile(g_sOptArg));
345 elif matchWithValue('--regex-test-out'):
346 asTestOutFilters.append(re.compile(g_sOptArg));
347 elif matchWithValue('--regex-value'):
348 asValueFilters.append(re.compile(g_sOptArg));
349 elif matchWithValue('--regex-value-out'):
350 asValueOutFilters.append(re.compile(g_sOptArg));
351
352 elif sArg == '--filter-out-empty-leaf-tests':
353 fFilterOutEmptyLeafTest = True;
354 elif sArg == '--no-filter-out-empty-leaf-tests':
355 fFilterOutEmptyLeafTest = False;
356
357 elif sArg == '--best':
358 sDistillationMethod = 'best';
359 elif sArg in ('--avg', '--average'):
360 sDistillationMethod = 'avg';
361
362 elif sArg == '--brief':
363 fBrief = True;
364 elif sArg == '--verbose':
365 fBrief = False;
366
367 elif matchWithValue('--pct') or matchWithValue('--pct-precision'):
368 cPctPrecision = int(g_sOptArg);
369 elif matchWithValue('--base') or matchWithValue('--baseline'):
370 iBaseline = int(g_sOptArg);
371
372 elif matchWithValue('--pct-same-value'):
373 rdPctSameValue = float(g_sOptArg);
374
375 # '--' starts a new collection. If current one is empty, drop it.
376 elif sArg == '--':
377 print("dbg: new collection");
378 #if oCurCollection.isEmpty():
379 # del aoCollections[-1];
380 oCurCollection = ResultCollection("#%s" % (len(aoCollections),));
381 aoCollections.append(oCurCollection);
382
383 # Name the current result collection.
384 elif matchWithValue('--name'):
385 oCurCollection.sName = g_sOptArg;
386
387 # Read in a file and add it to the current data set.
388 else:
389 if not oCurCollection.append(sArg):
390 return 1;
391 g_iOptInd += 1;
392
393 #
394 # Post argument parsing processing.
395 #
396
397 # Drop the last collection if empty.
398 if oCurCollection.isEmpty():
399 del aoCollections[-1];
400 if not aoCollections:
401 print("error: No input files given!");
402 return 1;
403
404 # Check the baseline value and mark the column as such.
405 if iBaseline < 0 or iBaseline > len(aoCollections):
406 print("error: specified baseline is out of range: %s, valid range 0 <= baseline < %s"
407 % (iBaseline, len(aoCollections),));
408 return 1;
409 aoCollections[iBaseline].sName += ' (baseline)';
410
411 #
412 # Apply filtering before distilling each collection into a single result tree.
413 #
414 if asTestFilters:
415 for oCollection in aoCollections:
416 oCollection.filterTests(asTestFilters);
417 if asTestOutFilters:
418 for oCollection in aoCollections:
419 oCollection.filterOutTests(asTestOutFilters);
420
421 if asValueFilters:
422 for oCollection in aoCollections:
423 oCollection.filterValues(asValueFilters);
424 if asValueOutFilters:
425 for oCollection in aoCollections:
426 oCollection.filterOutValues(asValueOutFilters);
427
428 if fFilterOutEmptyLeafTest:
429 for oCollection in aoCollections:
430 oCollection.filterOutEmptyLeafTests();
431
432 # Distillation.
433 for oCollection in aoCollections:
434 oCollection.distill(sDistillationMethod);
435
436 #
437 # Produce the report.
438 #
439 oTable = reporting.RunTable(iBaseline, fBrief, cPctPrecision, rdPctSameValue);
440 oTable.populateFromRuns([oCollection.oDistilled for oCollection in aoCollections],
441 [oCollection.sName for oCollection in aoCollections]);
442 print('\n'.join(oTable.formatAsText()));
443 return 0;
444
445if __name__ == '__main__':
446 sys.exit(main(sys.argv));
447
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette