| 1 | <?xml version="1.0" encoding="utf-8" ?>
|
|---|
| 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|---|
| 3 | <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
|---|
| 4 | <head>
|
|---|
| 5 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|---|
| 6 | <meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
|
|---|
| 7 | <title>AutomaticTestingRevamp.txt</title>
|
|---|
| 8 | <style type="text/css">
|
|---|
| 9 |
|
|---|
| 10 | /*
|
|---|
| 11 | :Author: David Goodger (goodger@python.org)
|
|---|
| 12 | :Id: $Id: AutomaticTestingRevamp.html 98107 2023-01-17 22:56:50Z vboxsync $
|
|---|
| 13 | :Copyright: This stylesheet has been placed in the public domain.
|
|---|
| 14 |
|
|---|
| 15 | Default cascading style sheet for the HTML output of Docutils.
|
|---|
| 16 |
|
|---|
| 17 | See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to
|
|---|
| 18 | customize this style sheet.
|
|---|
| 19 | */
|
|---|
| 20 |
|
|---|
| 21 | /* used to remove borders from tables and images */
|
|---|
| 22 | .borderless, table.borderless td, table.borderless th {
|
|---|
| 23 | border: 0 }
|
|---|
| 24 |
|
|---|
| 25 | table.borderless td, table.borderless th {
|
|---|
| 26 | /* Override padding for "table.docutils td" with "! important".
|
|---|
| 27 | The right padding separates the table cells. */
|
|---|
| 28 | padding: 0 0.5em 0 0 ! important }
|
|---|
| 29 |
|
|---|
| 30 | .first {
|
|---|
| 31 | /* Override more specific margin styles with "! important". */
|
|---|
| 32 | margin-top: 0 ! important }
|
|---|
| 33 |
|
|---|
| 34 | .last, .with-subtitle {
|
|---|
| 35 | margin-bottom: 0 ! important }
|
|---|
| 36 |
|
|---|
| 37 | .hidden {
|
|---|
| 38 | display: none }
|
|---|
| 39 |
|
|---|
| 40 | .subscript {
|
|---|
| 41 | vertical-align: sub;
|
|---|
| 42 | font-size: smaller }
|
|---|
| 43 |
|
|---|
| 44 | .superscript {
|
|---|
| 45 | vertical-align: super;
|
|---|
| 46 | font-size: smaller }
|
|---|
| 47 |
|
|---|
| 48 | a.toc-backref {
|
|---|
| 49 | text-decoration: none ;
|
|---|
| 50 | color: black }
|
|---|
| 51 |
|
|---|
| 52 | blockquote.epigraph {
|
|---|
| 53 | margin: 2em 5em ; }
|
|---|
| 54 |
|
|---|
| 55 | dl.docutils dd {
|
|---|
| 56 | margin-bottom: 0.5em }
|
|---|
| 57 |
|
|---|
| 58 | object[type="image/svg+xml"], object[type="application/x-shockwave-flash"] {
|
|---|
| 59 | overflow: hidden;
|
|---|
| 60 | }
|
|---|
| 61 |
|
|---|
| 62 | /* Uncomment (and remove this text!) to get bold-faced definition list terms
|
|---|
| 63 | dl.docutils dt {
|
|---|
| 64 | font-weight: bold }
|
|---|
| 65 | */
|
|---|
| 66 |
|
|---|
| 67 | div.abstract {
|
|---|
| 68 | margin: 2em 5em }
|
|---|
| 69 |
|
|---|
| 70 | div.abstract p.topic-title {
|
|---|
| 71 | font-weight: bold ;
|
|---|
| 72 | text-align: center }
|
|---|
| 73 |
|
|---|
| 74 | div.admonition, div.attention, div.caution, div.danger, div.error,
|
|---|
| 75 | div.hint, div.important, div.note, div.tip, div.warning {
|
|---|
| 76 | margin: 2em ;
|
|---|
| 77 | border: medium outset ;
|
|---|
| 78 | padding: 1em }
|
|---|
| 79 |
|
|---|
| 80 | div.admonition p.admonition-title, div.hint p.admonition-title,
|
|---|
| 81 | div.important p.admonition-title, div.note p.admonition-title,
|
|---|
| 82 | div.tip p.admonition-title {
|
|---|
| 83 | font-weight: bold ;
|
|---|
| 84 | font-family: sans-serif }
|
|---|
| 85 |
|
|---|
| 86 | div.attention p.admonition-title, div.caution p.admonition-title,
|
|---|
| 87 | div.danger p.admonition-title, div.error p.admonition-title,
|
|---|
| 88 | div.warning p.admonition-title, .code .error {
|
|---|
| 89 | color: red ;
|
|---|
| 90 | font-weight: bold ;
|
|---|
| 91 | font-family: sans-serif }
|
|---|
| 92 |
|
|---|
| 93 | /* Uncomment (and remove this text!) to get reduced vertical space in
|
|---|
| 94 | compound paragraphs.
|
|---|
| 95 | div.compound .compound-first, div.compound .compound-middle {
|
|---|
| 96 | margin-bottom: 0.5em }
|
|---|
| 97 |
|
|---|
| 98 | div.compound .compound-last, div.compound .compound-middle {
|
|---|
| 99 | margin-top: 0.5em }
|
|---|
| 100 | */
|
|---|
| 101 |
|
|---|
| 102 | div.dedication {
|
|---|
| 103 | margin: 2em 5em ;
|
|---|
| 104 | text-align: center ;
|
|---|
| 105 | font-style: italic }
|
|---|
| 106 |
|
|---|
| 107 | div.dedication p.topic-title {
|
|---|
| 108 | font-weight: bold ;
|
|---|
| 109 | font-style: normal }
|
|---|
| 110 |
|
|---|
| 111 | div.figure {
|
|---|
| 112 | margin-left: 2em ;
|
|---|
| 113 | margin-right: 2em }
|
|---|
| 114 |
|
|---|
| 115 | div.footer, div.header {
|
|---|
| 116 | clear: both;
|
|---|
| 117 | font-size: smaller }
|
|---|
| 118 |
|
|---|
| 119 | div.line-block {
|
|---|
| 120 | display: block ;
|
|---|
| 121 | margin-top: 1em ;
|
|---|
| 122 | margin-bottom: 1em }
|
|---|
| 123 |
|
|---|
| 124 | div.line-block div.line-block {
|
|---|
| 125 | margin-top: 0 ;
|
|---|
| 126 | margin-bottom: 0 ;
|
|---|
| 127 | margin-left: 1.5em }
|
|---|
| 128 |
|
|---|
| 129 | div.sidebar {
|
|---|
| 130 | margin: 0 0 0.5em 1em ;
|
|---|
| 131 | border: medium outset ;
|
|---|
| 132 | padding: 1em ;
|
|---|
| 133 | background-color: #ffffee ;
|
|---|
| 134 | width: 40% ;
|
|---|
| 135 | float: right ;
|
|---|
| 136 | clear: right }
|
|---|
| 137 |
|
|---|
| 138 | div.sidebar p.rubric {
|
|---|
| 139 | font-family: sans-serif ;
|
|---|
| 140 | font-size: medium }
|
|---|
| 141 |
|
|---|
| 142 | div.system-messages {
|
|---|
| 143 | margin: 5em }
|
|---|
| 144 |
|
|---|
| 145 | div.system-messages h1 {
|
|---|
| 146 | color: red }
|
|---|
| 147 |
|
|---|
| 148 | div.system-message {
|
|---|
| 149 | border: medium outset ;
|
|---|
| 150 | padding: 1em }
|
|---|
| 151 |
|
|---|
| 152 | div.system-message p.system-message-title {
|
|---|
| 153 | color: red ;
|
|---|
| 154 | font-weight: bold }
|
|---|
| 155 |
|
|---|
| 156 | div.topic {
|
|---|
| 157 | margin: 2em }
|
|---|
| 158 |
|
|---|
| 159 | h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
|
|---|
| 160 | h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
|
|---|
| 161 | margin-top: 0.4em }
|
|---|
| 162 |
|
|---|
| 163 | h1.title {
|
|---|
| 164 | text-align: center }
|
|---|
| 165 |
|
|---|
| 166 | h2.subtitle {
|
|---|
| 167 | text-align: center }
|
|---|
| 168 |
|
|---|
| 169 | hr.docutils {
|
|---|
| 170 | width: 75% }
|
|---|
| 171 |
|
|---|
| 172 | img.align-left, .figure.align-left, object.align-left, table.align-left {
|
|---|
| 173 | clear: left ;
|
|---|
| 174 | float: left ;
|
|---|
| 175 | margin-right: 1em }
|
|---|
| 176 |
|
|---|
| 177 | img.align-right, .figure.align-right, object.align-right, table.align-right {
|
|---|
| 178 | clear: right ;
|
|---|
| 179 | float: right ;
|
|---|
| 180 | margin-left: 1em }
|
|---|
| 181 |
|
|---|
| 182 | img.align-center, .figure.align-center, object.align-center {
|
|---|
| 183 | display: block;
|
|---|
| 184 | margin-left: auto;
|
|---|
| 185 | margin-right: auto;
|
|---|
| 186 | }
|
|---|
| 187 |
|
|---|
| 188 | table.align-center {
|
|---|
| 189 | margin-left: auto;
|
|---|
| 190 | margin-right: auto;
|
|---|
| 191 | }
|
|---|
| 192 |
|
|---|
| 193 | .align-left {
|
|---|
| 194 | text-align: left }
|
|---|
| 195 |
|
|---|
| 196 | .align-center {
|
|---|
| 197 | clear: both ;
|
|---|
| 198 | text-align: center }
|
|---|
| 199 |
|
|---|
| 200 | .align-right {
|
|---|
| 201 | text-align: right }
|
|---|
| 202 |
|
|---|
| 203 | /* reset inner alignment in figures */
|
|---|
| 204 | div.align-right {
|
|---|
| 205 | text-align: inherit }
|
|---|
| 206 |
|
|---|
| 207 | /* div.align-center * { */
|
|---|
| 208 | /* text-align: left } */
|
|---|
| 209 |
|
|---|
| 210 | .align-top {
|
|---|
| 211 | vertical-align: top }
|
|---|
| 212 |
|
|---|
| 213 | .align-middle {
|
|---|
| 214 | vertical-align: middle }
|
|---|
| 215 |
|
|---|
| 216 | .align-bottom {
|
|---|
| 217 | vertical-align: bottom }
|
|---|
| 218 |
|
|---|
| 219 | ol.simple, ul.simple {
|
|---|
| 220 | margin-bottom: 1em }
|
|---|
| 221 |
|
|---|
| 222 | ol.arabic {
|
|---|
| 223 | list-style: decimal }
|
|---|
| 224 |
|
|---|
| 225 | ol.loweralpha {
|
|---|
| 226 | list-style: lower-alpha }
|
|---|
| 227 |
|
|---|
| 228 | ol.upperalpha {
|
|---|
| 229 | list-style: upper-alpha }
|
|---|
| 230 |
|
|---|
| 231 | ol.lowerroman {
|
|---|
| 232 | list-style: lower-roman }
|
|---|
| 233 |
|
|---|
| 234 | ol.upperroman {
|
|---|
| 235 | list-style: upper-roman }
|
|---|
| 236 |
|
|---|
| 237 | p.attribution {
|
|---|
| 238 | text-align: right ;
|
|---|
| 239 | margin-left: 50% }
|
|---|
| 240 |
|
|---|
| 241 | p.caption {
|
|---|
| 242 | font-style: italic }
|
|---|
| 243 |
|
|---|
| 244 | p.credits {
|
|---|
| 245 | font-style: italic ;
|
|---|
| 246 | font-size: smaller }
|
|---|
| 247 |
|
|---|
| 248 | p.label {
|
|---|
| 249 | white-space: nowrap }
|
|---|
| 250 |
|
|---|
| 251 | p.rubric {
|
|---|
| 252 | font-weight: bold ;
|
|---|
| 253 | font-size: larger ;
|
|---|
| 254 | color: maroon ;
|
|---|
| 255 | text-align: center }
|
|---|
| 256 |
|
|---|
| 257 | p.sidebar-title {
|
|---|
| 258 | font-family: sans-serif ;
|
|---|
| 259 | font-weight: bold ;
|
|---|
| 260 | font-size: larger }
|
|---|
| 261 |
|
|---|
| 262 | p.sidebar-subtitle {
|
|---|
| 263 | font-family: sans-serif ;
|
|---|
| 264 | font-weight: bold }
|
|---|
| 265 |
|
|---|
| 266 | p.topic-title {
|
|---|
| 267 | font-weight: bold }
|
|---|
| 268 |
|
|---|
| 269 | pre.address {
|
|---|
| 270 | margin-bottom: 0 ;
|
|---|
| 271 | margin-top: 0 ;
|
|---|
| 272 | font: inherit }
|
|---|
| 273 |
|
|---|
| 274 | pre.literal-block, pre.doctest-block, pre.math, pre.code {
|
|---|
| 275 | margin-left: 2em ;
|
|---|
| 276 | margin-right: 2em }
|
|---|
| 277 |
|
|---|
| 278 | pre.code .ln { color: grey; } /* line numbers */
|
|---|
| 279 | pre.code, code { background-color: #eeeeee }
|
|---|
| 280 | pre.code .comment, code .comment { color: #5C6576 }
|
|---|
| 281 | pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
|
|---|
| 282 | pre.code .literal.string, code .literal.string { color: #0C5404 }
|
|---|
| 283 | pre.code .name.builtin, code .name.builtin { color: #352B84 }
|
|---|
| 284 | pre.code .deleted, code .deleted { background-color: #DEB0A1}
|
|---|
| 285 | pre.code .inserted, code .inserted { background-color: #A3D289}
|
|---|
| 286 |
|
|---|
| 287 | span.classifier {
|
|---|
| 288 | font-family: sans-serif ;
|
|---|
| 289 | font-style: oblique }
|
|---|
| 290 |
|
|---|
| 291 | span.classifier-delimiter {
|
|---|
| 292 | font-family: sans-serif ;
|
|---|
| 293 | font-weight: bold }
|
|---|
| 294 |
|
|---|
| 295 | span.interpreted {
|
|---|
| 296 | font-family: sans-serif }
|
|---|
| 297 |
|
|---|
| 298 | span.option {
|
|---|
| 299 | white-space: nowrap }
|
|---|
| 300 |
|
|---|
| 301 | span.pre {
|
|---|
| 302 | white-space: pre }
|
|---|
| 303 |
|
|---|
| 304 | span.problematic {
|
|---|
| 305 | color: red }
|
|---|
| 306 |
|
|---|
| 307 | span.section-subtitle {
|
|---|
| 308 | /* font-size relative to parent (h1..h6 element) */
|
|---|
| 309 | font-size: 80% }
|
|---|
| 310 |
|
|---|
| 311 | table.citation {
|
|---|
| 312 | border-left: solid 1px gray;
|
|---|
| 313 | margin-left: 1px }
|
|---|
| 314 |
|
|---|
| 315 | table.docinfo {
|
|---|
| 316 | margin: 2em 4em }
|
|---|
| 317 |
|
|---|
| 318 | table.docutils {
|
|---|
| 319 | margin-top: 0.5em ;
|
|---|
| 320 | margin-bottom: 0.5em }
|
|---|
| 321 |
|
|---|
| 322 | table.footnote {
|
|---|
| 323 | border-left: solid 1px black;
|
|---|
| 324 | margin-left: 1px }
|
|---|
| 325 |
|
|---|
| 326 | table.docutils td, table.docutils th,
|
|---|
| 327 | table.docinfo td, table.docinfo th {
|
|---|
| 328 | padding-left: 0.5em ;
|
|---|
| 329 | padding-right: 0.5em ;
|
|---|
| 330 | vertical-align: top }
|
|---|
| 331 |
|
|---|
| 332 | table.docutils th.field-name, table.docinfo th.docinfo-name {
|
|---|
| 333 | font-weight: bold ;
|
|---|
| 334 | text-align: left ;
|
|---|
| 335 | white-space: nowrap ;
|
|---|
| 336 | padding-left: 0 }
|
|---|
| 337 |
|
|---|
| 338 | /* "booktabs" style (no vertical lines) */
|
|---|
| 339 | table.docutils.booktabs {
|
|---|
| 340 | border: 0px;
|
|---|
| 341 | border-top: 2px solid;
|
|---|
| 342 | border-bottom: 2px solid;
|
|---|
| 343 | border-collapse: collapse;
|
|---|
| 344 | }
|
|---|
| 345 | table.docutils.booktabs * {
|
|---|
| 346 | border: 0px;
|
|---|
| 347 | }
|
|---|
| 348 | table.docutils.booktabs th {
|
|---|
| 349 | border-bottom: thin solid;
|
|---|
| 350 | text-align: left;
|
|---|
| 351 | }
|
|---|
| 352 |
|
|---|
| 353 | h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
|
|---|
| 354 | h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
|
|---|
| 355 | font-size: 100% }
|
|---|
| 356 |
|
|---|
| 357 | ul.auto-toc {
|
|---|
| 358 | list-style-type: none }
|
|---|
| 359 |
|
|---|
| 360 | </style>
|
|---|
| 361 | </head>
|
|---|
| 362 | <body>
|
|---|
| 363 | <div class="document">
|
|---|
| 364 |
|
|---|
| 365 |
|
|---|
| 366 | <div class="section" id="revamp-of-automatic-virtualbox-testing">
|
|---|
| 367 | <h1>Revamp of Automatic VirtualBox Testing</h1>
|
|---|
| 368 | <div class="section" id="introduction">
|
|---|
| 369 | <h2>Introduction</h2>
|
|---|
| 370 | <p>This is the design document for a revamped automatic testing framework.
|
|---|
| 371 | The revamp aims at replacing the current tinderbox based testing by a new
|
|---|
| 372 | system that is written from scratch.</p>
|
|---|
| 373 | <p>The old system is not easy to work with and was never meant to be used for
|
|---|
| 374 | managing tests, after all it just a simple a build manager tailored for
|
|---|
| 375 | contiguous building. Modifying the existing tinderbox system to do what
|
|---|
| 376 | we want would require fundamental changes that would render it useless as
|
|---|
| 377 | a build manager, it would therefore end up as a fork. The amount of work
|
|---|
| 378 | required would probably be about the same as writing a new system from
|
|---|
| 379 | scratch. Other considerations, such as the license of the tinderbox
|
|---|
| 380 | system (MPL) and language it is realized in (Perl), are also in favor of
|
|---|
| 381 | doing it from scratch.</p>
|
|---|
| 382 | <p>The language envisioned for the new automatic testing framework is Python. This
|
|---|
| 383 | is for several reasons:</p>
|
|---|
| 384 | <blockquote>
|
|---|
| 385 | <ul class="simple">
|
|---|
| 386 | <li>The VirtualBox API has Python bindings.</li>
|
|---|
| 387 | <li>Python is used quite a bit inside Sun (dunno about Oracle).</li>
|
|---|
| 388 | <li>Works relatively well with Apache for the server side bits.</li>
|
|---|
| 389 | <li>It is more difficult to produce write-only code in Python (alias the
|
|---|
| 390 | we-don't-like-perl argument).</li>
|
|---|
| 391 | <li>You don't need to compile stuff.</li>
|
|---|
| 392 | </ul>
|
|---|
| 393 | </blockquote>
|
|---|
| 394 | <p>Note that the author of this document has no special training as a test
|
|---|
| 395 | engineer and may therefore be using the wrong terms here and there. The
|
|---|
| 396 | primary focus is to express what we need to do in order to improve
|
|---|
| 397 | testing.</p>
|
|---|
| 398 | <p>This document is written in reStructuredText (rst) which just happens to
|
|---|
| 399 | be used by Python, the primary language for this revamp. For more
|
|---|
| 400 | information on reStructuredText: <a class="reference external" href="http://docutils.sourceforge.net/rst.html">http://docutils.sourceforge.net/rst.html</a></p>
|
|---|
| 401 | </div>
|
|---|
| 402 | </div>
|
|---|
| 403 | <div class="section" id="definitions-glossary">
|
|---|
| 404 | <h1>Definitions / Glossary</h1>
|
|---|
| 405 | <dl class="docutils">
|
|---|
| 406 | <dt>sub-test driver</dt>
|
|---|
| 407 | <dd>A set of test cases that can be used by more than one test driver. Could
|
|---|
| 408 | also be called a test unit, in the pascal sense of unit, if it wasn't so
|
|---|
| 409 | easily confused with 'unit test'.</dd>
|
|---|
| 410 | <dt>test</dt>
|
|---|
| 411 | <dd>This is somewhat ambiguous and this document try avoid using it where
|
|---|
| 412 | possible. When used it normally refers to doing testing by executing one or
|
|---|
| 413 | more testcases.</dd>
|
|---|
| 414 | <dt>test case</dt>
|
|---|
| 415 | <dd>A set of inputs, test programs and expected results. It validates system
|
|---|
| 416 | requirements and generates a pass or failed status. A basic unit of testing.
|
|---|
| 417 | Note that we use the term in a rather broad sense.</dd>
|
|---|
| 418 | <dt>test driver</dt>
|
|---|
| 419 | <dd>A program/script used to execute a test. Also known as a test harness.
|
|---|
| 420 | Generally abbreviated 'td'. It can have sub-test drivers.</dd>
|
|---|
| 421 | <dt>test manager</dt>
|
|---|
| 422 | <dd>Software managing the automatic testing. This is a web application that runs
|
|---|
| 423 | on a dedicated server (tindertux).</dd>
|
|---|
| 424 | <dt>test set</dt>
|
|---|
| 425 | <dd>The output of testing activity. Logs, results, ++. Our usage of this should
|
|---|
| 426 | probably be renamed to 'test run'.</dd>
|
|---|
| 427 | <dt>test group</dt>
|
|---|
| 428 | <dd>A collection of related test cases.</dd>
|
|---|
| 429 | <dt>testbox</dt>
|
|---|
| 430 | <dd>A computer that does testing.</dd>
|
|---|
| 431 | <dt>testbox script</dt>
|
|---|
| 432 | <dd>Script executing orders from the test manager on a testbox. Started
|
|---|
| 433 | automatically upon bootup.</dd>
|
|---|
| 434 | <dt>testing</dt>
|
|---|
| 435 | <dd>todo</dd>
|
|---|
| 436 | <dt>TODO: Check that we've got all this right and make them more exact</dt>
|
|---|
| 437 | <dd>where possible.</dd>
|
|---|
| 438 | </dl>
|
|---|
| 439 | <p>See also <a class="reference external" href="http://encyclopedia2.thefreedictionary.com/testing%20types">http://encyclopedia2.thefreedictionary.com/testing%20types</a>
|
|---|
| 440 | and <a class="reference external" href="http://www.aptest.com/glossary.html">http://www.aptest.com/glossary.html</a> .</p>
|
|---|
| 441 | </div>
|
|---|
| 442 | <div class="section" id="objectives">
|
|---|
| 443 | <h1>Objectives</h1>
|
|---|
| 444 | <blockquote>
|
|---|
| 445 | <ul class="simple">
|
|---|
| 446 | <li>A scalable test manager (>200 testboxes).</li>
|
|---|
| 447 | <li>Optimize the web user interface (WUI) for typical workflows and analysis.</li>
|
|---|
| 448 | <li>Efficient and flexibile test configuration.</li>
|
|---|
| 449 | <li>Import test result from other test systems (logo testing, VDI, ++).</li>
|
|---|
| 450 | <li>Easy to add lots of new testscripts.</li>
|
|---|
| 451 | <li>Run tests locally without a manager.</li>
|
|---|
| 452 | <li>Revamp a bit at the time.</li>
|
|---|
| 453 | </ul>
|
|---|
| 454 | </blockquote>
|
|---|
| 455 | </div>
|
|---|
| 456 | <div class="section" id="the-testbox-side">
|
|---|
| 457 | <h1>The Testbox Side</h1>
|
|---|
| 458 | <p>Each testbox has a unique name corresponding to its DNS zone entry. When booted
|
|---|
| 459 | a testbox script is started automatically. This script will query the test
|
|---|
| 460 | manager for orders and execute them. The core order downloads and executes a
|
|---|
| 461 | test driver with parameters (configuration) from the server. The test driver
|
|---|
| 462 | does all the necessary work for executing the test. In a typical VirtualBox
|
|---|
| 463 | test this means picking a build, installing it, configuring VMs, running the
|
|---|
| 464 | test VMs, collecting the results, submitting them to the server, and finally
|
|---|
| 465 | cleaning up afterwards.</p>
|
|---|
| 466 | <p>The testbox environment which the test drivers are executed in will have a
|
|---|
| 467 | number of environment variables for determining location of the source images
|
|---|
| 468 | and other test data, scratch space, test set id, server URL, and so on and so
|
|---|
| 469 | forth.</p>
|
|---|
| 470 | <p>On startup, the testbox script will look for crash dumps and similar on
|
|---|
| 471 | systems where this is possible. If any sign of a crash is found, it will
|
|---|
| 472 | put any dumps and reports in the upload directory and inform the test
|
|---|
| 473 | manager before reporting for duty. In order to generate the proper file
|
|---|
| 474 | names and report the crash in the right test set as well as prevent
|
|---|
| 475 | reporting crashes unrelated to automatic testing, the testbox script will
|
|---|
| 476 | keep information (test set id, ++) in a separate scratch directory
|
|---|
| 477 | (${TESTBOX_PATH_SCRATCH}/../testbox) and make sure it is synced to the
|
|---|
| 478 | disk (both files and directories).</p>
|
|---|
| 479 | <p>After checking for crashes, the testbox script will clean up any previous test
|
|---|
| 480 | which might be around. This involves first invoking the test script in cleanup
|
|---|
| 481 | mode and the wiping the scratch space.</p>
|
|---|
| 482 | <p>When reporting for duty the script will submit information about the host: OS
|
|---|
| 483 | name, OS version, OS bitness, CPU vendor, total number of cores, VT-x support,
|
|---|
| 484 | AMD-V support, amount of memory, amount of scratch space, and anything else that
|
|---|
| 485 | can be found useful for scheduling tests or filtering test configurations.</p>
|
|---|
| 486 | <div class="section" id="testbox-script-orders">
|
|---|
| 487 | <h2>Testbox Script Orders</h2>
|
|---|
| 488 | <p>The orders are kept in a queue on the server and the testbox script will fetch
|
|---|
| 489 | them one by one. Orders that cannot be executed at the moment will be masked in
|
|---|
| 490 | the query from the testbox.</p>
|
|---|
| 491 | <dl class="docutils">
|
|---|
| 492 | <dt>Execute Test Driver</dt>
|
|---|
| 493 | <dd>Downloads and executes the a specified test driver with the given
|
|---|
| 494 | configuration (arguments). Only one test driver can be executed at a time.
|
|---|
| 495 | The server can specify more than one ZIP file to be downloaded and unpacked
|
|---|
| 496 | before executing the test driver. The testbox script may cache these zip
|
|---|
| 497 | files using http time stamping.</dd>
|
|---|
| 498 | <dt>Abort Test Driver</dt>
|
|---|
| 499 | <dd>Aborts the current test driver. This will drop a hint to the driver and give
|
|---|
| 500 | it 60 seconds to shut down the normal way. If that fails, the testbox script
|
|---|
| 501 | will kill the driver processes (SIGKILL or equivalent), invoke the
|
|---|
| 502 | testdriver in cleanup mode, and finally wipe the scratch area. Should either
|
|---|
| 503 | of the last two steps fail in some way, the testbox will be rebooted.</dd>
|
|---|
| 504 | <dt>Idle</dt>
|
|---|
| 505 | <dd>Ask again in X seconds, where X is specified by the server.</dd>
|
|---|
| 506 | <dt>Reboot</dt>
|
|---|
| 507 | <dd>Reboot the testbox. If a test driver is current running, an attempt at
|
|---|
| 508 | aborting it (Abort Test Driver) will be made first.</dd>
|
|---|
| 509 | <dt>Update</dt>
|
|---|
| 510 | <dd>Updates the testbox script. The order includes a server relative path to the
|
|---|
| 511 | new testbox script. This can only be executed when no test driver is
|
|---|
| 512 | currently being executed.</dd>
|
|---|
| 513 | </dl>
|
|---|
| 514 | </div>
|
|---|
| 515 | <div class="section" id="testbox-environment-variables">
|
|---|
| 516 | <h2>Testbox Environment: Variables</h2>
|
|---|
| 517 | <dl class="docutils">
|
|---|
| 518 | <dt>COMSPEC</dt>
|
|---|
| 519 | <dd>This will be set to C:WindowsSystem32cmd.exe on Windows.</dd>
|
|---|
| 520 | <dt>PATH</dt>
|
|---|
| 521 | <dd>This will contain the kBuild binary directory for the host platform.</dd>
|
|---|
| 522 | <dt>SHELL</dt>
|
|---|
| 523 | <dd>This will be set to point to kmk_ash(.exe) on all platforms.</dd>
|
|---|
| 524 | <dt>TESTBOX_NAME</dt>
|
|---|
| 525 | <dd>The testbox name.
|
|---|
| 526 | This is not required by the local reporter.</dd>
|
|---|
| 527 | <dt>TESTBOX_PATH_BUILDS</dt>
|
|---|
| 528 | <dd>The absolute path to where the build repository can be found. This should be
|
|---|
| 529 | a read only mount when possible.</dd>
|
|---|
| 530 | <dt>TESTBOX_PATH_RESOURCES</dt>
|
|---|
| 531 | <dd>The absolute path to where static test resources like ISOs and VDIs can be
|
|---|
| 532 | found. The test drivers knows the layout of this. This should be a read only
|
|---|
| 533 | mount when possible.</dd>
|
|---|
| 534 | <dt>TESTBOX_PATH_SCRATCH</dt>
|
|---|
| 535 | <dd>The absolute path to the scratch space. This is the current directory when
|
|---|
| 536 | starting the test driver. It will be wiped automatically after executing the
|
|---|
| 537 | test.
|
|---|
| 538 | (Envisioned as ${TESTBOX_PATH_SCRIPTS}/../scratch and that
|
|---|
| 539 | ${TESTBOX_PATH_SCRATCH}/ will be automatically wiped by the testbox script.)</dd>
|
|---|
| 540 | <dt>TESTBOX_PATH_SCRIPTS</dt>
|
|---|
| 541 | <dd>The absolute path to the test driver and the other files that was unzipped
|
|---|
| 542 | together with it. This is also where the test-driver-abort file will be put.
|
|---|
| 543 | (Envisioned as ${TESTBOX_PATH_SCRATCH}/../driver, see above.)</dd>
|
|---|
| 544 | <dt>TESTBOX_PATH_UPLOAD</dt>
|
|---|
| 545 | <dd>The absolute path to the upload directory for the testbox. This is for
|
|---|
| 546 | putting VOBs, PNGs, core dumps, crash dumps, and such on. The files should be
|
|---|
| 547 | bzipped or zipped if they aren't compress already. The names should contain
|
|---|
| 548 | the testbox and test set ID.</dd>
|
|---|
| 549 | <dt>TESTBOX_REPORTER</dt>
|
|---|
| 550 | <dd>The name of the test reporter back end. If not present, it will default to
|
|---|
| 551 | the local reporter.</dd>
|
|---|
| 552 | <dt>TESTBOX_TEST_SET_ID</dt>
|
|---|
| 553 | <dd>The test set ID if we're running.
|
|---|
| 554 | This is not required by the local reporter.</dd>
|
|---|
| 555 | <dt>TESTBOX_MANAGER_URL</dt>
|
|---|
| 556 | <dd>The URL to the test manager.
|
|---|
| 557 | This is not required by the local reporter.</dd>
|
|---|
| 558 | <dt>TESTBOX_XYZ</dt>
|
|---|
| 559 | <dd>There will probably be some more of these.</dd>
|
|---|
| 560 | </dl>
|
|---|
| 561 | </div>
|
|---|
| 562 | <div class="section" id="testbox-environment-core-utilities">
|
|---|
| 563 | <h2>Testbox Environment: Core Utilities</h2>
|
|---|
| 564 | <p>The testbox will not provide the typical unix /bin and /usr/bin utilities. In
|
|---|
| 565 | other words, cygwin will not be used on Windows!</p>
|
|---|
| 566 | <p>The testbox will provide the unixy utilities that ships with kBuild and possibly
|
|---|
| 567 | some additional ones from tools/<em>.</em>/bin in the VirtualBox tree (wget, unzip,
|
|---|
| 568 | zip, and so on). The test drivers will avoid invoking any of these utilities
|
|---|
| 569 | directly and instead rely on generic utility methods in the test driver
|
|---|
| 570 | framework. That way we can more easily reimplement the functionality of the
|
|---|
| 571 | core utilities and drop the dependency on them. It also allows us to quickly
|
|---|
| 572 | work around platform specific oddities and bugs.</p>
|
|---|
| 573 | </div>
|
|---|
| 574 | <div class="section" id="test-drivers">
|
|---|
| 575 | <h2>Test Drivers</h2>
|
|---|
| 576 | <p>The test drivers are programs that will do the actual testing. In addition to
|
|---|
| 577 | run under the testbox script, they can be executed in the VirtualBox development
|
|---|
| 578 | environment. This is important for bug analysis and for simplifying local
|
|---|
| 579 | testing by the developers before committing changes. It also means the test
|
|---|
| 580 | drivers can be developed locally in the VirtualBox development environment.</p>
|
|---|
| 581 | <p>The main difference between executing a driver under the testbox script and
|
|---|
| 582 | running it manually is that there is no test manager in the latter case. The
|
|---|
| 583 | test result reporter will not talk to the server, but report things to a local
|
|---|
| 584 | log file and/or standard out/err. When invoked manually, all the necessary
|
|---|
| 585 | arguments will need to be specified by hand of course - it should be possible
|
|---|
| 586 | to extract them from a test set as well.</p>
|
|---|
| 587 | <p>For the early implementation stages, an implementation of the reporter interface
|
|---|
| 588 | that talks to the tinderbox base test manager will be needed. This will be
|
|---|
| 589 | dropped later on when a new test manager is ready.</p>
|
|---|
| 590 | <p>As hinted at in other sections, there will be a common framework
|
|---|
| 591 | (libraries/packages/classes) for taking care of the tedious bits that every
|
|---|
| 592 | test driver needs to do. Sharing code is essential to easing test driver
|
|---|
| 593 | development as well as reducing their complexity. The framework will contain:</p>
|
|---|
| 594 | <blockquote>
|
|---|
| 595 | <ul class="simple">
|
|---|
| 596 | <li>A generic way of submitting output. This will be a generic interface with
|
|---|
| 597 | multiple implementation, the TESTBOX_REPORTER environment variable
|
|---|
| 598 | will decide which of them to use. The interface will have very specific
|
|---|
| 599 | methods to allow the reporter to do a best possible job in reporting the
|
|---|
| 600 | results to the test manager.</li>
|
|---|
| 601 | <li><dl class="first docutils">
|
|---|
| 602 | <dt>Helpers for typical tasks, like:</dt>
|
|---|
| 603 | <dd><ul class="first last">
|
|---|
| 604 | <li>Copying files.</li>
|
|---|
| 605 | <li>Deleting files, directory trees and scratch space.</li>
|
|---|
| 606 | <li>Unzipping files.</li>
|
|---|
| 607 | <li>Creating ISOs</li>
|
|---|
| 608 | <li>And such things.</li>
|
|---|
| 609 | </ul>
|
|---|
| 610 | </dd>
|
|---|
| 611 | </dl>
|
|---|
| 612 | </li>
|
|---|
| 613 | <li>Helpers for installing and uninstalling VirtualBox.</li>
|
|---|
| 614 | <li>Helpers for defining VMs. (The VBox API where available.)</li>
|
|---|
| 615 | <li>Helpers for controlling VMs. (The VBox API where available.)</li>
|
|---|
| 616 | </ul>
|
|---|
| 617 | </blockquote>
|
|---|
| 618 | <p>The VirtualBox bits will be separate from the more generic ones, simply because
|
|---|
| 619 | this is cleaner it will allow us to reuse the system for testing other products.</p>
|
|---|
| 620 | <p>The framework will be packaged in a zip file other than the test driver so we
|
|---|
| 621 | don't waste time and space downloading the same common code.</p>
|
|---|
| 622 | <p>The test driver will poll for the file
|
|---|
| 623 | ${TESTBOX_PATH_SCRIPTS}/test-driver-abort and abort all testing when it sees it.</p>
|
|---|
| 624 | <p>The test driver can be invoked in three modes: execute, help and cleanup. The
|
|---|
| 625 | default is execute mode, the help shows an configuration summary and the cleanup
|
|---|
| 626 | is for cleaning up after a reboot or aborted run. The latter is done by the
|
|---|
| 627 | testbox script on startup and after abort - the driver is expected to clean up
|
|---|
| 628 | by itself after a normal run.</p>
|
|---|
| 629 | </div>
|
|---|
| 630 | </div>
|
|---|
| 631 | <div class="section" id="the-server-side">
|
|---|
| 632 | <h1>The Server Side</h1>
|
|---|
| 633 | <p>The server side will be implemented using a webserver (apache), a database
|
|---|
| 634 | (postgres) and cgi scripts (Python). In addition a cron job (Python) running
|
|---|
| 635 | once a minute will generate static html for frequently used pages and maybe
|
|---|
| 636 | execute some other tasks for driving the testing forwards. The order queries
|
|---|
| 637 | from the testbox script is the primary driving force in the system. The total
|
|---|
| 638 | makes up the test manager.</p>
|
|---|
| 639 | <p>The test manager can be split up into three rough parts:</p>
|
|---|
| 640 | <blockquote>
|
|---|
| 641 | <ul class="simple">
|
|---|
| 642 | <li>Configuration (of tests, testgroups and testboxes).</li>
|
|---|
| 643 | <li>Execution (of tests, collecting and organizing the output).</li>
|
|---|
| 644 | <li>Analysis (of test output, mostly about presentation).</li>
|
|---|
| 645 | </ul>
|
|---|
| 646 | </blockquote>
|
|---|
| 647 | </div>
|
|---|
| 648 | <div class="section" id="test-manager-requirements">
|
|---|
| 649 | <h1>Test Manager: Requirements</h1>
|
|---|
| 650 | <p>List of requirements:</p>
|
|---|
| 651 | <blockquote>
|
|---|
| 652 | <ul class="simple">
|
|---|
| 653 | <li>Two level testing - L1 quick smoke tests and L2 longer tests performed on
|
|---|
| 654 | builds passing L1. (Klaus (IIRC) meant this could be realized using
|
|---|
| 655 | test dependency.)</li>
|
|---|
| 656 | <li>Black listing builds (by revision or similar) known to be bad.</li>
|
|---|
| 657 | <li>Distinguish between build types so we can do a portion of the testing with
|
|---|
| 658 | strict builds.</li>
|
|---|
| 659 | <li>Easy to re-configure build source for testing different branch or for
|
|---|
| 660 | testing a release candidate. (Directory based is fine.)</li>
|
|---|
| 661 | <li>Useful to be able to partition testboxes (run specific builds on some
|
|---|
| 662 | boxes, let an engineer have a few boxes for a while).</li>
|
|---|
| 663 | <li>Interaction with ILOM/...: reset systems.</li>
|
|---|
| 664 | <li>Be able to suspend testing on selected testboxes when doing maintenance
|
|---|
| 665 | (where automatically resuming testing on reboot is undesired) or similar
|
|---|
| 666 | activity.</li>
|
|---|
| 667 | <li>Abort testing on selected testboxes.</li>
|
|---|
| 668 | <li>Scheduling of tests requiring more than one testbox.</li>
|
|---|
| 669 | <li>Scheduling of tests that cannot be executing concurrently on several
|
|---|
| 670 | machines because of some global resource like an iSCSI target.</li>
|
|---|
| 671 | <li>Jump the scheduling queue. Scheduling of specified test the next time a
|
|---|
| 672 | testbox is available (optionally specifying which testbox to schedule it
|
|---|
| 673 | on).</li>
|
|---|
| 674 | <li><dl class="first docutils">
|
|---|
| 675 | <dt>Configure tests with variable configuration to get better coverage. Two modes:</dt>
|
|---|
| 676 | <dd><ul class="first last">
|
|---|
| 677 | <li>TM generates the permutations based on one or more sets of test script arguments.</li>
|
|---|
| 678 | <li>Each configuration permutation is specified manually.</li>
|
|---|
| 679 | </ul>
|
|---|
| 680 | </dd>
|
|---|
| 681 | </dl>
|
|---|
| 682 | </li>
|
|---|
| 683 | <li>Test specification needs to be flexible (select tests, disable test, test
|
|---|
| 684 | scheduling (run certain tests nightly), ... ).</li>
|
|---|
| 685 | <li>Test scheduling by hour+weekday and by priority.</li>
|
|---|
| 686 | <li>Test dependencies (test A depends on test B being successful).</li>
|
|---|
| 687 | <li>Historize all configuration data, in particular test configs (permutations
|
|---|
| 688 | included) and testboxes.</li>
|
|---|
| 689 | <li>Test sets has at a minimum a build reference, a testbox reference and a
|
|---|
| 690 | primary log associated with it.</li>
|
|---|
| 691 | <li><dl class="first docutils">
|
|---|
| 692 | <dt>Test sets stores further result as a recursive collection of:</dt>
|
|---|
| 693 | <dd><ul class="first last">
|
|---|
| 694 | <li>hierarchical subtest name (slash sep)</li>
|
|---|
| 695 | <li>test parameters / config</li>
|
|---|
| 696 | <li>bool fail/succ</li>
|
|---|
| 697 | <li>attributes (typed?)</li>
|
|---|
| 698 | <li>test time</li>
|
|---|
| 699 | <li>e.g. throughput</li>
|
|---|
| 700 | <li>subresults</li>
|
|---|
| 701 | <li>log</li>
|
|---|
| 702 | <li>screenshots, video,...</li>
|
|---|
| 703 | </ul>
|
|---|
| 704 | </dd>
|
|---|
| 705 | </dl>
|
|---|
| 706 | </li>
|
|---|
| 707 | <li>The test sets database structure needs to designed such that data mining
|
|---|
| 708 | can be done in an efficient manner.</li>
|
|---|
| 709 | <li>Presentation/analysis: graphs!, categorize bugs, columns reorganizing
|
|---|
| 710 | grouped by test (hierarchical), overviews, result for last day.</li>
|
|---|
| 711 | </ul>
|
|---|
| 712 | </blockquote>
|
|---|
| 713 | </div>
|
|---|
| 714 | <div class="section" id="test-manager-configuration">
|
|---|
| 715 | <h1>Test Manager: Configuration</h1>
|
|---|
| 716 | <div class="section" id="testboxes">
|
|---|
| 717 | <h2>Testboxes</h2>
|
|---|
| 718 | <p>Configuration of testboxes doesn't involve much work normally. A testbox
|
|---|
| 719 | is added manually to the test manager by entering the DNS entry and/or IP
|
|---|
| 720 | address (the test manager resolves the missing one when necessary) as well as
|
|---|
| 721 | the system UUID (when obtainable - should be displayed by the testbox script
|
|---|
| 722 | installer). Queries from unregistered testboxes will be declined as a kind of
|
|---|
| 723 | security measure, the incident should be logged in the webserver log if
|
|---|
| 724 | possible. In later dealings with the client the System UUID will be the key
|
|---|
| 725 | identifier. It's permittable for the IP address to change when the testbox
|
|---|
| 726 | isn't online, but not while testing (just imagine live migration tests and
|
|---|
| 727 | network tests). Ideally, the testboxes should not change IP address.</p>
|
|---|
| 728 | <p>The testbox edit function must allow changing the name and system UUID.</p>
|
|---|
| 729 | <p>One further idea for the testbox configuration is indicating what they are
|
|---|
| 730 | capable of to filter out tests and test configurations that won't work on that
|
|---|
| 731 | testbox. To examplify this take the ACP2 installation test. If the test
|
|---|
| 732 | manager does not make sure the testbox have VT-x or AMD-v capabilities, the test
|
|---|
| 733 | is surely going to fail. Other testbox capabilities would be total number of
|
|---|
| 734 | CPU cores, memory size, scratch space. These testbox capabilities should be
|
|---|
| 735 | collected automatically on bootup by the testbox script together with OS name,
|
|---|
| 736 | OS version and OS bitness.</p>
|
|---|
| 737 | <p>A final thought, instead of outright declining all requests from new testboxes,
|
|---|
| 738 | we could record the unregistered testboxes with ip, UUID, name, os info and
|
|---|
| 739 | capabilities but mark them as inactive. The test operator can then activate
|
|---|
| 740 | them on an activation page or edit the testbox or something.</p>
|
|---|
| 741 | </div>
|
|---|
| 742 | <div class="section" id="testcases">
|
|---|
| 743 | <h2>Testcases</h2>
|
|---|
| 744 | <p>We use the term testcase for a test.</p>
|
|---|
| 745 | </div>
|
|---|
| 746 | <div class="section" id="testgroups">
|
|---|
| 747 | <h2>Testgroups</h2>
|
|---|
| 748 | <p>Testcases are organized into groups. A testcase can be member of more than one
|
|---|
| 749 | group. The testcase gets a priority assigned to it in connection with the
|
|---|
| 750 | group membership.</p>
|
|---|
| 751 | <p>Testgroups are picked up by a testbox partition (aka scheduling group) and a
|
|---|
| 752 | prioirty, scheduling time restriction and dependencies on other test groups are
|
|---|
| 753 | associated with the assignment. A testgroup can be used by several testbox
|
|---|
| 754 | partitions.</p>
|
|---|
| 755 | <p>(This used to be called 'testsuites' but was renamed to avoid confusion with
|
|---|
| 756 | the VBox Test Suite.)</p>
|
|---|
| 757 | </div>
|
|---|
| 758 | <div class="section" id="scheduling">
|
|---|
| 759 | <h2>Scheduling</h2>
|
|---|
| 760 | <p>The initial scheduler will be modelled after what we're doing already on in the
|
|---|
| 761 | tinderbox driven testing. It's best described as a best effort continuous
|
|---|
| 762 | integration scheduler. Meaning, it will always use the latest build suitable
|
|---|
| 763 | for a testcase. It will schedule on a testcase level, using the combined
|
|---|
| 764 | priority of the testcase in the test group and the test group with the testbox
|
|---|
| 765 | partition, trying to spread the test case argument variation out accordingly
|
|---|
| 766 | over the whole scheduilng queue. Which argument variation to start with, is
|
|---|
| 767 | not undefined (random would be best).</p>
|
|---|
| 768 | <p>Later, we may add other schedulers as needed.</p>
|
|---|
| 769 | </div>
|
|---|
| 770 | </div>
|
|---|
| 771 | <div class="section" id="the-test-manager-database">
|
|---|
| 772 | <h1>The Test Manager Database</h1>
|
|---|
| 773 | <p>First a general warning:</p>
|
|---|
| 774 | <blockquote>
|
|---|
| 775 | The guys working on this design are not database experts, web
|
|---|
| 776 | programming experts or similar, rather we are low level guys
|
|---|
| 777 | who's main job is x86 & AMD64 virtualization. So, please don't
|
|---|
| 778 | be too hard on us. :-)</blockquote>
|
|---|
| 779 | <p>A logical table layout can be found in TestManagerDatabaseMap.png (created by
|
|---|
| 780 | Oracle SQL Data Modeler, stored in TestManagerDatabase.dmd). The physical
|
|---|
| 781 | database layout can be found in TestManagerDatabaseInit.pgsql postgreSQL
|
|---|
| 782 | script. The script is commented.</p>
|
|---|
| 783 | <div class="section" id="data-history">
|
|---|
| 784 | <h2>Data History</h2>
|
|---|
| 785 | <p>We need to somehow track configuration changes over time. We also need to
|
|---|
| 786 | be able to query the exact configuration a test set was run with so we can
|
|---|
| 787 | understand and make better use of the results.</p>
|
|---|
| 788 | <p>There are different techniques for archiving this, one is tuple-versioning
|
|---|
| 789 | ( <a class="reference external" href="http://en.wikipedia.org/wiki/Tuple-versioning">http://en.wikipedia.org/wiki/Tuple-versioning</a> ), another is log trigger
|
|---|
| 790 | ( <a class="reference external" href="http://en.wikipedia.org/wiki/Log_trigger">http://en.wikipedia.org/wiki/Log_trigger</a> ). We use tuple-versioning in
|
|---|
| 791 | this database, with 'effective' as start date field name and 'expire' as
|
|---|
| 792 | the end (exclusive).</p>
|
|---|
| 793 | <p>Tuple-versioning has a shortcoming wrt to keys, both primary and foreign.
|
|---|
| 794 | The primary key of a table employing tuple-versioning is really
|
|---|
| 795 | 'id' + 'valid_period', where the latter is expressed using two fields
|
|---|
| 796 | ([effective...expire-1]). Only, how do you tell the database engine that
|
|---|
| 797 | it should not allow overlapping valid_periods? Useful suggestions are
|
|---|
| 798 | welcomed. :-)</p>
|
|---|
| 799 | <p>Foreign key references to a table using tuple-versioning is running into
|
|---|
| 800 | trouble because of the time axis and that to our knowledge foreign keys
|
|---|
| 801 | must reference exactly one row in the other table. When time is involved
|
|---|
| 802 | what we wish to tell the database is that at any given time, there actually
|
|---|
| 803 | is exactly one row we want to match in the other table, only we've no idea
|
|---|
| 804 | how to express this. So, many foreign keys are not expressed in SQL of this
|
|---|
| 805 | database.</p>
|
|---|
| 806 | <p>In some cases, we extend the tuple-versioning with a generation ID so that
|
|---|
| 807 | normal foreign key referencing can be used. We only use this for recording
|
|---|
| 808 | (references in testset) and scheduling (schedqueue), as using it more widely
|
|---|
| 809 | would force updates (gen_id changes) to propagate into all related tables.</p>
|
|---|
| 810 | <dl class="docutils">
|
|---|
| 811 | <dt>See also:</dt>
|
|---|
| 812 | <dd><ul class="first last simple">
|
|---|
| 813 | <li><a class="reference external" href="http://en.wikipedia.org/wiki/Slowly_changing_dimension">http://en.wikipedia.org/wiki/Slowly_changing_dimension</a></li>
|
|---|
| 814 | <li><a class="reference external" href="http://en.wikipedia.org/wiki/Change_data_capture">http://en.wikipedia.org/wiki/Change_data_capture</a></li>
|
|---|
| 815 | <li><a class="reference external" href="http://en.wikipedia.org/wiki/Temporal_database">http://en.wikipedia.org/wiki/Temporal_database</a></li>
|
|---|
| 816 | </ul>
|
|---|
| 817 | </dd>
|
|---|
| 818 | </dl>
|
|---|
| 819 | </div>
|
|---|
| 820 | </div>
|
|---|
| 821 | <div class="section" id="test-manager-execution">
|
|---|
| 822 | <h1>Test Manager: Execution</h1>
|
|---|
| 823 | </div>
|
|---|
| 824 | <div class="section" id="test-manager-scenarios">
|
|---|
| 825 | <h1>Test Manager: Scenarios</h1>
|
|---|
| 826 | <div class="section" id="testbox-signs-on-at-bootup">
|
|---|
| 827 | <h2>#1 - Testbox Signs On (At Bootup)</h2>
|
|---|
| 828 | <dl class="docutils">
|
|---|
| 829 | <dt>The testbox supplies a number of inputs when reporting for duty:</dt>
|
|---|
| 830 | <dd><ul class="first last simple">
|
|---|
| 831 | <li>IP address.</li>
|
|---|
| 832 | <li>System UUID.</li>
|
|---|
| 833 | <li>OS name.</li>
|
|---|
| 834 | <li>OS version.</li>
|
|---|
| 835 | <li>CPU architecture.</li>
|
|---|
| 836 | <li>CPU count (= threads).</li>
|
|---|
| 837 | <li>CPU VT-x/AMD-V capability.</li>
|
|---|
| 838 | <li>CPU nested paging capability.</li>
|
|---|
| 839 | <li>Chipset I/O MMU capability.</li>
|
|---|
| 840 | <li>Memory size.</li>
|
|---|
| 841 | <li>Scratch size space (for testing).</li>
|
|---|
| 842 | <li>Testbox Script revision.</li>
|
|---|
| 843 | </ul>
|
|---|
| 844 | </dd>
|
|---|
| 845 | <dt>Results:</dt>
|
|---|
| 846 | <dd><ul class="first last simple">
|
|---|
| 847 | <li>ACK or NACK.</li>
|
|---|
| 848 | <li>Testbox ID and name on ACK.</li>
|
|---|
| 849 | </ul>
|
|---|
| 850 | </dd>
|
|---|
| 851 | </dl>
|
|---|
| 852 | <p>After receiving a ACK the testbox will ask for work to do, i.e. continue with
|
|---|
| 853 | scenario #2. In the NACK case, it will sleep for 60 seconds and try again.</p>
|
|---|
| 854 | <p>Actions:</p>
|
|---|
| 855 | <ol class="arabic">
|
|---|
| 856 | <li><p class="first">Validate the testbox by looking the UUID up in the TestBoxes table.
|
|---|
| 857 | If not found, NACK the request. SQL:</p>
|
|---|
| 858 | <pre class="literal-block">
|
|---|
| 859 | SELECT idTestBox, sName
|
|---|
| 860 | FROM TestBoxes
|
|---|
| 861 | WHERE uuidSystem = :sUuid
|
|---|
| 862 | AND tsExpire = 'infinity'::timestamp;
|
|---|
| 863 | </pre>
|
|---|
| 864 | </li>
|
|---|
| 865 | <li><p class="first">Check if any of the information by testbox script has changed. The two
|
|---|
| 866 | sizes are normalized first, memory size rounded to nearest 4 MB and scratch
|
|---|
| 867 | space is rounded down to nearest 64 MB. If anything changed, insert a new
|
|---|
| 868 | row in the testbox table and historize the current one, i.e. set
|
|---|
| 869 | OLD.tsExpire to NEW.tsEffective and get a new value for NEW.idGenTestBox.</p>
|
|---|
| 870 | </li>
|
|---|
| 871 | <li><dl class="first docutils">
|
|---|
| 872 | <dt>Check with TestBoxStatuses:</dt>
|
|---|
| 873 | <dd><ol class="first last loweralpha simple">
|
|---|
| 874 | <li>If there is an row for the testbox in it already clean up change it
|
|---|
| 875 | to 'idle' state and deal with any open testset like described in
|
|---|
| 876 | scenario #9.</li>
|
|---|
| 877 | <li>If there is no row, add one with 'idle' state.</li>
|
|---|
| 878 | </ol>
|
|---|
| 879 | </dd>
|
|---|
| 880 | </dl>
|
|---|
| 881 | </li>
|
|---|
| 882 | <li><p class="first">ACK the request and pass back the idTestBox.</p>
|
|---|
| 883 | </li>
|
|---|
| 884 | </ol>
|
|---|
| 885 | <dl class="docutils">
|
|---|
| 886 | <dt>Note! Testbox.enabled is not checked here, that is only relevant when it asks</dt>
|
|---|
| 887 | <dd>for a new task (scenario #2 and #5).</dd>
|
|---|
| 888 | <dt>Note! Should the testbox script detect changes in any of the inputs, it should</dt>
|
|---|
| 889 | <dd>redo the sign in.</dd>
|
|---|
| 890 | <dt>Note! In scenario #8, the box will not sign on until it has done the reboot and</dt>
|
|---|
| 891 | <dd>cleanup reporting!</dd>
|
|---|
| 892 | </dl>
|
|---|
| 893 | </div>
|
|---|
| 894 | <div class="section" id="testbox-asks-for-work-to-do">
|
|---|
| 895 | <h2>#2 - Testbox Asks For Work To Do</h2>
|
|---|
| 896 | <dl class="docutils">
|
|---|
| 897 | <dt>Inputs:</dt>
|
|---|
| 898 | <dd><ul class="first last simple">
|
|---|
| 899 | <li>The testbox is supplying its IP indirectly.</li>
|
|---|
| 900 | <li>The testbox should supply its UUID and ID directly.</li>
|
|---|
| 901 | </ul>
|
|---|
| 902 | </dd>
|
|---|
| 903 | <dt>Results:</dt>
|
|---|
| 904 | <dd><ul class="first last simple">
|
|---|
| 905 | <li>IDLE, WAIT, EXEC, REBOOT, UPGRADE, UPGRADE-AND-REBOOT, SPECIAL or DEAD.</li>
|
|---|
| 906 | </ul>
|
|---|
| 907 | </dd>
|
|---|
| 908 | </dl>
|
|---|
| 909 | <p>Actions:</p>
|
|---|
| 910 | <ol class="arabic">
|
|---|
| 911 | <li><p class="first">Validate the ID and IP by selecting the currently valid testbox row:</p>
|
|---|
| 912 | <pre class="literal-block">
|
|---|
| 913 | SELECT idGenTestBox, fEnabled, idSchedGroup, enmPendingCmd
|
|---|
| 914 | FROM TestBoxes
|
|---|
| 915 | WHERE id = :id
|
|---|
| 916 | AND uuidSystem = :sUuid
|
|---|
| 917 | AND ip = :ip
|
|---|
| 918 | AND tsExpire = 'infinity'::timestamp;
|
|---|
| 919 | </pre>
|
|---|
| 920 | <p>If NOT found return DEAD to the testbox client (it will go back to sign on
|
|---|
| 921 | mode and retry every 60 seconds or so - see scenario #1).</p>
|
|---|
| 922 | <dl class="docutils">
|
|---|
| 923 | <dt>Note! The WUI will do all necessary clean-ups when deleting a testbox, so</dt>
|
|---|
| 924 | <dd><p class="first last">contrary to the initial plans, we don't need to do anything more for
|
|---|
| 925 | the DEAD status.</p>
|
|---|
| 926 | </dd>
|
|---|
| 927 | </dl>
|
|---|
| 928 | </li>
|
|---|
| 929 | <li><p class="first">Check with TestBoxStatuses (maybe joined with query from 1).</p>
|
|---|
| 930 | <p>If enmState is 'gang-gathering': Goto scenario #6 on timeout or pending
|
|---|
| 931 | 'abort' or 'reboot' command. Otherwise, tell the testbox to WAIT [done].</p>
|
|---|
| 932 | <p>If enmState is 'gang-testing': The gang has been gathered and execution
|
|---|
| 933 | has been triggered. Goto 5.</p>
|
|---|
| 934 | <p>If enmState is not 'idle', change it to 'idle'.</p>
|
|---|
| 935 | <p>If idTestSet is not NULL, CALL scenario #9 to it up.</p>
|
|---|
| 936 | <p>If there is a pending abort command, remove it.</p>
|
|---|
| 937 | <p>If there is a pending command and the old state doesn't indicate that it was
|
|---|
| 938 | being executed, GOTO scenario #3.</p>
|
|---|
| 939 | <dl class="docutils">
|
|---|
| 940 | <dt>Note! There should be a TestBoxStatuses row after executing scenario #1,</dt>
|
|---|
| 941 | <dd><p class="first last">however should none be found for some funky reason, returning DEAD
|
|---|
| 942 | will fix the problem (see above)</p>
|
|---|
| 943 | </dd>
|
|---|
| 944 | </dl>
|
|---|
| 945 | </li>
|
|---|
| 946 | <li><p class="first">If the testbox was marked as disabled, respond with an IDLE command to the
|
|---|
| 947 | testbox [done]. (Note! Must do this after TestBoxStatuses maintenance from
|
|---|
| 948 | point 2, or abandoned tests won't be cleaned up after a testbox is disabled.)</p>
|
|---|
| 949 | </li>
|
|---|
| 950 | <li><p class="first">Consider testcases in the scheduling queue, pick the first one which the
|
|---|
| 951 | testbox can execute. There is a concurrency issue here, so we put and
|
|---|
| 952 | exclusive lock on the SchedQueues table while considering its content.</p>
|
|---|
| 953 | <p>The cursor we open looks something like this:</p>
|
|---|
| 954 | <pre class="literal-block">
|
|---|
| 955 | SELECT idItem, idGenTestCaseArgs,
|
|---|
| 956 | idTestSetGangLeader, cMissingGangMembers
|
|---|
| 957 | FROM SchedQueues
|
|---|
| 958 | WHERE idSchedGroup = :idSchedGroup
|
|---|
| 959 | AND ( bmHourlySchedule is NULL
|
|---|
| 960 | OR get_bit(bmHourlySchedule, :iHourOfWeek) = 1 ) --< does this work?
|
|---|
| 961 | ORDER BY ASC idItem;
|
|---|
| 962 | </pre>
|
|---|
| 963 | </li>
|
|---|
| 964 | </ol>
|
|---|
| 965 | <blockquote>
|
|---|
| 966 | <p>If there no rows are returned (this can happen because no testgroups are
|
|---|
| 967 | associated with this scheduling group, the scheduling group is disabled,
|
|---|
| 968 | or because the queue is being regenerated), we will tell the testbox to
|
|---|
| 969 | IDLE [done].</p>
|
|---|
| 970 | <dl class="docutils">
|
|---|
| 971 | <dt>For each returned row we will:</dt>
|
|---|
| 972 | <dd><ol class="first last loweralpha">
|
|---|
| 973 | <li><p class="first">Check testcase/group dependencies.</p>
|
|---|
| 974 | </li>
|
|---|
| 975 | <li><p class="first">Select a build (and default testsuite) satisfying the dependencies.</p>
|
|---|
| 976 | </li>
|
|---|
| 977 | <li><p class="first">Check the testcase requirements with that build in mind.</p>
|
|---|
| 978 | </li>
|
|---|
| 979 | <li><p class="first">If idTestSetGangLeader is NULL, try allocate the necessary resources.</p>
|
|---|
| 980 | </li>
|
|---|
| 981 | <li><p class="first">If it didn't check out, fetch the next row and redo from (a).</p>
|
|---|
| 982 | </li>
|
|---|
| 983 | <li><p class="first">Tentatively create a new test set row.</p>
|
|---|
| 984 | </li>
|
|---|
| 985 | <li><dl class="first docutils">
|
|---|
| 986 | <dt>If not gang scheduling:</dt>
|
|---|
| 987 | <dd><ul class="first last simple">
|
|---|
| 988 | <li>Next state: 'testing'</li>
|
|---|
| 989 | </ul>
|
|---|
| 990 | </dd>
|
|---|
| 991 | <dt>ElIf we're the last gang participant:</dt>
|
|---|
| 992 | <dd><ul class="first last simple">
|
|---|
| 993 | <li>Set idTestSetGangLeader to NULL.</li>
|
|---|
| 994 | <li>Set cMissingGangMembers to 0.</li>
|
|---|
| 995 | <li>Next state: 'gang-testing'</li>
|
|---|
| 996 | </ul>
|
|---|
| 997 | </dd>
|
|---|
| 998 | <dt>ElIf we're the first gang member:</dt>
|
|---|
| 999 | <dd><ul class="first last simple">
|
|---|
| 1000 | <li>Set cMissingGangMembers to TestCaseArgs.cGangMembers - 1.</li>
|
|---|
| 1001 | <li>Set idTestSetGangLeader to our idTestSet.</li>
|
|---|
| 1002 | <li>Next state: 'gang-gathering'</li>
|
|---|
| 1003 | </ul>
|
|---|
| 1004 | </dd>
|
|---|
| 1005 | <dt>Else:</dt>
|
|---|
| 1006 | <dd><ul class="first last simple">
|
|---|
| 1007 | <li>Decrement cMissingGangMembers.</li>
|
|---|
| 1008 | <li>Next state: 'gang-gathering'</li>
|
|---|
| 1009 | </ul>
|
|---|
| 1010 | </dd>
|
|---|
| 1011 | <dt>If we're not gang scheduling OR cMissingGangMembers is 0:</dt>
|
|---|
| 1012 | <dd><p class="first last">Move the scheduler queue entry to the end of the queue.</p>
|
|---|
| 1013 | </dd>
|
|---|
| 1014 | </dl>
|
|---|
| 1015 | <p>Update our TestBoxStatuses row with the new state and test set.
|
|---|
| 1016 | COMMIT;</p>
|
|---|
| 1017 | </li>
|
|---|
| 1018 | </ol>
|
|---|
| 1019 | </dd>
|
|---|
| 1020 | </dl>
|
|---|
| 1021 | </blockquote>
|
|---|
| 1022 | <ol class="arabic" start="5">
|
|---|
| 1023 | <li><dl class="first docutils">
|
|---|
| 1024 | <dt>If state is 'testing' or 'gang-testing':</dt>
|
|---|
| 1025 | <dd><p class="first">EXEC reponse.</p>
|
|---|
| 1026 | <p class="last">The EXEC response for a gang scheduled testcase includes a number of
|
|---|
| 1027 | extra arguments so that the script knows the position of the testbox
|
|---|
| 1028 | it is running on and of the other members. This means the that the
|
|---|
| 1029 | TestSet.iGangMemberNo is passed using --gang-member-no and the IP
|
|---|
| 1030 | addresses of the all gang members using --gang-ipv4-<memb-no> <ip>.</p>
|
|---|
| 1031 | </dd>
|
|---|
| 1032 | <dt>Else (state is 'gang-gathering'):</dt>
|
|---|
| 1033 | <dd><p class="first last">WAIT</p>
|
|---|
| 1034 | </dd>
|
|---|
| 1035 | </dl>
|
|---|
| 1036 | </li>
|
|---|
| 1037 | </ol>
|
|---|
| 1038 | </div>
|
|---|
| 1039 | <div class="section" id="pending-command-when-testbox-asks-for-work">
|
|---|
| 1040 | <h2>#3 - Pending Command When Testbox Asks For Work</h2>
|
|---|
| 1041 | <p>This is a subfunction of scenario #2 and #5.</p>
|
|---|
| 1042 | <p>As seen in scenario #2, the testbox will send 'abort' commands to /dev/null
|
|---|
| 1043 | when it finds one when not executing a test. This includes when it reports
|
|---|
| 1044 | that the test has completed (no need to abort a completed test, wasting lot
|
|---|
| 1045 | of effort when standing at the finish line).</p>
|
|---|
| 1046 | <p>The other commands, though, are passed back to the testbox. The testbox
|
|---|
| 1047 | script will respond with an ACK or NACK as it sees fit. If NACKed, the
|
|---|
| 1048 | pending command will be removed (pending_cmd set to none) and that's it.
|
|---|
| 1049 | If ACKed, the state of the testbox will change to that appropriate for the
|
|---|
| 1050 | command and the pending_cmd set to none. Should the testbox script fail to
|
|---|
| 1051 | respond, the command will be repeated the next time it asks for work.</p>
|
|---|
| 1052 | </div>
|
|---|
| 1053 | <div class="section" id="testbox-uploads-results-during-test">
|
|---|
| 1054 | <h2>#4 - Testbox Uploads Results During Test</h2>
|
|---|
| 1055 | <p>TODO</p>
|
|---|
| 1056 | </div>
|
|---|
| 1057 | <div class="section" id="testbox-completes-test-and-asks-for-work">
|
|---|
| 1058 | <h2>#5 - Testbox Completes Test and Asks For Work</h2>
|
|---|
| 1059 | <p>This is very similar to scenario #2</p>
|
|---|
| 1060 | <p>TODO</p>
|
|---|
| 1061 | </div>
|
|---|
| 1062 | <div class="section" id="gang-gathering-timeout">
|
|---|
| 1063 | <h2>#6 - Gang Gathering Timeout</h2>
|
|---|
| 1064 | <p>This is a subfunction of scenario #2.</p>
|
|---|
| 1065 | <p>When gathering a gang of testboxes for a testcase, we do not want to wait
|
|---|
| 1066 | forever and have testboxes doing nothing for hours while waiting for partners.
|
|---|
| 1067 | So, the gathering has a reasonable timeout (imagine something like 20-30 mins).</p>
|
|---|
| 1068 | <p>Also, we need some way of dealing with 'abort' and 'reboot' commands being
|
|---|
| 1069 | issued while waiting. The easy way out is pretend it's a time out.</p>
|
|---|
| 1070 | <p>When changing the status to 'gang-timeout' we have to be careful. First of all,
|
|---|
| 1071 | we need to exclusively lock the SchedQueues and TestBoxStatuses (in that order)
|
|---|
| 1072 | and re-query our status. If it changed redo the checks in scenario #2 point 2.</p>
|
|---|
| 1073 | <p>If we still want to timeout/abort, change the state from 'gang-gathering' to
|
|---|
| 1074 | 'gang-gathering-timedout' on all the gang members that has gathered so far.
|
|---|
| 1075 | Then reset the scheduling queue record and move it to the end of the queue.</p>
|
|---|
| 1076 | <p>When acting on 'gang-timeout' the TM will fail the testset in a manner similar
|
|---|
| 1077 | to scenario #9. No need to repeat that.</p>
|
|---|
| 1078 | </div>
|
|---|
| 1079 | <div class="section" id="gang-cleanup">
|
|---|
| 1080 | <h2>#7 - Gang Cleanup</h2>
|
|---|
| 1081 | <p>When a testbox completes a gang scheduled test, we will have to serialize
|
|---|
| 1082 | resource cleanup (both globally and on testboxes) as they stop. More details
|
|---|
| 1083 | can be found in the documentation of 'gang-cleanup'.</p>
|
|---|
| 1084 | <p>So, the transition from 'gang-testing' is always to 'gang-cleanup'. When we
|
|---|
| 1085 | can safely leave 'gang-cleanup' is decided by the query:</p>
|
|---|
| 1086 | <pre class="literal-block">
|
|---|
| 1087 | SELECT COUNT(*)
|
|---|
| 1088 | FROM TestBoxStatuses,
|
|---|
| 1089 | TestSets
|
|---|
| 1090 | WHERE TestSets.idTestSetGangLeader = :idTestSetGangLeader
|
|---|
| 1091 | AND TestSets.idTestBox = TestBoxStatuses.idTestBox
|
|---|
| 1092 | AND TestBoxStatuses.enmState = 'gang-running'::TestBoxState_T;
|
|---|
| 1093 | </pre>
|
|---|
| 1094 | <p>As long as there are testboxes still running, we stay in the 'gang-cleanup'
|
|---|
| 1095 | state. Once there are none, we continue closing the testset and such.</p>
|
|---|
| 1096 | </div>
|
|---|
| 1097 | <div class="section" id="testbox-reports-a-crash-during-test-execution">
|
|---|
| 1098 | <h2>#8 - Testbox Reports A Crash During Test Execution</h2>
|
|---|
| 1099 | <p>TODO</p>
|
|---|
| 1100 | </div>
|
|---|
| 1101 | <div class="section" id="cleaning-up-abandoned-testcase">
|
|---|
| 1102 | <h2>#9 - Cleaning Up Abandoned Testcase</h2>
|
|---|
| 1103 | <p>This is a subfunction of scenario #1 and #2. The actions taken are the same in
|
|---|
| 1104 | both situations. The precondition for taking this path is that the row in the
|
|---|
| 1105 | testboxstatus table is referring to a testset (i.e. testset_id is not NULL).</p>
|
|---|
| 1106 | <p>Actions:</p>
|
|---|
| 1107 | <ol class="arabic simple">
|
|---|
| 1108 | <li><dl class="first docutils">
|
|---|
| 1109 | <dt>If the testset is incomplete, we need to completed:</dt>
|
|---|
| 1110 | <dd><ol class="first last loweralpha">
|
|---|
| 1111 | <li>Add a message to the root TestResults row, creating one if necessary,
|
|---|
| 1112 | that explains that the test was abandoned. This is done
|
|---|
| 1113 | by inserting/finding the string into/in TestResultStrTab and adding
|
|---|
| 1114 | a row to TestResultMsgs with idStrMsg set to that string id and
|
|---|
| 1115 | enmLevel set to 'failure'.</li>
|
|---|
| 1116 | <li>Mark the testset as failed.</li>
|
|---|
| 1117 | </ol>
|
|---|
| 1118 | </dd>
|
|---|
| 1119 | </dl>
|
|---|
| 1120 | </li>
|
|---|
| 1121 | <li>Free any global resources referenced by the test set. This is done by
|
|---|
| 1122 | deleting all rows in GlobalResourceStatuses matching the testbox id.</li>
|
|---|
| 1123 | <li>Set the idTestSet to NULL in the TestBoxStatuses row.</li>
|
|---|
| 1124 | </ol>
|
|---|
| 1125 | </div>
|
|---|
| 1126 | <div class="section" id="cleaning-up-a-disabled-dead-testbox">
|
|---|
| 1127 | <h2>#10 - Cleaning Up a Disabled/Dead TestBox</h2>
|
|---|
| 1128 | <p>The UI needs to be able to clean up the remains of a testbox which for some
|
|---|
| 1129 | reason is out of action. Normal cleaning up of abandoned testcases requires
|
|---|
| 1130 | that the testbox signs on or asks for work, but if the testbox is dead or
|
|---|
| 1131 | in some way indisposed, it won't be doing any of that. So, the testbox
|
|---|
| 1132 | sheriff needs to have a way of cleaning up after it.</p>
|
|---|
| 1133 | <p>It's basically a manual scenario #9 but with some safe guards, like checking
|
|---|
| 1134 | that the box hasn't been active for the last 1-2 mins (max idle/wait time * 2).</p>
|
|---|
| 1135 | <dl class="docutils">
|
|---|
| 1136 | <dt>Note! When disabling a box that still executing the testbox script, this</dt>
|
|---|
| 1137 | <dd>cleanup isn't necessary as it will happen automatically. Also, it's
|
|---|
| 1138 | probably desirable that the testbox finishes what ever it is doing first
|
|---|
| 1139 | before going dormant.</dd>
|
|---|
| 1140 | </dl>
|
|---|
| 1141 | </div>
|
|---|
| 1142 | </div>
|
|---|
| 1143 | <div class="section" id="test-manager-analysis">
|
|---|
| 1144 | <h1>Test Manager: Analysis</h1>
|
|---|
| 1145 | <p>One of the testbox sheriff's tasks is to try figure out the reason why something
|
|---|
| 1146 | failed. The test manager will provide facilities for doing so from very early
|
|---|
| 1147 | in it's implementation.</p>
|
|---|
| 1148 | <p>We need to work out some useful status reports for the early implementation.
|
|---|
| 1149 | Later there will be more advanced analysis tools, where for instance we can
|
|---|
| 1150 | create graphs from selected test result values or test execution times.</p>
|
|---|
| 1151 | </div>
|
|---|
| 1152 | <div class="section" id="implementation-plan">
|
|---|
| 1153 | <h1>Implementation Plan</h1>
|
|---|
| 1154 | <p>This has changed for various reasons. The current plan is to implement the
|
|---|
| 1155 | infrastructure (TM & testbox script) first and do a small deployment with the
|
|---|
| 1156 | 2-5 test drivers in the Testsuite as basis. Once the bugs are worked out, we
|
|---|
| 1157 | will convert the rest of the tests and start adding new ones.</p>
|
|---|
| 1158 | <p>We just need to finally get this done, no point in doing it piecemeal by now!</p>
|
|---|
| 1159 | <div class="section" id="test-manager-implementation-sub-tasks">
|
|---|
| 1160 | <h2>Test Manager Implementation Sub-Tasks</h2>
|
|---|
| 1161 | <p>The implementation of the test manager and adjusting/completing of the testbox
|
|---|
| 1162 | script and the test drivers are tasks which can be done by more than one
|
|---|
| 1163 | person. Splitting up the TM implementation into smaller tasks should allow
|
|---|
| 1164 | parallel development of different tasks and get us working code sooner.</p>
|
|---|
| 1165 | </div>
|
|---|
| 1166 | <div class="section" id="milestone-1">
|
|---|
| 1167 | <h2>Milestone #1</h2>
|
|---|
| 1168 | <p>The goal is to getting the fundamental testmanager engine implemented, debugged
|
|---|
| 1169 | and working. With the exception of testboxes, the configuration will be done
|
|---|
| 1170 | via SQL inserts.</p>
|
|---|
| 1171 | <p>Tasks in somewhat prioritized order:</p>
|
|---|
| 1172 | <blockquote>
|
|---|
| 1173 | <ul class="simple">
|
|---|
| 1174 | <li>Kick off test manager. It will live in testmanager/. Salvage as much as
|
|---|
| 1175 | possible from att/testserv. Create basic source and file layout.</li>
|
|---|
| 1176 | <li>Adjust the testbox script, part one. There currently is a testbox script
|
|---|
| 1177 | in att/testbox, this shall be moved up into testboxscript/. The script
|
|---|
| 1178 | needs to be adjusted according to the specification layed down earlier
|
|---|
| 1179 | in this document. Installers or installation scripts for all relevant
|
|---|
| 1180 | host OSes are required. Left for part two is result reporting beyond the
|
|---|
| 1181 | primary log. This task must be 100% feature complete, on all host OSes,
|
|---|
| 1182 | there is no room for FIXME, XXX or @todo here.</li>
|
|---|
| 1183 | <li>Implement the schedule queue generator.</li>
|
|---|
| 1184 | <li>Implement the testbox dispatcher in TM. Support all the testbox script
|
|---|
| 1185 | responses implemented above, including upgrading the testbox script.</li>
|
|---|
| 1186 | <li>Implement simple testbox management page.</li>
|
|---|
| 1187 | <li>Implement some basic activity and result reports so that we can see
|
|---|
| 1188 | what's going on.</li>
|
|---|
| 1189 | <li>Create a testmanager / testbox test setup. This lives in selftest/.<ol class="arabic">
|
|---|
| 1190 | <li>Set up something that runs, no fiddly bits. Debug till it works.</li>
|
|---|
| 1191 | <li>Create a setup that tests testgroup dependencies, i.e. real tests
|
|---|
| 1192 | depending on smoke tests.</li>
|
|---|
| 1193 | <li>Create a setup that exercises testcase dependency.</li>
|
|---|
| 1194 | <li>Create a setup that exercises global resource allocation.</li>
|
|---|
| 1195 | <li>Create a setup that exercises gang scheduling.</li>
|
|---|
| 1196 | </ol>
|
|---|
| 1197 | </li>
|
|---|
| 1198 | <li>Check that all features work.</li>
|
|---|
| 1199 | </ul>
|
|---|
| 1200 | </blockquote>
|
|---|
| 1201 | </div>
|
|---|
| 1202 | <div class="section" id="milestone-2">
|
|---|
| 1203 | <h2>Milestone #2</h2>
|
|---|
| 1204 | <p>The goal is getting to VBox testing.</p>
|
|---|
| 1205 | <p>Tasks in somewhat prioritized order:</p>
|
|---|
| 1206 | <blockquote>
|
|---|
| 1207 | <ul class="simple">
|
|---|
| 1208 | <li>Implement full result reporting in the testbox script and testbox driver.
|
|---|
| 1209 | A testbox script specific reporter needs to be implemented for the
|
|---|
| 1210 | testdriver framework. The testbox script needs to forward the results to
|
|---|
| 1211 | the test manager, or alternatively the testdriver report can talk
|
|---|
| 1212 | directly to the TM.</li>
|
|---|
| 1213 | <li>Implement the test manager side of the test result reporting.</li>
|
|---|
| 1214 | <li>Extend the selftest with some setup that report all kinds of test
|
|---|
| 1215 | results.</li>
|
|---|
| 1216 | <li>Implement script/whatever feeding builds to the test manager from the
|
|---|
| 1217 | tinderboxes.</li>
|
|---|
| 1218 | <li>The toplevel test driver is a VBox thing that must be derived from the
|
|---|
| 1219 | base TestDriver class or maybe the VBox one. It should move from
|
|---|
| 1220 | toptestdriver to testdriver and be renamed to vboxtltd or smth.</li>
|
|---|
| 1221 | <li>Create a vbox testdriver that boots the t-xppro VM once and that's it.</li>
|
|---|
| 1222 | <li>Create a selftest setup which tests booting t-xppro taking builds from
|
|---|
| 1223 | the tinderbox.</li>
|
|---|
| 1224 | </ul>
|
|---|
| 1225 | </blockquote>
|
|---|
| 1226 | </div>
|
|---|
| 1227 | <div class="section" id="milestone-3">
|
|---|
| 1228 | <h2>Milestone #3</h2>
|
|---|
| 1229 | <p>The goal for this milestone is configuration and converting current testcases,
|
|---|
| 1230 | the result will be the a minimal test deployment (4-5 new testboxes).</p>
|
|---|
| 1231 | <p>Tasks in somewhat prioritized order:</p>
|
|---|
| 1232 | <blockquote>
|
|---|
| 1233 | <ul class="simple">
|
|---|
| 1234 | <li>Implement testcase configuration.</li>
|
|---|
| 1235 | <li>Implement testgroup configuration.</li>
|
|---|
| 1236 | <li>Implement build source configuration.</li>
|
|---|
| 1237 | <li>Implement scheduling group configuration.</li>
|
|---|
| 1238 | <li>Implement global resource configuration.</li>
|
|---|
| 1239 | <li>Re-visit the testbox configuration.</li>
|
|---|
| 1240 | <li>Black listing of builds.</li>
|
|---|
| 1241 | <li>Implement simple failure analysis and reporting.</li>
|
|---|
| 1242 | <li>Implement the initial smoke tests modelled on the current smoke tests.</li>
|
|---|
| 1243 | <li>Implement installation tests for Windows guests.</li>
|
|---|
| 1244 | <li>Implement installation tests for Linux guests.</li>
|
|---|
| 1245 | <li>Implement installation tests for Solaris guest.</li>
|
|---|
| 1246 | <li>Implement installation tests for OS/2 guest.</li>
|
|---|
| 1247 | <li>Set up a small test deployment.</li>
|
|---|
| 1248 | </ul>
|
|---|
| 1249 | </blockquote>
|
|---|
| 1250 | </div>
|
|---|
| 1251 | <div class="section" id="further-work">
|
|---|
| 1252 | <h2>Further work</h2>
|
|---|
| 1253 | <p>After milestone #3 has been reached and issues found by the other team members
|
|---|
| 1254 | have been addressed, we will probably go for full deployment.</p>
|
|---|
| 1255 | <p>Beyond this point we will need to improve reporting and analysis. There may be
|
|---|
| 1256 | configuration aspects needing reporting as well.</p>
|
|---|
| 1257 | <p>Once deployed, a golden rule will be that all new features shall have test
|
|---|
| 1258 | coverage. Preferably, implemented by someone else and prior to the feature
|
|---|
| 1259 | implementation.</p>
|
|---|
| 1260 | </div>
|
|---|
| 1261 | </div>
|
|---|
| 1262 | <div class="section" id="discussion-logs">
|
|---|
| 1263 | <h1>Discussion Logs</h1>
|
|---|
| 1264 | <div class="section" id="various-discussions-with-michal-and-or-klaus">
|
|---|
| 1265 | <h2>2009-07-21,22,23 Various Discussions with Michal and/or Klaus</h2>
|
|---|
| 1266 | <ul class="simple">
|
|---|
| 1267 | <li>Scheduling of tests requiring more than one testbox.</li>
|
|---|
| 1268 | <li>Scheduling of tests that cannot be executing concurrently on several machines
|
|---|
| 1269 | because of some global resource like an iSCSI target.</li>
|
|---|
| 1270 | <li>Manually create the test config permutations instead of having the test
|
|---|
| 1271 | manager create all possible ones and wasting time.</li>
|
|---|
| 1272 | <li>Distinguish between built types so we can run smoke tests on strick builds as
|
|---|
| 1273 | well as release ones.</li>
|
|---|
| 1274 | </ul>
|
|---|
| 1275 | </div>
|
|---|
| 1276 | <div class="section" id="brief-discussion-with-michal">
|
|---|
| 1277 | <h2>2009-07-20 Brief Discussion with Michal</h2>
|
|---|
| 1278 | <ul class="simple">
|
|---|
| 1279 | <li>Installer for the testbox script to make bringing up a new testbox even
|
|---|
| 1280 | smoother.</li>
|
|---|
| 1281 | </ul>
|
|---|
| 1282 | </div>
|
|---|
| 1283 | <div class="section" id="raw-input">
|
|---|
| 1284 | <h2>2009-07-16 Raw Input</h2>
|
|---|
| 1285 | <ul class="simple">
|
|---|
| 1286 | <li><dl class="first docutils">
|
|---|
| 1287 | <dt>test set. recursive collection of:</dt>
|
|---|
| 1288 | <dd><ul class="first last">
|
|---|
| 1289 | <li>hierachical subtest name (slash sep)</li>
|
|---|
| 1290 | <li>test parameters / config</li>
|
|---|
| 1291 | <li>bool fail/succ</li>
|
|---|
| 1292 | <li>attributes (typed?)</li>
|
|---|
| 1293 | <li>test time</li>
|
|---|
| 1294 | <li>e.g. throughput</li>
|
|---|
| 1295 | <li>subresults</li>
|
|---|
| 1296 | <li>log</li>
|
|---|
| 1297 | <li>screenshots,....</li>
|
|---|
| 1298 | </ul>
|
|---|
| 1299 | </dd>
|
|---|
| 1300 | </dl>
|
|---|
| 1301 | </li>
|
|---|
| 1302 | <li>client package (zip) dl from server (maybe client caching)</li>
|
|---|
| 1303 | <li><dl class="first docutils">
|
|---|
| 1304 | <dt>thoughts on bits to do at once.</dt>
|
|---|
| 1305 | <dd><ul class="first last">
|
|---|
| 1306 | <li>We <em>really</em> need the basic bits ASAP.</li>
|
|---|
| 1307 | <li>client -> support for test driver</li>
|
|---|
| 1308 | <li>server -> controls configs</li>
|
|---|
| 1309 | <li>cleanup on both sides</li>
|
|---|
| 1310 | </ul>
|
|---|
| 1311 | </dd>
|
|---|
| 1312 | </dl>
|
|---|
| 1313 | </li>
|
|---|
| 1314 | </ul>
|
|---|
| 1315 | </div>
|
|---|
| 1316 | <div class="section" id="raw-input-1">
|
|---|
| 1317 | <h2>2009-07-15 Raw Input</h2>
|
|---|
| 1318 | <ul class="simple">
|
|---|
| 1319 | <li>testing should start automatically</li>
|
|---|
| 1320 | <li>switching to branch too tedious</li>
|
|---|
| 1321 | <li>useful to be able to partition testboxes (run specific builds on some boxes, let an engineer have a few boxes for a while).</li>
|
|---|
| 1322 | <li>test specification needs to be more flexible (select tests, disable test, test scheduling (run certain tests nightly), ... )</li>
|
|---|
| 1323 | <li>testcase dependencies (blacklisting builds, run smoketests on box A before long tests on box B, ...)</li>
|
|---|
| 1324 | <li>more testing flexibility, more test than just install/moke. For instance unit tests, benchmarks, ...</li>
|
|---|
| 1325 | <li>presentation/analysis: graphs!, categorize bugs, columns reorganizing grouped by test (hierarchical), overviews, result for last day.</li>
|
|---|
| 1326 | <li>testcase specificion, variables (e.g. I/O-APIC, SMP, HWVIRT, SATA...) as sub-tests</li>
|
|---|
| 1327 | <li>interation with ILOM/...: reset systems</li>
|
|---|
| 1328 | <li>Changes needs LDAP authentication</li>
|
|---|
| 1329 | <li>historize all configuration w/ name</li>
|
|---|
| 1330 | <li>ability to run testcase locally (provided the VDI/ISO/whatever extra requirements can be met).</li>
|
|---|
| 1331 | </ul>
|
|---|
| 1332 | <hr class="docutils" />
|
|---|
| 1333 | <table class="docutils footnote" frame="void" id="footnote-1" rules="none">
|
|---|
| 1334 | <colgroup><col class="label" /><col /></colgroup>
|
|---|
| 1335 | <tbody valign="top">
|
|---|
| 1336 | <tr><td class="label">[1]</td><td>no such footnote</td></tr>
|
|---|
| 1337 | </tbody>
|
|---|
| 1338 | </table>
|
|---|
| 1339 | <hr class="docutils" />
|
|---|
| 1340 | <table class="docutils field-list" frame="void" rules="none">
|
|---|
| 1341 | <col class="field-name" />
|
|---|
| 1342 | <col class="field-body" />
|
|---|
| 1343 | <tbody valign="top">
|
|---|
| 1344 | <tr class="field"><th class="field-name">Status:</th><td class="field-body">$Id: AutomaticTestingRevamp.html 98107 2023-01-17 22:56:50Z vboxsync $</td>
|
|---|
| 1345 | </tr>
|
|---|
| 1346 | <tr class="field"><th class="field-name">Copyright:</th><td class="field-body">Copyright (C) 2010-2023 Oracle Corporation.</td>
|
|---|
| 1347 | </tr>
|
|---|
| 1348 | </tbody>
|
|---|
| 1349 | </table>
|
|---|
| 1350 | </div>
|
|---|
| 1351 | </div>
|
|---|
| 1352 | </div>
|
|---|
| 1353 | </body>
|
|---|
| 1354 | </html>
|
|---|
