VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/tests/storage/remoteexecutor.py@ 103914

Last change on this file since 103914 was 103857, checked in by vboxsync, 9 months ago

ValKit, storage test, add more logging

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 11.4 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: remoteexecutor.py 103857 2024-03-14 15:39:03Z vboxsync $
3
4"""
5VirtualBox Validation Kit - Storage benchmark, test execution helpers.
6"""
7
8__copyright__ = \
9"""
10Copyright (C) 2016-2023 Oracle and/or its affiliates.
11
12This file is part of VirtualBox base platform packages, as
13available from https://www.virtualbox.org.
14
15This program is free software; you can redistribute it and/or
16modify it under the terms of the GNU General Public License
17as published by the Free Software Foundation, in version 3 of the
18License.
19
20This program is distributed in the hope that it will be useful, but
21WITHOUT ANY WARRANTY; without even the implied warranty of
22MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
23General Public License for more details.
24
25You should have received a copy of the GNU General Public License
26along with this program; if not, see <https://www.gnu.org/licenses>.
27
28The contents of this file may alternatively be used under the terms
29of the Common Development and Distribution License Version 1.0
30(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
31in the VirtualBox distribution, in which case the provisions of the
32CDDL are applicable instead of those of the GPL.
33
34You may elect to license modified versions of this file under the
35terms and conditions of either the GPL or the CDDL or both.
36
37SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
38"""
39__version__ = "$Revision: 103857 $"
40
41
42# Standard Python imports.
43import array;
44import os;
45import shutil;
46import sys;
47if sys.version_info[0] >= 3:
48 from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
49else:
50 from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
51import subprocess;
52
53# Validation Kit imports.
54from common import utils;
55from testdriver import reporter;
56
57
58
59class StdInOutBuffer(object):
60 """ Standard input output buffer """
61
62 def __init__(self, sInput = None):
63 self.sInput = StringIO();
64 if sInput is not None:
65 self.sInput.write(self._toString(sInput));
66 self.sInput.seek(0);
67 self.sOutput = '';
68
69 def _toString(self, sText):
70 """
71 Converts any possible array to
72 a string.
73 """
74 if isinstance(sText, array.array):
75 try:
76 if sys.version_info < (3, 9, 0):
77 # Removed since Python 3.9.
78 return str(sText.tostring()); # pylint: disable=no-member
79 return str(sText.tobytes());
80 except:
81 pass;
82 elif isinstance(sText, bytes):
83 return sText.decode('utf-8');
84
85 return sText;
86
87 def read(self, cb):
88 """file.read"""
89 return self.sInput.read(cb);
90
91 def write(self, sText):
92 """file.write"""
93 self.sOutput += self._toString(sText);
94 return None;
95
96 def getOutput(self):
97 """
98 Returns the output of the buffer.
99 """
100 return self.sOutput;
101
102 def close(self):
103 """ file.close """
104 return;
105
106class RemoteExecutor(object):
107 """
108 Helper for executing tests remotely through TXS or locally
109 """
110
111 def __init__(self, oTxsSession = None, asBinaryPaths = None, sScratchPath = None):
112 self.oTxsSession = oTxsSession;
113 self.asPaths = asBinaryPaths;
114 self.sScratchPath = sScratchPath;
115 if self.asPaths is None:
116 self.asPaths = [ ];
117
118 def _getBinaryPath(self, sBinary):
119 """
120 Returns the complete path of the given binary if found
121 from the configured search path or None if not found.
122 """
123 for sPath in self.asPaths:
124 sFile = sPath + '/' + sBinary;
125 if self.isFile(sFile):
126 return sFile;
127 return sBinary;
128
129 def _sudoExecuteSync(self, asArgs, sInput):
130 """
131 Executes a sudo child process synchronously.
132 Returns a tuple [True, 0] if the process executed successfully
133 and returned 0, otherwise [False, rc] is returned.
134 """
135 reporter.log('Executing [sudo]: %s' % (asArgs, ));
136 reporter.flushall();
137 fRc = True;
138 sOutput = '';
139 sError = '';
140 try:
141 oProcess = utils.sudoProcessPopen(asArgs, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
142 stderr=subprocess.PIPE, shell = False, close_fds = False);
143
144 sOutput, sError = oProcess.communicate(sInput);
145 iExitCode = oProcess.poll();
146
147 if iExitCode != 0:
148 fRc = False;
149 except:
150 reporter.errorXcpt();
151 fRc = False;
152 reporter.log('Exit code [sudo]: %s (%s)' % (fRc, asArgs));
153 return (fRc, str(sOutput), str(sError));
154
155 def _execLocallyOrThroughTxs(self, sExec, asArgs, sInput, cMsTimeout):
156 """
157 Executes the given program locally or through TXS based on the
158 current config.
159 """
160 fRc = False;
161 sOutput = None;
162 if self.oTxsSession is not None:
163 reporter.log('Executing [remote]: %s %s %s' % (sExec, asArgs, sInput));
164 reporter.flushall();
165 oStdOut = StdInOutBuffer();
166 oStdErr = StdInOutBuffer();
167 oTestPipe = reporter.FileWrapperTestPipe();
168 oStdIn = None;
169 if sInput is not None:
170 oStdIn = StdInOutBuffer(sInput);
171 else:
172 oStdIn = '/dev/null'; # pylint: disable=redefined-variable-type
173 fRc = self.oTxsSession.syncExecEx(sExec, (sExec,) + asArgs,
174 oStdIn = oStdIn, oStdOut = oStdOut,
175 oStdErr = oStdErr, oTestPipe = oTestPipe,
176 cMsTimeout = cMsTimeout);
177 sOutput = oStdOut.getOutput();
178 sError = oStdErr.getOutput();
179 if fRc is False:
180 reporter.log('Exit code [remote]: %s (stdout: %s stderr: %s)' % (fRc, sOutput, sError));
181 else:
182 reporter.log('Exit code [remote]: %s' % (fRc,));
183 else:
184 fRc, sOutput, sError = self._sudoExecuteSync([sExec, ] + list(asArgs), sInput);
185 return (fRc, sOutput, sError);
186
187 def execBinary(self, sExec, asArgs, sInput = None, cMsTimeout = 3600000):
188 """
189 Executes the given binary with the given arguments
190 providing some optional input through stdin and
191 returning whether the process exited successfully and the output
192 in a string.
193 """
194
195 fRc = True;
196 sOutput = None;
197 sError = None;
198 sBinary = self._getBinaryPath(sExec);
199 if sBinary is not None:
200 fRc, sOutput, sError = self._execLocallyOrThroughTxs(sBinary, asArgs, sInput, cMsTimeout);
201 # If verbose logging is enabled and the process failed for whatever reason, log its output to the reporter.
202 fLog = False;
203 if not fRc \
204 and reporter.getVerbosity() >= 2: # Verbose logging starts at level 2.
205 fLog = True;
206 elif reporter.getVerbosity() >= 3:
207 fLog = True;
208
209 if fLog:
210 asOutput = sOutput.splitlines();
211 for sLine in asOutput:
212 reporter.log('%s [stdout]: %s' % (sExec, sLine.encode(encoding = 'UTF-8', errors = 'strict'),));
213 asError = sError.splitlines();
214 for sLine in asError:
215 reporter.log('%s [stderr]: %s' % (sExec, sLine.encode(encoding = 'UTF-8', errors = 'strict'),));
216 else:
217 fRc = False;
218 return (fRc, sOutput, sError);
219
220 def execBinaryNoStdOut(self, sExec, asArgs, sInput = None):
221 """
222 Executes the given binary with the given arguments
223 providing some optional input through stdin and
224 returning whether the process exited successfully.
225 """
226 fRc, _, _ = self.execBinary(sExec, asArgs, sInput);
227 return fRc;
228
229 def copyFile(self, sLocalFile, sFilename, cMsTimeout = 30000):
230 """
231 Copies the local file to the remote destination
232 if configured
233
234 Returns a file ID which can be used as an input parameter
235 to execBinary() resolving to the real filepath on the remote side
236 or locally.
237 """
238 sFileId = None;
239 if self.oTxsSession is not None:
240 sFileId = '${SCRATCH}/' + sFilename;
241 fRc = self.oTxsSession.syncUploadFile(sLocalFile, sFileId, cMsTimeout);
242 if not fRc:
243 sFileId = None;
244 else:
245 sFileId = self.sScratchPath + '/' + sFilename;
246 try:
247 shutil.copy(sLocalFile, sFileId);
248 except:
249 sFileId = None;
250
251 return sFileId;
252
253 def copyString(self, sContent, sFilename, cMsTimeout = 30000):
254 """
255 Creates a file remotely or locally with the given content.
256
257 Returns a file ID which can be used as an input parameter
258 to execBinary() resolving to the real filepath on the remote side
259 or locally.
260 """
261 sFileId = None;
262 if self.oTxsSession is not None:
263 sFileId = '${SCRATCH}/' + sFilename;
264 fRc = self.oTxsSession.syncUploadString(sContent, sFileId, cMsTimeout);
265 if not fRc:
266 sFileId = None;
267 else:
268 sFileId = self.sScratchPath + '/' + sFilename;
269 try:
270 with open(sFileId, 'wb') as oFile:
271 oFile.write(sContent);
272 except:
273 sFileId = None;
274
275 return sFileId;
276
277 def mkDir(self, sDir, fMode = 0o700, cMsTimeout = 30000):
278 """
279 Creates a new directory at the given location.
280 """
281 fRc = True;
282 if self.oTxsSession is not None:
283 fRc = self.oTxsSession.syncMkDir(sDir, fMode, cMsTimeout, fIgnoreErrors=False);
284 elif not os.path.isdir(sDir):
285 reporter.log("if no txs session found and os.path.isdir is False do os.mkdir for %s" % sDir)
286 fRc = os.mkdir(sDir, fMode);
287
288 return fRc;
289
290 def rmDir(self, sDir, cMsTimeout = 30000):
291 """
292 Removes the given directory.
293 """
294 fRc = True;
295 if self.oTxsSession is not None:
296 fRc = self.oTxsSession.syncRmDir(sDir, cMsTimeout);
297 else:
298 fRc = self.execBinaryNoStdOut('rmdir', (sDir,));
299
300 return fRc;
301
302 def rmTree(self, sDir, cMsTimeout = 30000):
303 """
304 Recursively removes all files and subdirectories including the given directory.
305 """
306 fRc = True;
307 if self.oTxsSession is not None:
308 reporter.log("rmTree (%s) using txs" % sDir);
309 fRc = self.oTxsSession.syncRmTree(sDir, cMsTimeout);
310 else:
311 try:
312 if os.path.isfile(sDir):
313 reporter.log("deleting file: %s" % sDir)
314 os.remove(sDir)
315 else:
316 reporter.log("rmTree (%s) using shutil.rmtree function" % sDir);
317 shutil.rmtree(sDir);
318 except:
319 fRc = False;
320
321 return fRc;
322
323 def isFile(self, sPath, cMsTimeout = 30000):
324 """
325 Checks that the given file exists.
326 """
327 fRc = True;
328 if self.oTxsSession is not None:
329 fRc = self.oTxsSession.syncIsFile(sPath, cMsTimeout);
330 else:
331 try:
332 fRc = os.path.isfile(sPath);
333 except:
334 fRc = False;
335
336 return fRc;
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette