Index: /trunk/src/VBox/ValidationKit/testmanager/batch/vcs_import.py =================================================================== --- /trunk/src/VBox/ValidationKit/testmanager/batch/vcs_import.py (revision 84549) +++ /trunk/src/VBox/ValidationKit/testmanager/batch/vcs_import.py (revision 84550) @@ -2,5 +2,4 @@ # -*- coding: utf-8 -*- # $Id$ -# pylint: disable=line-too-long """ @@ -44,7 +43,14 @@ # Test Manager imports -from testmanager.core.db import TMDatabaseConnection; -from testmanager.core.vcsrevisions import VcsRevisionData, VcsRevisionLogic; -from common import utils; +from testmanager.config import g_kaBugTrackers; +from testmanager.core.db import TMDatabaseConnection; +from testmanager.core.vcsrevisions import VcsRevisionData, VcsRevisionLogic; +from testmanager.core.vcsbugreference import VcsBugReferenceData, VcsBugReferenceLogic; +from common import utils; + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + class VcsImport(object): # pylint: disable=too-few-public-methods @@ -52,4 +58,10 @@ Imports revision history from a VSC into the Test Manager database. """ + + class BugTracker(object): + def __init__(self, sDbName, sTag): + self.sDbName = sDbName; + self.sTag = sTag; + def __init__(self): @@ -59,5 +71,7 @@ oParser = OptionParser() - oParser.add_option('-e', '--extra-option', dest = 'asExtraOptions', action = 'append', + oParser.add_option('-b', '--only-bug-refs', dest = 'fBugRefsOnly', action = 'store_true', + help = 'Only do bug references, not revisions.'); + oParser.add_option('-e', '--extra-option', dest = 'asExtraOptions', metavar = 'vcsoption', action = 'append', help = 'Adds a extra option to the command retrieving the log.'); oParser.add_option('-f', '--full', dest = 'fFull', action = 'store_true', @@ -93,9 +107,13 @@ oDb = TMDatabaseConnection(); oLogic = VcsRevisionLogic(oDb); + oBugLogic = VcsBugReferenceLogic(oDb); # Where to start. iStartRev = 0; if not self.oConfig.fFull: - iStartRev = oLogic.getLastRevision(self.oConfig.sRepository); + if not self.oConfig.fBugRefsOnly: + iStartRev = oLogic.getLastRevision(self.oConfig.sRepository); + else: + iStartRev = oBugLogic.getLastRevision(self.oConfig.sRepository); if iStartRev == 0: iStartRev = self.oConfig.iStartRevision; @@ -118,12 +136,13 @@ # Parse the XML and add the entries to the database. oParser = ET.XMLParser(target = ET.TreeBuilder(), encoding = 'utf-8'); - oParser.feed(sLogXml.encode('utf-8')); # does its own decoding and processOutputChecked always gives us decoded utf-8 now. + oParser.feed(sLogXml.encode('utf-8')); # Does its own decoding; processOutputChecked always gives us decoded utf-8 now. oRoot = oParser.close(); for oLogEntry in oRoot.findall('logentry'): iRevision = int(oLogEntry.get('revision')); - sAuthor = oLogEntry.findtext('author').strip(); + sAuthor = oLogEntry.findtext('author', 'unspecified').strip(); # cvs2svn entries doesn't have an author. sDate = oLogEntry.findtext('date').strip(); - sMessage = oLogEntry.findtext('msg', '').strip(); + sRawMsg = oLogEntry.findtext('msg', '').strip(); + sMessage = sRawMsg; if sMessage == '': sMessage = ' '; @@ -133,6 +152,39 @@ utils.printOut(u'sDate=%s iRev=%u sAuthor=%s sMsg[%s]=%s' % (sDate, iRevision, sAuthor, type(sMessage).__name__, sMessage)); - oData = VcsRevisionData().initFromValues(self.oConfig.sRepository, iRevision, sDate, sAuthor, sMessage); - oLogic.addVcsRevision(oData); + + if not self.oConfig.fBugRefsOnly: + oData = VcsRevisionData().initFromValues(self.oConfig.sRepository, iRevision, sDate, sAuthor, sMessage); + oLogic.addVcsRevision(oData); + + # Analyze the raw message looking for bug tracker references. + for sBugTrackerKey in g_kaBugTrackers: + oBugTracker = g_kaBugTrackers[sBugTrackerKey]; + for sTag in oBugTracker.asCommitTags: + off = sRawMsg.find(sTag); + while off >= 0: + off += len(sTag); + while off < len(sRawMsg) and sRawMsg[off].isspace(): + off += 1; + + if off < len(sRawMsg) and sRawMsg[off].isdigit(): + offNum = off; + while off < len(sRawMsg) and sRawMsg[off].isdigit(): + off += 1; + try: + iBugNo = long(sRawMsg[offNum:off]); + except Exception as oXcpt: + utils.printErr(u'error! exception(r%s,"%s"): -> %s' % (iRevision, sRawMsg[offNum:off], oXcpt,)); + else: + if not self.oConfig.fQuiet: + utils.printOut(u' r%u -> sBugTracker=%s iBugNo=%s' + % (iRevision, oBugTracker.sDbId, iBugNo,)); + + oBugData = VcsBugReferenceData().initFromValues(self.oConfig.sRepository, iRevision, + oBugTracker.sDbId, iBugNo); + oBugLogic.addVcsBugReference(oBugData); + + # next + off = sRawMsg.find(sTag, off); + oDb.commit(); Index: /trunk/src/VBox/ValidationKit/testmanager/config.py =================================================================== --- /trunk/src/VBox/ValidationKit/testmanager/config.py (revision 84549) +++ /trunk/src/VBox/ValidationKit/testmanager/config.py (revision 84550) @@ -148,4 +148,47 @@ +## @name Bug Trackers and VCS reference tags. +## @{ +class BugTrackerConfig(object): + """ Bug tracker config """ + def __init__(self, sDbId, sName, sBugUrl, asCommitTags): + assert len(sDbId) == 4; + self.sDbId = sDbId; + self.sName = sName; + self.sBugUrl = sBugUrl; + self.asCommitTags = asCommitTags; + +## The key is the database table +g_kaBugTrackers = { + 'xtrk': BugTrackerConfig('xtrk', 'xTracker', 'https://linserv.de.oracle.com/vbox/xTracker/index.php?bug=', + ['bugref:', '@bugref{', 'bugef:', 'bugrf:', ], ), + 'bgdb': BugTrackerConfig('bgdb', 'BugDB', 'https://bug.oraclecorp.com/pls/bug/webbug_edit.edit_info_top?rptno=', + ['bugdbref:', '@bugdbref{', 'bugdb:', ], ), + 'vorg': BugTrackerConfig('vorg', 'External Trac', 'https://www.virtualbox.org/ticket/', + ['ticketref:', '@ticketref{', 'ticket:', ], ), +}; +## @} + + + +## @name Virtual Sheriff email alerts +## @{ + +## SMTP server host name. +g_ksSmtpHost = 'internal-mail-router.oracle.com'; +## SMTP server port number. +g_kcSmtpPort = 25; +## Default email 'From' for email alert. +g_ksAlertFrom = 'vsheriff@oracle.com'; +## Subject for email alert. +g_ksAlertSubject = 'Virtual Test Sheriff Alert'; +## List of users to send alerts. +g_asAlertList = ['lelik', 'werner']; +## iLOM password. +g_ksLomPassword = 'password'; + +## @} + + ## @name Partial Database Dump ## @{ @@ -207,19 +250,2 @@ ## @} -## @name Virtual Sheriff email alerts -## @{ - -## SMTP server host name. -g_ksSmtpHost = 'internal-mail-router.oracle.com'; -## SMTP server port number. -g_kcSmtpPort = 25; -## Default email 'From' for email alert. -g_ksAlertFrom = 'vsheriff@oracle.com'; -## Subject for email alert. -g_ksAlertSubject = 'Virtual Test Sheriff Alert'; -## List of users to send alerts. -g_asAlertList = ['lelik', 'werner']; -## iLOM password. -g_ksLomPassword = 'password'; - -## @} Index: /trunk/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py =================================================================== --- /trunk/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py (revision 84550) +++ /trunk/src/VBox/ValidationKit/testmanager/core/vcsbugreference.py (revision 84550) @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# $Id$ + +""" +Test Manager - VcsBugReferences +""" + +__copyright__ = \ +""" +Copyright (C) 2012-2020 Oracle Corporation + +This file is part of VirtualBox Open Source Edition (OSE), as +available from http://www.virtualbox.org. This file is free software; +you can redistribute it and/or modify it under the terms of the GNU +General Public License (GPL) as published by the Free Software +Foundation, in version 2 as it comes in the "COPYING" file of the +VirtualBox OSE distribution. VirtualBox OSE is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL) only, as it comes in the "COPYING.CDDL" file of the +VirtualBox OSE distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. +""" +__version__ = "$Revision$" + + +# Standard python imports. +import unittest; + +# Validation Kit imports. +from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase; + + +class VcsBugReferenceData(ModelDataBase): + """ + A version control system (VCS) bug tracker reference (commit message tag). + """ + + #kasIdAttr = ['sRepository','iRevision', 'sBugTracker', 'iBugNo']; + + ksParam_sRepository = 'VcsBugReference_sRepository'; + ksParam_iRevision = 'VcsBugReference_iRevision'; + ksParam_sBugTracker = 'VcsBugReference_sBugTracker'; + ksParam_lBugNo = 'VcsBugReference_lBugNo'; + + kasAllowNullAttributes = [ ]; + + def __init__(self): + ModelDataBase.__init__(self); + + # + # Initialize with defaults. + # See the database for explanations of each of these fields. + # + self.sRepository = None; + self.iRevision = None; + self.sBugTracker = None; + self.lBugNo = None; + + def initFromDbRow(self, aoRow): + """ + Re-initializes the object from a SELECT * FROM VcsBugReferences row. + Returns self. Raises exception if aoRow is None. + """ + if aoRow is None: + raise TMExceptionBase('VcsBugReference not found.'); + + self.sRepository = aoRow[0]; + self.iRevision = aoRow[1]; + self.sBugTracker = aoRow[2]; + self.lBugNo = aoRow[3]; + return self; + + def initFromValues(self, sRepository, iRevision, sBugTracker, lBugNo): + """ + Reinitializes form a set of values. + return self. + """ + self.sRepository = sRepository; + self.iRevision = iRevision; + self.sBugTracker = sBugTracker; + self.lBugNo = lBugNo; + return self; + + +class VcsBugReferenceLogic(ModelLogicBase): # pylint: disable=too-few-public-methods + """ + VCS revision <-> bug tracker references database logic. + """ + + # + # Standard methods. + # + + def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None): + """ + Fetches VCS revisions for listing. + + Returns an array (list) of VcsBugReferenceData items, empty list if none. + Raises exception on error. + """ + _ = tsNow; _ = aiSortColumns; + self._oDb.execute(''' +SELECT * +FROM VcsBugReferences +ORDER BY sRepository, iRevision, sBugTracker, lBugNo +LIMIT %s OFFSET %s +''', (cMaxRows, iStart,)); + + aoRows = []; + for _ in range(self._oDb.getRowCount()): + aoRows.append(VcsBugReferenceData().initFromDbRow(self._oDb.fetchOne())); + return aoRows; + + def exists(self, oData): + """ + Checks if the data is already present in the DB. + Returns True / False. + Raises exception on input and database errors. + """ + self._oDb.execute(''' +SELECT COUNT(*) +FROM VcsBugReferences +WHERE sRepository = %s + AND iRevision = %s + AND sBugTracker = %s + AND lBugNo = %s +''', ( oData.sRepository, oData.iRevision, oData.sBugTracker, oData.lBugNo)); + cRows = self._oDb.fetchOne()[0]; + if cRows < 0 or cRows > 1: + raise TMExceptionBase('VcsBugReferences has a primary key problem: %u duplicates' % (cRows,)); + return cRows != 0; + + + # + # Other methods. + # + + def addVcsBugReference(self, oData, fCommit = False): + """ + Adds (or updates) a tree revision record. + Raises exception on input and database errors. + """ + + # Check VcsBugReferenceData before do anything + dDataErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add); + if dDataErrors: + raise TMExceptionBase('Invalid data passed to addVcsBugReference(): %s' % (dDataErrors,)); + + # Does it already exist? + if not self.exists(oData): + # New row. + self._oDb.execute('INSERT INTO VcsBugReferences (sRepository, iRevision, sBugTracker, lBugNo)\n' + 'VALUES (%s, %s, %s, %s)\n' + , ( oData.sRepository, + oData.iRevision, + oData.sBugTracker, + oData.lBugNo, + )); + + self._oDb.maybeCommit(fCommit); + return oData; + + def getLastRevision(self, sRepository): + """ + Get the last known revision number for the given repository, returns 0 + if the repository is not known to us: + """ + self._oDb.execute(''' +SELECT iRevision +FROM VcsBugReferences +WHERE sRepository = %s +ORDER BY iRevision DESC +LIMIT 1 +''', ( sRepository, )); + if self._oDb.getRowCount() == 0: + return 0; + return self._oDb.fetchOne()[0]; + + +# +# Unit testing. +# + +# pylint: disable=missing-docstring +class VcsBugReferenceDataTestCase(ModelDataBaseTestCase): + def setUp(self): + self.aoSamples = [VcsBugReferenceData(),]; + +if __name__ == '__main__': + unittest.main(); + # not reached. +