diff --git a/CondCore/BeamSpotPlugins/plugins/BeamSpotObjects_toXML.cc b/CondCore/BeamSpotPlugins/plugins/BeamSpotObjects_toXML.cc deleted file mode 100644 index 1a77d974cbef4..0000000000000 --- a/CondCore/BeamSpotPlugins/plugins/BeamSpotObjects_toXML.cc +++ /dev/null @@ -1,21 +0,0 @@ - - -#include "CondCore/Utilities/interface/PayloadToXML.h" -#include "CondCore/Utilities/src/CondFormats.h" - -namespace { // Avoid cluttering the global namespace. - - // converter methods - std::string BeamSpotObjects2xml( std::string const &payloadData, std::string const &payloadType ) { - return cond::convertToXML (payloadData, payloadType); - } - -} // end namespace - - -BOOST_PYTHON_MODULE( pluginBeamSpotObjects_toXML ) -{ - using namespace boost::python; - def ("BeamSpotObjects2xml" , &BeamSpotObjects2xml); - -} diff --git a/CondCore/EcalPlugins/plugins/BuildFile.xml b/CondCore/EcalPlugins/plugins/BuildFile.xml index 85dc68adc1e5d..d1947a2ae3f93 100644 --- a/CondCore/EcalPlugins/plugins/BuildFile.xml +++ b/CondCore/EcalPlugins/plugins/BuildFile.xml @@ -8,14 +8,6 @@ - - - - - - - - diff --git a/CondCore/EcalPlugins/plugins/EcalCondObjectContainer_EcalTPGCrystalStatusCode_toXML.cc b/CondCore/EcalPlugins/plugins/EcalCondObjectContainer_EcalTPGCrystalStatusCode_toXML.cc deleted file mode 100644 index 6c7d00090fd81..0000000000000 --- a/CondCore/EcalPlugins/plugins/EcalCondObjectContainer_EcalTPGCrystalStatusCode_toXML.cc +++ /dev/null @@ -1,20 +0,0 @@ - -#include "CondCore/Utilities/interface/PayloadToXML.h" -#include "CondCore/Utilities/src/CondFormats.h" - -namespace { // Avoid cluttering the global namespace. - - // converter methods - std::string EcalCondObjectContainer_EcalTPGCrystalStatusCode2xml( std::string const &payloadData, std::string const &payloadType ) { - return cond::convertToXML > (payloadData, payloadType); - } - -} // end namespace - - -BOOST_PYTHON_MODULE( pluginEcalCondObjectContainer_EcalTPGCrystalStatusCode_toXML ) -{ - using namespace boost::python; - def ("EcalCondObjectContainer_EcalTPGCrystalStatusCode2xml" , &EcalCondObjectContainer_EcalTPGCrystalStatusCode2xml); - -} diff --git a/CondCore/SiStripPlugins/plugins/BuildFile.xml b/CondCore/SiStripPlugins/plugins/BuildFile.xml index 29a8ef6b45eee..687f9cb0db119 100644 --- a/CondCore/SiStripPlugins/plugins/BuildFile.xml +++ b/CondCore/SiStripPlugins/plugins/BuildFile.xml @@ -1,12 +1,5 @@ - - - - - - - diff --git a/CondCore/SiStripPlugins/plugins/SiStripObjects_toXML.cc b/CondCore/SiStripPlugins/plugins/SiStripObjects_toXML.cc deleted file mode 100644 index 8868c4a06c964..0000000000000 --- a/CondCore/SiStripPlugins/plugins/SiStripObjects_toXML.cc +++ /dev/null @@ -1,26 +0,0 @@ - -#include "CondCore/Utilities/interface/PayloadToXML.h" -#include "CondCore/Utilities/src/CondFormats.h" - -namespace { // Avoid cluttering the global namespace. - - // converter methods - std::string SiStripLatency2xml( std::string const &payloadData, std::string const &payloadType ) { - return cond::convertToXML (payloadData, payloadType); - } - - std::string SiStripConfObject2xml( const std::string &payloadData, const std::string &payloadType ) { - return cond::convertToXML (payloadData, payloadType); - } - -} // end namespace - - -BOOST_PYTHON_MODULE( pluginSiStripObjects_toXML ) -{ - using namespace boost::python; - def ("SiStripLatency2xml" , &SiStripLatency2xml); - def ("SiStripConfObject2xml", &SiStripConfObject2xml); - -} - diff --git a/CondCore/Utilities/interface/Payload2XMLModule.h b/CondCore/Utilities/interface/Payload2XMLModule.h new file mode 100644 index 0000000000000..4d6ce0202b7b2 --- /dev/null +++ b/CondCore/Utilities/interface/Payload2XMLModule.h @@ -0,0 +1,52 @@ +#ifndef CondCore_Utilities_Payload2XMLModule_h +#define CondCore_Utilities_Payload2XMLModule_h + +#include +#include + +#include +#include "boost/archive/xml_oarchive.hpp" + +#include "CondFormats/Serialization/interface/Archive.h" + +#define XML_CONVERTER_NAME( CLASS_NAME ) (std::string( #CLASS_NAME )+"2xml").c_str() + +#define PAYLOAD_2XML_MODULE( MODULE_NAME ) \ + BOOST_PYTHON_MODULE( MODULE_NAME ) + +#define PAYLOAD_2XML_CLASS( CLASS_NAME ) \ + boost::python::class_< Payload2xml >( XML_CONVERTER_NAME( CLASS_NAME ), boost::python::init<>()) \ + .def("write",&Payload2xml::write ) \ + ; + +namespace { // Avoid cluttering the global namespace. + + template class Payload2xml { + public: + Payload2xml(){ + } + // + std::string write( const std::string &payloadData ){ + // now to convert + std::unique_ptr< PayloadType > payload; + std::stringbuf sdataBuf; + sdataBuf.pubsetbuf( const_cast ( payloadData.c_str() ), payloadData.size() ); + + std::istream inBuffer( &sdataBuf ); + eos::portable_iarchive ia( inBuffer ); + payload.reset( new PayloadType ); + ia >> (*payload); + + // now we have the object in memory, convert it to xml in a string and return it + std::ostringstream outBuffer; + { + boost::archive::xml_oarchive xmlResult( outBuffer ); + xmlResult << boost::serialization::make_nvp( "cmsCondPayload", *payload ); + } + return outBuffer.str(); + } + }; + +} // end namespace + +#endif diff --git a/CondCore/Utilities/interface/PayloadToXML.h b/CondCore/Utilities/interface/PayloadToXML.h deleted file mode 100644 index 120b66c044b98..0000000000000 --- a/CondCore/Utilities/interface/PayloadToXML.h +++ /dev/null @@ -1,42 +0,0 @@ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "boost/archive/xml_oarchive.hpp" -#include "CondFormats/Serialization/interface/Serializable.h" -#include "CondFormats/Serialization/interface/Archive.h" - -namespace cond { - - template - std::string convertToXML( const std::string &payloadData, const std::string &payloadType ) { - - std::unique_ptr< T > payload; - std::stringbuf sdataBuf; - sdataBuf.pubsetbuf( const_cast ( payloadData.c_str() ), payloadData.size() ); - - std::istream inBuffer( &sdataBuf ); - eos::portable_iarchive ia( inBuffer ); - payload.reset( new T ); - ia >> (*payload); - - // now we have the object in memory, convert it to xml in a string and return it - - std::ostringstream outBuffer; - boost::archive::xml_oarchive xmlResult( outBuffer ); - xmlResult << boost::serialization::make_nvp( "cmsCondPayload", *payload ); - - return outBuffer.str(); - - } -} // end namespace cond - diff --git a/CondCore/Utilities/plugins/BuildFile.xml b/CondCore/Utilities/plugins/BuildFile.xml index 46e7bb2b68570..9c456fcb0ad14 100644 --- a/CondCore/Utilities/plugins/BuildFile.xml +++ b/CondCore/Utilities/plugins/BuildFile.xml @@ -13,3 +13,7 @@ + + + + diff --git a/CondCore/Utilities/plugins/Module_2XML.cc b/CondCore/Utilities/plugins/Module_2XML.cc new file mode 100644 index 0000000000000..fec980e35c57d --- /dev/null +++ b/CondCore/Utilities/plugins/Module_2XML.cc @@ -0,0 +1,11 @@ +#include "CondCore/Utilities/interface/Payload2XMLModule.h" +#include "CondCore/Utilities/src/CondFormats.h" + +PAYLOAD_2XML_MODULE( pluginUtilities_payload2xml ){ + PAYLOAD_2XML_CLASS( BeamSpotObjects ); + PAYLOAD_2XML_CLASS( EcalCondObjectContainer ); + PAYLOAD_2XML_CLASS( EcalLaserAPDPNRatios ); + PAYLOAD_2XML_CLASS( RunInfo ); + PAYLOAD_2XML_CLASS( SiStripLatency ); + PAYLOAD_2XML_CLASS( SiStripConfObject ); +} diff --git a/CondCore/Utilities/python/cond2xml.py b/CondCore/Utilities/python/cond2xml.py index 75148867d3b51..32ebb5b283b62 100644 --- a/CondCore/Utilities/python/cond2xml.py +++ b/CondCore/Utilities/python/cond2xml.py @@ -5,6 +5,8 @@ import time import glob import importlib +import logging +import subprocess # as we need to load the shared lib from here, make sure it's in our path: if os.path.join( os.environ['CMSSW_BASE'], 'src') not in sys.path: @@ -14,66 +16,21 @@ payload2xmlCodeTemplate = """ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "boost/archive/xml_oarchive.hpp" -#include "CondFormats/Serialization/interface/Serializable.h" -#include "CondFormats/Serialization/interface/Archive.h" - +#include "CondCore/Utilities/interface/Payload2XMLModule.h" #include "CondCore/Utilities/src/CondFormats.h" -namespace { // Avoid cluttering the global namespace. - - std::string %(plTypeSan)s2xml( const std::string &payloadData, const std::string &payloadType ) { - - // now to convert - std::unique_ptr< %(plType)s > payload; - - std::stringbuf sdataBuf; - sdataBuf.pubsetbuf( const_cast ( payloadData.c_str() ), payloadData.size() ); - - std::istream inBuffer( &sdataBuf ); - eos::portable_iarchive ia( inBuffer ); - payload.reset( new %(plType)s ); - ia >> (*payload); - - // now we have the object in memory, convert it to xml in a string and return it - - std::ostringstream outBuffer; - { - boost::archive::xml_oarchive xmlResult( outBuffer ); - xmlResult << boost::serialization::make_nvp( "cmsCondPayload", *payload ); - } - return outBuffer.str(); - } - -} // end namespace - - -BOOST_PYTHON_MODULE(%(mdName)s) -{ - using namespace boost::python; - def ("%(plTypeSan)s2xml", %(plTypeSan)s2xml); +PAYLOAD_2XML_MODULE( %s ){ + PAYLOAD_2XML_CLASS( %s ); } """ buildFileTemplate = """ - - - - + + + + @@ -83,11 +40,17 @@ def ("%(plTypeSan)s2xml", %(plTypeSan)s2xml); def sanitize(typeName): return typeName.replace(' ','').replace('<','_').replace('>','') +def localLibName( payloadType ): + # required to avoid ( unlikely ) clashes between lib names from templates and lib names from classes + prefix = '' + if '<' in payloadType and '>' in payloadType: + prefix = 't' + return "%s_%spayload2xml" %(sanitize(payloadType),prefix) + class CondXmlProcessor(object): def __init__(self, condDBIn): self.conddb = condDBIn - self._pl2xml_isPrepared = False if not os.path.exists( os.path.join( os.environ['CMSSW_BASE'], 'src') ): raise Exception("Looks like you are not running in a CMSSW developer area, $CMSSW_BASE/src/ does not exist") @@ -95,115 +58,120 @@ def __init__(self, condDBIn): self.fakePkgName = "fakeSubSys4pl/fakePkg4pl" self._pl2xml_tmpDir = os.path.join( os.environ['CMSSW_BASE'], 'src', self.fakePkgName ) - self.doCleanup = True + self.doCleanup = False def __del__(self): if self.doCleanup: shutil.rmtree( '/'.join( self._pl2xml_tmpDir.split('/')[:-1] ) ) - os.unlink( os.path.join( os.environ['CMSSW_BASE'], 'src', './pl2xmlComp.so') ) return def discover(self, payloadType): - # print "discover> checking for plugin of type %s" % payloadType - - # first search in developer area: - libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] ) - pluginList = glob.glob( libDir + '/plugin%s_toXML.so' % sanitize(payloadType) ) - - # if nothing found there, check release: - if not pluginList: - libDir = os.path.join( os.environ["CMSSW_RELEASE_BASE"], 'lib', os.environ["SCRAM_ARCH"] ) - pluginList = glob.glob( libDir + '/plugin%s_toXML.so' % sanitize(payloadType) ) - - # if pluginList: - # print "found plugin for %s (in %s) : %s " % (payloadType, libDir, pluginList) - # else: - # print "no plugin found for type %s" % payloadType - - xmlConverter = None - if len(pluginList) > 0: - dirPath, libName = os.path.split( pluginList[0] ) - sys.path.append(dirPath) - # print "going to import %s from %s" % (libName, dirPath) - xmlConverter = importlib.import_module( libName.replace('.so', '') ) - # print "found methods: ", dir(xmlConverter) - self.doCleanup = False - - return xmlConverter - - def prepPayload2xml(self, session, payload): - - startTime = time.time() - - Payload = session.get_dbtype(self.conddb.Payload) - # get payload from DB: - result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payload).one() - data, plType = result + libName = 'pluginUtilities_payload2xml.so' + # first search: developer area or main release + libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] ) + devLibDir = libDir + libPath = os.path.join( devLibDir, libName ) + devCheckout = ("CMSSW_RELEASE_BASE" in os.environ) + if not os.path.exists( libPath ) and devCheckout: + # main release ( for dev checkouts ) + libDir = os.path.join( os.environ["CMSSW_RELEASE_BASE"], 'lib', os.environ["SCRAM_ARCH"] ) + libPath = os.path.join( libDir, libName ) + if not os.path.exists( libPath ): + # it should never happen! + raise Exception('No built-in library found with XML converters.') + module = importlib.import_module( libName.replace('.so', '') ) + functors = dir(module) + funcName = payloadType+'2xml' + if funcName in functors: + logging.info('XML converter for payload class %s found in the built-in library.' %payloadType) + return getattr( module, funcName) + if not devCheckout: + # give-up if it is a read-only release... + raise Exception('No XML converter suitable for payload class %s has been found in the built-in library.') + libName = 'plugin%s.so' %localLibName( payloadType ) + libPath = os.path.join( devLibDir, libName ) + if os.path.exists( libPath ): + logging.info('Found local library with XML converter for class %s' %payloadType ) + module = importlib.import_module( libName.replace('.so', '') ) + return getattr( module, funcName) + logging.warning('No XML converter for payload class %s found in the built-in library.' %payloadType) + return None + + def prepPayload2xml(self, payloadType): - info = { "mdName" : "pl2xmlComp", - 'plType' : plType, - 'plTypeSan' : sanitize(plType), - } - - converter = self.discover(plType) + converter = self.discover(payloadType) if converter: return converter - code = payload2xmlCodeTemplate % info + #otherwise, go for the code generation in the local checkout area. + startTime = time.time() + + libName = localLibName( payloadType ) + pluginName = 'plugin%s' % libName + tmpLibName = "Tmp_payload2xml" + tmpPluginName = 'plugin%s' %tmpLibName + + libDir = os.path.join( os.environ["CMSSW_BASE"], 'lib', os.environ["SCRAM_ARCH"] ) + tmpLibFile = os.path.join( libDir,tmpPluginName+'.so' ) + code = payload2xmlCodeTemplate %(pluginName,payloadType) + tmpSrcFileName = 'Local_2XML.cpp' tmpDir = self._pl2xml_tmpDir if ( os.path.exists( tmpDir ) ) : msg = '\nERROR: %s already exists, please remove if you did not create that manually !!' % tmpDir - self.doCleanup = False raise Exception(msg) - os.makedirs( tmpDir+'/src' ) + logging.debug('Creating temporary package %s' %self._pl2xml_tmpDir) + os.makedirs( tmpDir+'/plugins' ) - buildFileName = "%s/BuildFile.xml" % (tmpDir,) + buildFileName = "%s/plugins/BuildFile.xml" % (tmpDir,) with open(buildFileName, 'w') as buildFile: - buildFile.write( buildFileTemplate ) + buildFile.write( buildFileTemplate %(tmpSrcFileName,tmpLibName) ) buildFile.close() - tmpFileName = "%s/src/%s" % (tmpDir, info['mdName'],) - with open(tmpFileName+'.cpp', 'w') as codeFile: + tmpSrcFilePath = "%s/plugins/%s" % (tmpDir, tmpSrcFileName,) + with open(tmpSrcFilePath, 'w') as codeFile: codeFile.write(code) codeFile.close() - libDir = os.path.join( os.environ["CMSSW_BASE"], 'tmp', os.environ["SCRAM_ARCH"], 'src', self.fakePkgName, 'src', self.fakePkgName.replace('/','')) - libName = libDir + '/lib%s.so' % self.fakePkgName.replace('/','') cmd = "source /afs/cern.ch/cms/cmsset_default.sh;" - cmd += "(cd %s ; scram b 2>&1 >build.log && cp %s $CMSSW_BASE/src/pl2xmlComp.so )" % (tmpDir, libName) - ret = os.system(cmd) - if ret != 0 : self.doCleanup = False + cmd += "(cd %s ; scram b 2>&1 >build.log)" %tmpDir + pipe = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + out, err = pipe.communicate() + ret = pipe.returncode buildTime = time.time()-startTime - print >> sys.stderr, "buillding done in ", buildTime, 'sec., return code from build: ', ret + logging.info("Building done in %s sec., return code from build: %s" %(buildTime,ret) ) if (ret != 0): return None - return importlib.import_module( 'pl2xmlComp' ) - - def payload2xml(self, session, payload): - - if not self._pl2xml_isPrepared: - xmlConverter = self.prepPayload2xml(session, payload) - if not xmlConverter: - msg = "Error preparing code for "+payload - raise Exception(msg) - self._pl2xml_isPrepared = True + libFile = os.path.join(libDir,pluginName + '.so') + shutil.copyfile(tmpLibFile,libFile) + module = importlib.import_module( pluginName ) + funcName = payloadType+'2xml' + functor = getattr( module, funcName ) + self.doCleanup = True + return functor + + def payload2xml(self, session, payloadHash, destFile): Payload = session.get_dbtype(self.conddb.Payload) # get payload from DB: - result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payload).one() + result = session.query(Payload.data, Payload.object_type).filter(Payload.hash == payloadHash).one() data, plType = result + logging.info('Found payload of type %s' %plType) convFuncName = sanitize(plType)+'2xml' - sys.path.append('.') - func = getattr(xmlConverter, convFuncName) - resultXML = func( str(data), str(plType) ) - - print resultXML - + xmlConverter = self.prepPayload2xml(plType) + + obj = xmlConverter() + resultXML = obj.write( str(data) ) + if destFile is None: + print resultXML + else: + with open(destFile, 'w') as outFile: + outFile.write(resultXML) + outFile.close() diff --git a/CondCore/Utilities/scripts/conddb b/CondCore/Utilities/scripts/conddb index 7150c50918fdd..2c455b3d78c3a 100755 --- a/CondCore/Utilities/scripts/conddb +++ b/CondCore/Utilities/scripts/conddb @@ -1986,7 +1986,7 @@ def dump(args): if args.type == 'payload': if args.format == 'xml': - xmlProcessor.payload2xml(session, name) + xmlProcessor.payload2xml(session, name, args.destfile) else: _dump_payload(session, name, args.loadonly) @@ -2115,6 +2115,7 @@ def main(): parser_dump.add_argument('--loadonly', action='store_true', help='Load only: Do not dump, only load the (deserialize) payload in memory -- useful for testing the load of an entire global tag with the current CMSSW release.') parser_dump.add_argument('--type', default=None, choices=['payload', 'tag', 'gt'], help='Type of the object. Use it if there is ambiguity (should be really rare).') parser_dump.add_argument('--format', default="xml", choices=['xml', 'raw'], help='Output format. Choice between XML and raw hexdump.') + parser_dump.add_argument('--destfile','-d',default=None,help="Destination file for the dump.") parser_dump.set_defaults(func=dump) parser_showFcsr = parser_subparsers.add_parser('showFCSR', description='Dumps the FCSR values for hlt and pcl')