diff --git a/CondCore/CondDB/interface/Session.h b/CondCore/CondDB/interface/Session.h index 6f095f0708215..7a2d628296c62 100644 --- a/CondCore/CondDB/interface/Session.h +++ b/CondCore/CondDB/interface/Session.h @@ -100,7 +100,10 @@ namespace cond { // bool existsIov( const std::string& tag ); - + + // + bool getTagInfo( const std::string& tag, cond::Tag_t& info ); + // retrieves an IOV range. Peforms a query at every call. bool getIovRange( const std::string& tag, cond::Time_t begin, cond::Time_t end, diff --git a/CondCore/CondDB/interface/Types.h b/CondCore/CondDB/interface/Types.h index f8ab4edf42da4..1ba0afd9428b5 100644 --- a/CondCore/CondDB/interface/Types.h +++ b/CondCore/CondDB/interface/Types.h @@ -63,6 +63,7 @@ namespace cond { std::string tag; std::string payloadType; TimeType timeType; + SynchronizationType synchronizationType; Time_t endOfValidity; Time_t lastValidatedTime; }; diff --git a/CondCore/CondDB/src/Session.cc b/CondCore/CondDB/src/Session.cc index e33a412c11d52..109ce12d287e5 100644 --- a/CondCore/CondDB/src/Session.cc +++ b/CondCore/CondDB/src/Session.cc @@ -98,6 +98,14 @@ namespace cond { m_session->openIovDb(); return m_session->iovSchema().tagTable().select( tag ); } + + bool Session::getTagInfo( const std::string& tag, + cond::Tag_t& info ){ + m_session->openIovDb(); + std::string description; + return m_session->iovSchema().tagTable().select( tag, info.timeType, info.payloadType, info.synchronizationType, + info.endOfValidity, description, info.lastValidatedTime ); + } IOVProxy Session::iovProxy(){ m_session->openIovDb(); diff --git a/CondCore/Utilities/interface/PayloadInspector.h b/CondCore/Utilities/interface/PayloadInspector.h index e910c4127262b..da48898c84b15 100644 --- a/CondCore/Utilities/interface/PayloadInspector.h +++ b/CondCore/Utilities/interface/PayloadInspector.h @@ -31,6 +31,7 @@ namespace cond { std::string get( const std::string& key ) const; std::map m; bool singleIov = false; + bool twoTags = false; }; static const char* const JSON_FORMAT_VERSION = "1.0"; @@ -147,12 +148,18 @@ namespace cond { // required in the browser bool isSingleIov() const; + // required in the browser + bool isTwoTags() const; + // returns the json file with the plot data std::string data() const; // triggers the processing producing the plot bool process( const std::string& connectionString, const std::string& tag, const std::string& timeType, cond::Time_t begin, cond::Time_t end ); + //bool process( const std::string& connectionString, const std::string& tag, cond::Time_t begin, cond::Time_t end ); + bool processTwoTags( const std::string& connectionString, const std::string& tag0, const std::string& tag1, cond::Time_t time0, cond::Time_t time1 ); + // not exposed in python: // called internally in process() virtual void init(); @@ -164,13 +171,17 @@ namespace cond { // to be set in order to limit the iov selection ( and processing ) to 1 iov void setSingleIov( bool flag ); + void setTwoTags( bool flag ); + // access to the fetch function of the configured reader, to be used in the processData implementations template std::shared_ptr fetchPayload( const cond::Hash& payloadHash ){ return m_dbSession.fetchPayload( payloadHash ); } + cond::Tag_t getTagInfo( const std::string& tag ); + // access to the timeType of the tag in process - cond::TimeType tagTimeType() const; + //cond::TimeType tagTimeType() const; // access to the underlying db session cond::persistency::Session dbSession(); @@ -178,11 +189,13 @@ namespace cond { protected: PlotAnnotations m_plotAnnotations; + std::string m_tag0 = ""; + std::string m_tag1 = ""; private: cond::persistency::Session m_dbSession; - cond::TimeType m_tagTimeType; + //cond::TimeType m_tagTimeType; std::string m_data = ""; }; @@ -279,9 +292,11 @@ namespace cond { } ~RunHistoryPlot() override = default; bool fill( const std::vector >& iovs ) override { + // for the lumi iovs we need to count the number of lumisections in every runs std::map runs; - if( Base::tagTimeType()==cond::lumiid ){ + cond::Tag_t tagInfo = Base::getTagInfo( Base::m_tag0 ); + if( tagInfo.timeType==cond::lumiid ){ for( auto iov : iovs ) { unsigned int run = std::get<0>(iov) >> 32; auto it = runs.find( run ); @@ -298,7 +313,7 @@ namespace cond { for( auto iov : iovs ) { unsigned long long since = std::get<0>(iov); // for the lumi iovs we squeeze the lumi section available in the constant run 'slot' of witdth=1 - if( Base::tagTimeType()==cond::lumiid ){ + if( tagInfo.timeType==cond::lumiid ){ unsigned int run = since >> 32; unsigned int lumi = since & 0xFFFFFFFF; if( run != currentRun ) { @@ -319,7 +334,7 @@ namespace cond { } else { ind++; // for the timestamp based iovs, it does not really make much sense to use this plot... - if( Base::tagTimeType()==cond::timestamp ){ + if( tagInfo.timeType==cond::timestamp ){ boost::posix_time::ptime t = cond::time::to_boost( since ); label = boost::posix_time::to_simple_string( t ); } else { @@ -349,10 +364,11 @@ namespace cond { ~TimeHistoryPlot() override = default; bool fill( const std::vector >& iovs ) override { cond::persistency::RunInfoProxy runInfo; - if( Base::tagTimeType()==cond::lumiid || Base::tagTimeType()==cond::runnumber){ + cond::Tag_t tagInfo = Base::getTagInfo( Base::m_tag0 ); + if( tagInfo.timeType==cond::lumiid || tagInfo.timeType==cond::runnumber){ cond::Time_t min = std::get<0>(iovs.front()); cond::Time_t max = std::get<0>( iovs.back() ); - if( Base::tagTimeType()==cond::lumiid ){ + if( tagInfo.timeType==cond::lumiid ){ min = min >> 32; max = max >> 32; } @@ -362,9 +378,9 @@ namespace cond { cond::Time_t since = std::get<0>(iov); boost::posix_time::ptime time; std::string label(""); - if( Base::tagTimeType()==cond::lumiid || Base::tagTimeType()==cond::runnumber){ + if( tagInfo.timeType==cond::lumiid || tagInfo.timeType==cond::runnumber){ unsigned int nlumi = since & 0xFFFFFFFF; - if( Base::tagTimeType()==cond::lumiid ) since = since >> 32; + if( tagInfo.timeType==cond::lumiid ) since = since >> 32; label = std::to_string(since ); auto it = runInfo.find( since ); if ( it == runInfo.end() ){ @@ -373,11 +389,11 @@ namespace cond { } time = (*it).start; // add the lumi sections... - if( Base::tagTimeType()==cond::lumiid ){ + if( tagInfo.timeType==cond::lumiid ){ time += boost::posix_time::seconds( cond::time::SECONDS_PER_LUMI*nlumi ); label += (" : "+std::to_string(nlumi ) ); } - } else if ( Base::tagTimeType()==cond::timestamp ){ + } else if ( tagInfo.timeType==cond::timestamp ){ time = cond::time::to_boost( since ); label = boost::posix_time::to_simple_string( time ); } diff --git a/CondCore/Utilities/interface/PayloadInspectorModule.h b/CondCore/Utilities/interface/PayloadInspectorModule.h index 106ec5062ff48..d5b38371ecf8e 100644 --- a/CondCore/Utilities/interface/PayloadInspectorModule.h +++ b/CondCore/Utilities/interface/PayloadInspectorModule.h @@ -11,8 +11,10 @@ #define PAYLOAD_INSPECTOR_CLASS( CLASS_NAME ) \ boost::python::class_< CLASS_NAME, boost::python::bases >( STRINGIZE(PPCAT(plot_,CLASS_NAME)), boost::python::init<>()) \ .def("process",&cond::payloadInspector::PlotBase::process ) \ + .def("processTwoTags",&cond::payloadInspector::PlotBase::processTwoTags ) \ .def("payloadType",&cond::payloadInspector::PlotBase::payloadType ) \ .def("title",&cond::payloadInspector::PlotBase::title ) \ .def("isSingleIov",&cond::payloadInspector::PlotBase::isSingleIov ) \ + .def("isTwoTags",&cond::payloadInspector::PlotBase::isTwoTags ) \ .def("data",&cond::payloadInspector::PlotBase::data ) \ ; diff --git a/CondCore/Utilities/plugins/BasicP_PayloadInspector.cc b/CondCore/Utilities/plugins/BasicP_PayloadInspector.cc index 187c71c34a631..20b1ba3409403 100644 --- a/CondCore/Utilities/plugins/BasicP_PayloadInspector.cc +++ b/CondCore/Utilities/plugins/BasicP_PayloadInspector.cc @@ -177,6 +177,90 @@ namespace { } }; + class BasicPayload_data7 : public cond::payloadInspector::PlotImage { + public: + BasicPayload_data7() : cond::payloadInspector::PlotImage( "Example delivery picture" ){ + setTwoTags( true ); + } + + bool fill( const std::vector >& iovs ) override{ + auto iov0 = iovs.front(); + auto iov1 = iovs.back(); + std::shared_ptr payload0 = fetchPayload( std::get<1>(iov0) ); + std::shared_ptr payload1 = fetchPayload( std::get<1>(iov1) ); + + double xmax(100.),ymax(100.); + + TH2D h2D("h2D","Example",100,0.,xmax,100,0.,ymax); + + if( payload0.get() && payload1.get() ){ + if(payload0->m_vec.size()==10000 && payload1->m_vec.size()==10000 ){ + for( size_t i=0;i<100;i++ ) + for( size_t j=0;j<100;j++ ) { + auto diff = abs(payload0->m_vec[i*100+j] - payload1->m_vec[i*100+j]); + h2D.Fill(i,j,diff); + } + h2D.SetStats(false); + } + } + + TCanvas c("c","",20,20,900,500); + c.cd(); + c.SetLogz(); + h2D.SetNdivisions(18, "X"); + h2D.GetXaxis()->SetTickLength(0.00); + h2D.GetYaxis()->SetTickLength(0.00); + h2D.GetXaxis()->SetTitle("iphi"); + h2D.GetYaxis()->SetTitle("ieta"); + h2D.Draw("col"); + + //======= drawing lines ======== + ///// this is quite specific to the line style they need + + TLine l; + l.SetLineStyle(2); + l.DrawLine(0., ymax/2., xmax, ymax/2.); + for(int m = 0; m < int(xmax); m+=10) { + l.DrawLine(m, 0., m, 100.); + } + + c.RedrawAxis(); + + //========== writing text in the canvas============== + //// This is again quite specific part. I just tried to emulate what is there in DQM for EB. + + TLatex Tl; + TLatex Tll; + Tl.SetTextAlign(23); + Tl.SetTextSize(0.04); + + Tll.SetTextAlign(23); + Tll.SetTextSize(0.04); + + int j = 0; + for(int i = 1; i <=10; i++){ + std::string s = "+" + std::to_string(i); + char const *pchar = s.c_str(); + j+=10; + Tl.DrawLatex(j-5,int(ymax)/1.33,pchar); + } + + int z = 0; + for(int g = -10; g <0; g++){ + std::string ss = std::to_string(g); + char const *pchar1 = ss.c_str(); + z+= 10; + Tll.DrawLatex(z-5,int(ymax)/4,pchar1); + } + //========================= + + std::string fileName(m_imageFileName); + c.SaveAs(fileName.c_str()); + + return true; + } + }; + } PAYLOAD_INSPECTOR_MODULE( BasicPayload ){ @@ -187,4 +271,5 @@ PAYLOAD_INSPECTOR_MODULE( BasicPayload ){ PAYLOAD_INSPECTOR_CLASS( BasicPayload_data4 ); PAYLOAD_INSPECTOR_CLASS( BasicPayload_data5 ); PAYLOAD_INSPECTOR_CLASS( BasicPayload_data6 ); + PAYLOAD_INSPECTOR_CLASS( BasicPayload_data7 ); } diff --git a/CondCore/Utilities/plugins/Module_PayloadInspector.cc b/CondCore/Utilities/plugins/Module_PayloadInspector.cc index 3e96ed0262d2f..6accbfaac2ef5 100644 --- a/CondCore/Utilities/plugins/Module_PayloadInspector.cc +++ b/CondCore/Utilities/plugins/Module_PayloadInspector.cc @@ -6,9 +6,11 @@ BOOST_PYTHON_MODULE( pluginModule_PayloadInspector ) { .def_readonly("label", &cond::payloadInspector::ModuleVersion::label ); boost::python::class_< cond::payloadInspector::PlotBase > ( "PlotBase" ) .def("process",&cond::payloadInspector::PlotBase::process ) + .def("processTwoTags",&cond::payloadInspector::PlotBase::processTwoTags ) .def("payloadType",&cond::payloadInspector::PlotBase::payloadType ) .def("type",&cond::payloadInspector::PlotBase::type ) .def("title",&cond::payloadInspector::PlotBase::title ) .def("isSingleIov",&cond::payloadInspector::PlotBase::isSingleIov ) + .def("isTwoTags",&cond::payloadInspector::PlotBase::isTwoTags ) .def("data",&cond::payloadInspector::PlotBase::data ); } diff --git a/CondCore/Utilities/scripts/getPayloadData.py b/CondCore/Utilities/scripts/getPayloadData.py index c0d7f48f95669..ee7c7792a6a4d 100755 --- a/CondCore/Utilities/scripts/getPayloadData.py +++ b/CondCore/Utilities/scripts/getPayloadData.py @@ -88,8 +88,9 @@ def deserialize_iovs(db, plugin_name, plot_name, tag, time_type, iovs): db_name = 'frontier://FrontierPrep/CMS_CONDITIONS' else: db_name = db + output('full DB name: ', db_name) - + success = plot.process(db_name, tag, time_type, int(iovs['start_iov']), int(iovs['end_iov'])) output('plot processed data successfully: ', success) @@ -97,6 +98,46 @@ def deserialize_iovs(db, plugin_name, plot_name, tag, time_type, iovs): return False + result = plot.data() + output('deserialized data: ', result) + return result +@supress_output +def deserialize_twoiovs(db, plugin_name, plot_name, tag,tagtwo,iovs,iovstwo): + ''' Deserializes given iovs data and returns plot coordinates ''' + #print "Starting to deserialize iovs:" + #print 'First Iovs',iovs + #print 'Two Iovs', iovstwo + output('Starting to deserialize iovs: ', '') + output('db: ', db) + output('plugin name: ', plugin_name) + output('plot name: ', plot_name) + output('tag name: ', tag) + output('tagtwo name: ', tagtwo) + #output('tag time type: ', time_type) + output('iovs: ', iovs) + output('iovstwo: ', iovstwo) + + plugin_base = import_module('pluginModule_PayloadInspector') + output('PI plugin base: ', plugin_base) + + plugin_obj = import_module(plugin_name) + output('PI plugin object: ', plugin_obj) + + # get plot method and execute it with given iovs + plot = getattr(plugin_obj, plot_name)() + output('plot object: ', plot) + + db_name = 'oracle://cms_orcon_adg/CMS_CONDITIONS' if db == 'Prod' else 'oracle://cms_orcoff_prep/CMS_CONDITIONS' + output('full DB name: ', db_name) + + + success = plot.processTwoTags(db_name, tag,tagtwo,int(iovs['start_iov']), int(iovstwo['end_iov'])) + #print "All good",success + output('plot processed data successfully: ', success) + if not success: + return False + + result = plot.data() output('deserialized data: ', result) return result @@ -169,15 +210,17 @@ def discover(): output(' - plot type: ', plot_type) single_iov = plot_method.isSingleIov() output(' - is single iov: ', single_iov) - result.setdefault(payload_type, []).append({'plot': plot, 'plugin_name': plugin_name, 'title': plot_title, 'plot_type': plot_type, 'single_iov': single_iov}) + two_tags = plot_method.isTwoTags() + output(' - is Two Tags: ', two_tags) + result.setdefault(payload_type, []).append({'plot': plot, 'plugin_name': plugin_name, 'title': plot_title, 'plot_type': plot_type, 'single_iov': single_iov, 'two_tags': two_tags}) output('currently discovered info: ', result) output('*** final output:', '') return json.dumps(result) def output(description, param): if args.verbose: - print('') - print(description, param) + print '' + print description, param if __name__ == '__main__': @@ -213,9 +256,11 @@ def output(description, param): parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter) parser.add_argument("-d", "--discover", help="discovers object types and plots \nfor a given cmssw release", action="store_true") parser.add_argument("-i", "--iovs", help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64") + parser.add_argument("-i2", "--iovstwo", help="deserializes given iovs data encoded in base64 and returns plot coordinates also encoded in base64") parser.add_argument("-o", "--plugin", help="Payload Inspector plugin name needed for iovs deserialization") parser.add_argument("-p", "--plot", help="plot name needed for iovs deserialization") parser.add_argument("-t", "--tag", help="tag name needed for iovs deserialization") + parser.add_argument("-t2", "--tagtwo", help="tag name needed for iovs deserialization") parser.add_argument("-tt", "--time_type", help="tag time type name needed for iovs deserialization") parser.add_argument("-b", "--db", help="db (Prod or Prep) needed for iovs deserialization") parser.add_argument("-test", "--test", help="add this flag if you want to test the deserialization function and want to see a readable output", action="store_true") @@ -235,9 +280,44 @@ def output(description, param): os.write( 1, discover() ) # Return a plot if iovs are provided - if args.iovs: + #print '* getiovs: ',args.iovs + #print '* getiovstwo: ',args.iovstwo + if args.iovstwo: # Run plugin with arguments + #print 'We are here' + a=json.loads(args.iovs) + #print 'A',a + b=json.loads(args.iovstwo) + #print 'B',b + result = deserialize_twoiovs(args.db, args.plugin, args.plot, args.tag,args.tagtwo,a,b) + # If test -> output the result as formatted json + if args.test: + os.write( 1, json.dumps( json.loads( result ), indent=4 )) + #print 'Result:',result + if args.image_plot: + try: + filename = json.loads( result )['file'] + #print 'File name',filename + except ValueError, e: + os.write( 2, 'Value error when getting image name: %s\n' % str( e )) + except KeyError, e: + os.write( 2, 'Key error when getting image name: %s\n' % str( e )) + + if not filename or not os.path.isfile( filename ): + os.write( 2, 'Error: Generated image file (%s) not found\n' % filename ) + + try: + with open( filename, 'r' ) as f: + shutil.copyfileobj( f, sys.stdout ) + except IOError, e: + os.write( 2, 'IO error when streaming image: %s' % str( e )) + finally: + os.remove( filename ) + + + # Else -> output result json string with base 64 encoding + elif args.iovs: result = deserialize_iovs(args.db, args.plugin, args.plot, args.tag, args.time_type, json.loads(args.iovs)) # If test -> output the result as formatted json @@ -251,9 +331,10 @@ def output(description, param): try: filename = json.loads( result )['file'] - except ValueError as e: + #print 'File name',filename + except ValueError, e: os.write( 2, 'Value error when getting image name: %s\n' % str( e )) - except KeyError as e: + except KeyError, e: os.write( 2, 'Key error when getting image name: %s\n' % str( e )) if not filename or not os.path.isfile( filename ): @@ -262,7 +343,7 @@ def output(description, param): try: with open( filename, 'r' ) as f: shutil.copyfileobj( f, sys.stdout ) - except IOError as e: + except IOError, e: os.write( 2, 'IO error when streaming image: %s' % str( e )) finally: os.remove( filename ) diff --git a/CondCore/Utilities/src/PayloadInspector.cc b/CondCore/Utilities/src/PayloadInspector.cc index e5f843d35f2e5..ac3d9ad9fe0f2 100644 --- a/CondCore/Utilities/src/PayloadInspector.cc +++ b/CondCore/Utilities/src/PayloadInspector.cc @@ -41,11 +41,15 @@ namespace cond { return m_plotAnnotations.singleIov; } + bool PlotBase::isTwoTags() const { + return m_plotAnnotations.twoTags; + } + std::string PlotBase::data() const { return m_data; } - bool PlotBase::process( const std::string& connectionString, const std::string& tag, const std::string& timeType, cond::Time_t begin, cond::Time_t end ){ + bool PlotBase::process( const std::string& connectionString, const std::string& tag, const std::string&, cond::Time_t begin, cond::Time_t end ){ init(); std::vector psets; @@ -55,7 +59,8 @@ namespace cond { static const edm::ServiceToken services(edm::ServiceRegistry::createSet(psets)); static const edm::ServiceRegistry::Operate operate(services); - m_tagTimeType = cond::time::timeTypeFromName(timeType); + m_tag0 = tag; + //m_tagTimeType = cond::time::timeTypeFromName(timeType); cond::persistency::ConnectionPool connection; m_dbSession = connection.createSession( connectionString ); m_dbSession.transaction().start(); @@ -67,6 +72,55 @@ namespace cond { return true; } + /** + bool PlotBase::process( const std::string& connectionString, const std::string& tag, cond::Time_t begin, cond::Time_t end ){ + init(); + + std::vector psets; + edm::ParameterSet pSet; + pSet.addParameter("@service_type",std::string("SiteLocalConfigService")); + psets.push_back(pSet); + static const edm::ServiceToken services(edm::ServiceRegistry::createSet(psets)); + static const edm::ServiceRegistry::Operate operate(services); + + m_tag0 = tag; + cond::persistency::ConnectionPool connection; + m_dbSession = connection.createSession( connectionString ); + m_dbSession.transaction().start(); + std::vector > iovs; + m_dbSession.getIovRange( tag, begin, end, iovs ); + m_data = processData( iovs ); + m_dbSession.transaction().commit(); + // fixme... + return true; + } + **/ + + bool PlotBase::processTwoTags( const std::string& connectionString, const std::string& tag0, const std::string& tag1, cond::Time_t time0, cond::Time_t time1 ){ + init(); + + std::vector psets; + edm::ParameterSet pSet; + pSet.addParameter("@service_type",std::string("SiteLocalConfigService")); + psets.push_back(pSet); + static const edm::ServiceToken services(edm::ServiceRegistry::createSet(psets)); + static const edm::ServiceRegistry::Operate operate(services); + + m_tag0 = tag0; + m_tag1 = tag1; + cond::persistency::ConnectionPool connection; + m_dbSession = connection.createSession( connectionString ); + m_dbSession.transaction().start(); + std::vector > iovs; + m_dbSession.getIovRange( tag0, time0, time0, iovs ); + m_dbSession.getIovRange( tag1, time1, time1, iovs ); + m_data = processData( iovs ); + m_dbSession.transaction().commit(); + // fixme... + return true; + } + + void PlotBase::init(){ } @@ -78,8 +132,19 @@ namespace cond { m_plotAnnotations.singleIov = flag; } - cond::TimeType PlotBase::tagTimeType() const { - return m_tagTimeType; + void PlotBase::setTwoTags( bool flag ){ + m_plotAnnotations.twoTags = flag; + if(flag) m_plotAnnotations.singleIov = flag; + } + + //cond::TimeType PlotBase::tagTimeType() const { + // return m_tagTimeType; + //} + + cond::Tag_t PlotBase::getTagInfo( const std::string& tag ){ + cond::Tag_t info; + m_dbSession.getTagInfo( tag, info ); + return info; } cond::persistency::Session PlotBase::dbSession(){