diff --git a/Utilities/StaticAnalyzers/scripts/callgraph.py b/Utilities/StaticAnalyzers/scripts/callgraph.py index 1b5c248e633a9..cd8c4c80f27ab 100755 --- a/Utilities/StaticAnalyzers/scripts/callgraph.py +++ b/Utilities/StaticAnalyzers/scripts/callgraph.py @@ -2,82 +2,117 @@ from __future__ import print_function import networkx as nx import re -import yaml +import json +from yaml import load, dump, FullLoader +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper -topfunc = re.compile(r"::(accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(") +topfunc = r"::(accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(" +topfuncre = re.compile(topfunc) + +baseclass = r"\b(edm::)?(one::|stream::|global::)?((DQM)?(Global|One)?ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator|RunWatcher|RunCacheHolder))\b" +baseclassre = re.compile(baseclass) +assert(baseclassre.match('edm::global::EDFilter::filter() virtual')) +assert(topfuncre.search('edm::global::EDFilterBase::filter(&) const virtual')) -baseclass = re.compile( - r"edm::(one::|stream::|global::)?(ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator))") farg = re.compile(r"\(.*?\)") tmpl = re.compile(r'<.*?>') toplevelfuncs = set() -epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<.*>\(.*\)") + +epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<(class|struct) edm::ES(Transient)?Handle<(class|struct) .*>>\((const char \*, |const std::string &, )(class|struct) edm::ES(Transient)?Handle<(class|struct).*> &\) const") +f= 'edm::eventsetup::EventSetupRecord::get>(const std::string &, class edm::ESHandle &) const' +assert(epfuncre.search(f)) + skipfunc = re.compile(r"TGraph::IsA\(.*\)") epfuncs = set() -G = nx.DiGraph() +function='(anonymous namespace in /src/RecoTracker/TkHitPairs/plugins/HitPairEDProducer.cc)::Impl<(anonymous namespace)::ImplSeedingHitSets, (anonymous namespace)::DoNothing, (anonymous namespace)::RegionsLayersSeparate>::produce(const _Bool, edm::Event &, const edm::EventSetup &);' +function2='HitPairEDProducer::produce(edm::Event &, const edm::EventSetup &);' +value='HitPairEDProducer' +valuere=r'\b%s\b' % value +vre=re.compile(valuere) +m=vre.search(function) +assert(m) +m2=vre.search(function2) +assert(m2) + +modules = list() +with open('modules_in_ib.txt', 'r') as m: + for line in m: + modules.append(line.strip()) +mods='|'.join(modules) +mods_regex=r'\b(' + mods+ r'QQQQQQQQ)\b' +comp_mods_regex=re.compile(mods_regex) +n=comp_mods_regex.search(function) +assert(n) +n2=comp_mods_regex.search(function2) +assert(n2) + +module2package = dict() +with open('module_to_package.yaml') as m: + module2package=load(m, Loader=FullLoader) -# g = open('module_to_package.txt') -# module2package=dict() -# for line in g: -# fields = line.strip().split(';') -# if len(fields) <2: -# continue -# module2package.setdefault(fields[1], []).append(fields[0]) -# -# i = open('module_to_package.yaml', 'w') -# yaml.dump(module2package, i) -# i.close() +with open('packages.json', 'r') as j: + p = json.load(j) -h = open('module_to_package.yaml', 'r') -module2package = yaml.load(h, Loader=yaml.FullLoader) +for k,v in p.items(): + module=k.split('|')[0] + dirs=v.split('|') + package=dirs[0]+'/'+dirs[1] + module2package.setdefault(package, []).append(module) +for k in module2package.keys(): + module2package[k]=sorted(set(module2package[k])) + +G = nx.DiGraph() with open('function-calls-db.txt') as f: for line in f: fields = line.split("'") if len(fields) < 3: continue if fields[2] == ' calls function ': - if not skipfunc.search(line): - G.add_edge(fields[1], fields[3], kind=fields[2]) - if epfuncre.search(fields[3]): - epfuncs.add(fields[3]) + G.add_edge(fields[1], fields[3], kind=fields[2]) + if epfuncre.search(fields[3]): + epfuncs.add(fields[3]) if fields[2] == ' overrides function ': - if baseclass.search(fields[3]): - if topfunc.search(fields[3]): - toplevelfuncs.add(fields[1]) + if baseclassre.match(fields[3]): G.add_edge(fields[1], fields[3], kind=' overrides function ') + if topfuncre.search(fields[1]) and comp_mods_regex.search(fields[1]): + toplevelfuncs.add(fields[1]) else: - if not skipfunc.search(line): - G.add_edge(fields[3], fields[1], - kind=' calls override function ') - if epfuncre.search(fields[1]): - epfuncs.add(fields[1]) + G.add_edge(fields[3], fields[1], kind=' overriden function calls function ') + if epfuncre.search(fields[3]): + epfuncs.add(fields[3]) callstacks = set() for tfunc in toplevelfuncs: for epfunc in epfuncs: if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G, tfunc, epfunc): - path = nx.shortest_path(G, tfunc, epfunc) - cs = str("") - previous = str("") - for p in path: - if epfuncre.search(p): - break - stripped = re.sub(farg, "()", p) - if previous != stripped: - cs += ' ' + stripped + ";" - previous = stripped - callstacks.add(cs) - break - + for path in nx.all_shortest_paths(G, tfunc, epfunc): + cs = str("") + previous = str("") + for p in path: + if epfuncre.match(p): + break + #stripped=re.sub(farg, "()", p) + stripped=p + if previous != stripped: + cs += stripped + '; ' + previous = stripped + if cs not in callstacks: + callstacks.add(cs) report = dict() -for key in sorted(module2package.keys()): - for value in sorted(module2package[key]): - vre = re.compile(' %s::.*();' % value) - for cs in sorted(callstacks): - if vre.search(cs): - report.setdefault(key, {}).setdefault(value, []).append(cs) +for key in sorted(set(module2package.keys())): + for value in sorted(set(module2package[key])): + if comp_mods_regex.match(value): + regex_str = r'\b%s\b'%value + vre=re.compile(regex_str) + for callstack in sorted(callstacks): + if vre.search(callstack): + report.setdefault(str(key), {}).setdefault(str(value), []).append(str(callstack)) + r = open('eventsetuprecord-get.yaml', 'w') -yaml.dump(report, r) +dump(report, r, width=float("inf")) diff --git a/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh b/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh index 2f00100625e32..b47d4e75e8f9b 100755 --- a/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh +++ b/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh @@ -49,6 +49,8 @@ if [ ! -f ./callgraph.py ] then cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/callgraph.py . cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/module_to_package.yaml . + cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/modules_in_ib.yaml . + curl -OL https://raw.githubusercontent.com/fwyzard/circles/master/web/groups/packages.json fi touch eventsetuprecord-get-all.txt eventsetuprecord-get.txt ./callgraph.py 2>&1 | tee eventsetuprecord-get.txt diff --git a/Utilities/StaticAnalyzers/scripts/module_to_package.yaml b/Utilities/StaticAnalyzers/scripts/module_to_package.yaml index 4fd1a6201e84d..d2de80e6a6fcf 100644 --- a/Utilities/StaticAnalyzers/scripts/module_to_package.yaml +++ b/Utilities/StaticAnalyzers/scripts/module_to_package.yaml @@ -156,7 +156,6 @@ RecoEgamma/EgammaPhotonProducers: - ConversionTrackCandidateProducer - GEDPhotonProducer - PhotonProducer -- TrackProducer - TrackProducerWithSCAssociation RecoEgamma/Examples: - SiStripElectronAnalyzer @@ -276,10 +275,10 @@ RecoParticleFlow/PFTracking: - PFNuclearProducer - PFV0Producer RecoPixelVertexing/PixelLowPtUtilities: -- PixelTrackProducer - PixelVertexProducerClusters - SiPixelClusterShapeCacheProducer RecoPixelVertexing/PixelTrackFitting: +- PixelTrackProducer - PixelFitterByConformalMappingAndLineProducer - PixelFitterByHelixProjectionsProducer RecoPixelVertexing/PixelVertexFinding: @@ -317,6 +316,7 @@ RecoTracker/TkSeedingLayers: RecoTracker/TkTrackingRegions: - PixelInactiveAreaTrackingRegionsSeedingLayersProducer RecoTracker/TrackProducer: +- TrackProducer - DAFTrackProducer - GsfTrackProducer - GsfTrackRefitter diff --git a/Utilities/StaticAnalyzers/scripts/modules_in_ib.txt b/Utilities/StaticAnalyzers/scripts/modules_in_ib.txt index 68169c2a1ef89..6783bc04bd1ed 100644 --- a/Utilities/StaticAnalyzers/scripts/modules_in_ib.txt +++ b/Utilities/StaticAnalyzers/scripts/modules_in_ib.txt @@ -86,7 +86,6 @@ CSCDcsInfo CSCDigiProducer CSCDigiToRawModule CSCDigiValidation -CSCHaloDataProducer CSCMonitorModule CSCOfflineClient CSCOfflineMonitor @@ -203,7 +202,6 @@ CosmicGenFilterHelix CosmicMuonProducer CosmicMuonSeedGenerator CosmicSeedGenerator -CosmicTrackFinder CosmicTrackSelector CosmicTrackSplitter CosmicsMuonIdProducer @@ -791,7 +789,6 @@ JetAnalyzer_HeavyIons JetAnalyzer_HeavyIons_matching JetChargeProducer JetCoreClusterSplitter -JetCorrFactorsProducer JetExtender JetFlavourClustering JetFlavourIdentifier @@ -1101,8 +1098,6 @@ NoiseRatesClient ObjMonitor OffsetAnalyzerDQM OffsetDQMPostProcessor -OmtfPacker -OmtfUnpacker Onia2MuMuPAT OniaAddV0TracksProducer OniaPhotonConversionProducer @@ -1115,41 +1110,30 @@ OuterTrackerMonitorTTTrack OuterTrackerMonitorTrackingParticles OutsideInMuonSeeder PATElectronCollectionMerger -PATElectronProducer PATElectronRefSelector PATElectronSelector -PATElectronSlimmer PATElectronUpdater PATElectronUserDataEmbedder PATGenJetSlimmer -PATGenericParticleProducer PATGenericParticleSelector -PATIsolatedTrackProducer PATJetCleaner PATJetCleanerForType1MET -PATJetProducer PATJetRefSelector PATJetSelector -PATJetSlimmer PATJetUpdater PATJetUserDataEmbedder PATLostTracks -PATMETProducer PATMETSlimmer -PATMuonProducer PATMuonRefSelector PATMuonSelector -PATMuonSlimmer PATMuonUpdater PATMuonUserDataEmbedder PATObjectCrossLinker PATPFJetMETcorrInputProducer PATPackedCandidateProducer PATPackedGenParticleProducer -PATPhotonProducer PATPhotonRefSelector PATPhotonSelector -PATPhotonSlimmer PATPhotonUserDataEmbedder PATSecondaryVertexSlimmer PATSingleVertexSelector @@ -1158,15 +1142,12 @@ PATTauDiscriminationAgainstElectronDeadECAL PATTauDiscriminationAgainstElectronMVA6 PATTauDiscriminationByMVAIsolationRun2 PATTauIDEmbedder -PATTauProducer PATTauRefSelector PATTauSelector -PATTauSlimmer PATTracksToPackedCandidates PATTriggerObjectStandAloneSelector PATTriggerObjectStandAloneSlimmer PATTriggerObjectStandAloneUnpacker -PATTriggerProducer PATVertexSlimmer PCLMetadataWriter PFBadHcalPseudoClusterProducer @@ -1211,7 +1192,6 @@ PFJetsMatchedToFilteredCaloJetsProducer PFJetsTauOverlapRemoval PFLinker PFMETDQMAnalyzer -PFMETProducer PFMultiDepthClusterProducer PFMuonDQMAnalyzer PFNuclearProducer @@ -1239,6 +1219,7 @@ PFTICLProducer PFTauDiscriminatorLogicalAndProducer PFTauFwdPtrProducer PFTauPrimaryVertexProducer +PFTauPrimaryVertexProducerBase PFTauSecondaryVertexProducer PFTauSelector PFTauTransverseImpactParameters @@ -1551,6 +1532,7 @@ TTbar_GenLepAnalyzer TTbar_Kinematics TagAndProbeBtagTriggerMonitor Tau3MuMonitor +TauDiscriminationAgainstElectronDeadECAL TauDQMFileLoader TauDQMHistEffProducer TauGenJetDecayModeSelector @@ -1672,3 +1654,27 @@ dEdxHitAnalyzer photonIsolationHIProducer trackerDTC::ProducerED trgMatchGsfElectronProducer +omtf::OmtfPacker +omtf::OmtfUnpacker +edm::BMixingModule +pat::JetCorrFactorsProducer +pat::PATElectronProducer +pat::PATElectronSlimmer +pat::PATGenericParticleProducer +pat::PATIsolatedTrackProducer +pat::PATJetProducer +pat::PATJetSlimmer +pat::PATMETProducer +pat::PATMuonProducer +pat::PATMuonSlimmer +pat::PATPhotonProducer +pat::PATPhotonSlimmer +pat::PATTauProducer +pat::PATTauSlimmer +pat::PATTriggerProducer +pat::TauJetCorrFactorsProducer +cms::PFMETProducer +reco::PFMETProducerMVA +reco::CSCHaloDataProducer +cms::CosmicTrackFinder +