Utilities/StaticAnalyzers: Update generation of EventSetupRecord::get report.

Utilities/StaticAnalyzers/scripts/callgraph.py

@@ -2,82 +2,117 @@
 from __future__ import print_function
 import networkx as nx
 import re
-import yaml
+import json
+from yaml import load, dump, FullLoader
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
 
-topfunc = re.compile(r"::(accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(")
+topfunc = r"::(accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\("
+topfuncre = re.compile(topfunc)
+
+baseclass = r"\b(edm::)?(one::|stream::|global::)?((DQM)?(Global|One)?ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator|RunWatcher|RunCacheHolder))\b"
+baseclassre = re.compile(baseclass)
+assert(baseclassre.match('edm::global::EDFilter::filter() virtual'))
+assert(topfuncre.search('edm::global::EDFilterBase::filter(&) const virtual'))
 
-baseclass = re.compile(
-    r"edm::(one::|stream::|global::)?(ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator))")
 farg = re.compile(r"\(.*?\)")
 tmpl = re.compile(r'<.*?>')
 toplevelfuncs = set()
-epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<.*>\(.*\)")
+
+epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<(class|struct) edm::ES(Transient)?Handle<(class|struct) .*>>\((const char \*, |const std::string &, )(class|struct) edm::ES(Transient)?Handle<(class|struct).*> &\) const")
+f= 'edm::eventsetup::EventSetupRecord::get<class edm::ESHandle<class CaloTopology>>(const std::string &, class edm::ESHandle<class CaloTopology> &) const'
+assert(epfuncre.search(f))
+
 skipfunc = re.compile(r"TGraph::IsA\(.*\)")
 epfuncs = set()
 
-G = nx.DiGraph()
+function='(anonymous namespace in /src/RecoTracker/TkHitPairs/plugins/HitPairEDProducer.cc)::Impl<(anonymous namespace)::ImplSeedingHitSets, (anonymous namespace)::DoNothing, (anonymous namespace)::RegionsLayersSeparate>::produce(const _Bool, edm::Event &, const edm::EventSetup &);'
+function2='HitPairEDProducer::produce(edm::Event &, const edm::EventSetup &);'
+value='HitPairEDProducer'
+valuere=r'\b%s\b' % value
+vre=re.compile(valuere)
+m=vre.search(function)
+assert(m)
+m2=vre.search(function2)
+assert(m2)
+
+modules = list()
+with open('modules_in_ib.txt', 'r') as m:
+    for line in m:
+        modules.append(line.strip())
+mods='|'.join(modules)
+mods_regex=r'\b(' + mods+ r'QQQQQQQQ)\b'
+comp_mods_regex=re.compile(mods_regex)
+n=comp_mods_regex.search(function)
+assert(n)
+n2=comp_mods_regex.search(function2)
+assert(n2)
+
+module2package = dict()
+with open('module_to_package.yaml') as m:
+    module2package=load(m, Loader=FullLoader)
 
-# g = open('module_to_package.txt')
-# module2package=dict()
-# for line in g:
-#     fields = line.strip().split(';')
-#     if len(fields) <2:
-#         continue
-#     module2package.setdefault(fields[1], []).append(fields[0])
-#
-# i = open('module_to_package.yaml', 'w')
-# yaml.dump(module2package, i)
-# i.close()
+with open('packages.json', 'r') as j:
+    p = json.load(j)
 
-h = open('module_to_package.yaml', 'r')
-module2package = yaml.load(h, Loader=yaml.FullLoader)
+for k,v in p.items():
+    module=k.split('|')[0]
+    dirs=v.split('|')
+    package=dirs[0]+'/'+dirs[1]
+    module2package.setdefault(package, []).append(module)
 
+for k in module2package.keys():
+    module2package[k]=sorted(set(module2package[k]))
+
+G = nx.DiGraph()
 with open('function-calls-db.txt') as f:
     for line in f:
         fields = line.split("'")
         if len(fields) < 3:
             continue
         if fields[2] == ' calls function ':
-            if not skipfunc.search(line):
-                G.add_edge(fields[1], fields[3], kind=fields[2])
-                if epfuncre.search(fields[3]):
-                    epfuncs.add(fields[3])
+            G.add_edge(fields[1], fields[3], kind=fields[2])
+            if epfuncre.search(fields[3]):
+                epfuncs.add(fields[3])
         if fields[2] == ' overrides function ':
-            if baseclass.search(fields[3]):
-                if topfunc.search(fields[3]):
-                    toplevelfuncs.add(fields[1])
+            if baseclassre.match(fields[3]):
                 G.add_edge(fields[1], fields[3], kind=' overrides function ')
+                if topfuncre.search(fields[1]) and comp_mods_regex.search(fields[1]):
+                    toplevelfuncs.add(fields[1])
             else:
-                if not skipfunc.search(line):
-                    G.add_edge(fields[3], fields[1],
-                               kind=' calls override function ')
-                    if epfuncre.search(fields[1]):
-                        epfuncs.add(fields[1])
+                G.add_edge(fields[3], fields[1], kind=' overriden function calls function ')
+            if epfuncre.search(fields[3]):
+                epfuncs.add(fields[3])
 
 callstacks = set()
 for tfunc in toplevelfuncs:
     for epfunc in epfuncs:
         if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G, tfunc, epfunc):
-            path = nx.shortest_path(G, tfunc, epfunc)
-            cs = str("")
-            previous = str("")
-            for p in path:
-                if epfuncre.search(p):
-                    break
-                stripped = re.sub(farg, "()", p)
-                if previous != stripped:
-                    cs += ' ' + stripped + ";"
-                    previous = stripped
-                callstacks.add(cs)
-                break
-
+            for path in nx.all_shortest_paths(G, tfunc, epfunc):
+                cs = str("")
+                previous = str("")
+                for p in path:
+                    if epfuncre.match(p):
+                        break
+                    #stripped=re.sub(farg, "()", p)
+                    stripped=p
+                    if previous != stripped:
+                        cs += stripped + '; '
+                        previous = stripped
+                if cs not in callstacks:
+                    callstacks.add(cs)
 
 report = dict()
-for key in sorted(module2package.keys()):
-    for value in sorted(module2package[key]):
-        vre = re.compile(' %s::.*();' % value)
-        for cs in sorted(callstacks):
-            if vre.search(cs):
-                report.setdefault(key, {}).setdefault(value, []).append(cs)
+for key in sorted(set(module2package.keys())):
+    for value in sorted(set(module2package[key])):
+        if comp_mods_regex.match(value):
+            regex_str = r'\b%s\b'%value
+            vre=re.compile(regex_str)
+            for callstack in sorted(callstacks):
+                if vre.search(callstack):
+                     report.setdefault(str(key), {}).setdefault(str(value), []).append(str(callstack))
+
 r = open('eventsetuprecord-get.yaml', 'w')
-yaml.dump(report, r)
+dump(report, r, width=float("inf"))

Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh

@@ -49,6 +49,8 @@ if [ ! -f ./callgraph.py ]
    then
    cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/callgraph.py .
    cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/module_to_package.yaml .
+   cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/modules_in_ib.yaml .
+   curl -OL https://raw.githubusercontent.com/fwyzard/circles/master/web/groups/packages.json
 fi
 touch eventsetuprecord-get-all.txt eventsetuprecord-get.txt
 ./callgraph.py 2>&1 | tee eventsetuprecord-get.txt

Utilities/StaticAnalyzers/scripts/module_to_package.yaml

@@ -156,7 +156,6 @@ RecoEgamma/EgammaPhotonProducers:
 - ConversionTrackCandidateProducer
 - GEDPhotonProducer
 - PhotonProducer
-- TrackProducer
 - TrackProducerWithSCAssociation
 RecoEgamma/Examples:
 - SiStripElectronAnalyzer
@@ -276,10 +275,10 @@ RecoParticleFlow/PFTracking:
 - PFNuclearProducer
 - PFV0Producer
 RecoPixelVertexing/PixelLowPtUtilities:
-- PixelTrackProducer
 - PixelVertexProducerClusters
 - SiPixelClusterShapeCacheProducer
 RecoPixelVertexing/PixelTrackFitting:
+- PixelTrackProducer
 - PixelFitterByConformalMappingAndLineProducer
 - PixelFitterByHelixProjectionsProducer
 RecoPixelVertexing/PixelVertexFinding:
@@ -317,6 +316,7 @@ RecoTracker/TkSeedingLayers:
 RecoTracker/TkTrackingRegions:
 - PixelInactiveAreaTrackingRegionsSeedingLayersProducer
 RecoTracker/TrackProducer:
+- TrackProducer
 - DAFTrackProducer
 - GsfTrackProducer
 - GsfTrackRefitter

Utilities/StaticAnalyzers/scripts/modules_in_ib.txt

@@ -86,7 +86,6 @@ CSCDcsInfo
 CSCDigiProducer
 CSCDigiToRawModule
 CSCDigiValidation
-CSCHaloDataProducer
 CSCMonitorModule
 CSCOfflineClient
 CSCOfflineMonitor
@@ -203,7 +202,6 @@ CosmicGenFilterHelix
 CosmicMuonProducer
 CosmicMuonSeedGenerator
 CosmicSeedGenerator
-CosmicTrackFinder
 CosmicTrackSelector
 CosmicTrackSplitter
 CosmicsMuonIdProducer
@@ -791,7 +789,6 @@ JetAnalyzer_HeavyIons
 JetAnalyzer_HeavyIons_matching
 JetChargeProducer
 JetCoreClusterSplitter
-JetCorrFactorsProducer
 JetExtender
 JetFlavourClustering
 JetFlavourIdentifier
@@ -1101,8 +1098,6 @@ NoiseRatesClient
 ObjMonitor
 OffsetAnalyzerDQM
 OffsetDQMPostProcessor
-OmtfPacker
-OmtfUnpacker
 Onia2MuMuPAT
 OniaAddV0TracksProducer
 OniaPhotonConversionProducer
@@ -1115,41 +1110,30 @@ OuterTrackerMonitorTTTrack
 OuterTrackerMonitorTrackingParticles
 OutsideInMuonSeeder
 PATElectronCollectionMerger
-PATElectronProducer
 PATElectronRefSelector
 PATElectronSelector
-PATElectronSlimmer
 PATElectronUpdater
 PATElectronUserDataEmbedder
 PATGenJetSlimmer
-PATGenericParticleProducer
 PATGenericParticleSelector
-PATIsolatedTrackProducer
 PATJetCleaner
 PATJetCleanerForType1MET
-PATJetProducer
 PATJetRefSelector
 PATJetSelector
-PATJetSlimmer
 PATJetUpdater
 PATJetUserDataEmbedder
 PATLostTracks
-PATMETProducer
 PATMETSlimmer
-PATMuonProducer
 PATMuonRefSelector
 PATMuonSelector
-PATMuonSlimmer
 PATMuonUpdater
 PATMuonUserDataEmbedder
 PATObjectCrossLinker
 PATPFJetMETcorrInputProducer
 PATPackedCandidateProducer
 PATPackedGenParticleProducer
-PATPhotonProducer
 PATPhotonRefSelector
 PATPhotonSelector
-PATPhotonSlimmer
 PATPhotonUserDataEmbedder
 PATSecondaryVertexSlimmer
 PATSingleVertexSelector
@@ -1158,15 +1142,12 @@ PATTauDiscriminationAgainstElectronDeadECAL
 PATTauDiscriminationAgainstElectronMVA6
 PATTauDiscriminationByMVAIsolationRun2
 PATTauIDEmbedder
-PATTauProducer
 PATTauRefSelector
 PATTauSelector
-PATTauSlimmer
 PATTracksToPackedCandidates
 PATTriggerObjectStandAloneSelector
 PATTriggerObjectStandAloneSlimmer
 PATTriggerObjectStandAloneUnpacker
-PATTriggerProducer
 PATVertexSlimmer
 PCLMetadataWriter
 PFBadHcalPseudoClusterProducer
@@ -1211,7 +1192,6 @@ PFJetsMatchedToFilteredCaloJetsProducer
 PFJetsTauOverlapRemoval
 PFLinker
 PFMETDQMAnalyzer
-PFMETProducer
 PFMultiDepthClusterProducer
 PFMuonDQMAnalyzer
 PFNuclearProducer
@@ -1239,6 +1219,7 @@ PFTICLProducer
 PFTauDiscriminatorLogicalAndProducer
 PFTauFwdPtrProducer
 PFTauPrimaryVertexProducer
+PFTauPrimaryVertexProducerBase
 PFTauSecondaryVertexProducer
 PFTauSelector
 PFTauTransverseImpactParameters
@@ -1551,6 +1532,7 @@ TTbar_GenLepAnalyzer
 TTbar_Kinematics
 TagAndProbeBtagTriggerMonitor
 Tau3MuMonitor
+TauDiscriminationAgainstElectronDeadECAL
 TauDQMFileLoader
 TauDQMHistEffProducer
 TauGenJetDecayModeSelector
@@ -1672,3 +1654,27 @@ dEdxHitAnalyzer
 photonIsolationHIProducer
 trackerDTC::ProducerED
 trgMatchGsfElectronProducer
+omtf::OmtfPacker
+omtf::OmtfUnpacker
+edm::BMixingModule
+pat::JetCorrFactorsProducer
+pat::PATElectronProducer
+pat::PATElectronSlimmer
+pat::PATGenericParticleProducer
+pat::PATIsolatedTrackProducer
+pat::PATJetProducer
+pat::PATJetSlimmer
+pat::PATMETProducer
+pat::PATMuonProducer
+pat::PATMuonSlimmer
+pat::PATPhotonProducer
+pat::PATPhotonSlimmer
+pat::PATTauProducer
+pat::PATTauSlimmer
+pat::PATTriggerProducer
+pat::TauJetCorrFactorsProducer
+cms::PFMETProducer
+reco::PFMETProducerMVA
+reco::CSCHaloDataProducer
+cms::CosmicTrackFinder
+