Initial implementation of the FixMissingStreamerInfos service

IOPool/Input/scripts/makeFileContainingStreamerInfos.C

@@ -0,0 +1,90 @@
+// Build a working release including the desired class versions in the
+// class definitions and classes_def.xml files. You might need to add or
+// remove lines below to get the StreamerInfo objects for the desired classes.
+// Then run the following to execute this script:
+//
+//   root -l -b -q makeFileContainingStreamerInfos.C
+//
+// Then rename the output file as appropriate. If it is of general use
+// you might want to reposit the file in the IOPool/Input data repository.
+// This output file can be used by the service named FixMissingStreamerInfos.
+
+#include <iostream>
+
+void makeFileContainingStreamerInfos() {
+  std::cout << "Executing makeFileContainingStreamerInfos()" << std::endl;
+  auto f = TFile::Open("fileContainingStreamerInfos.root", "NEW");
+
+  TClass::GetClass("BeamSpotOnline")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("CTPPSLocalTrackLite")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("CTPPSPixelDataError")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("CTPPSPixelDigi")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("CorrMETData")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("DcsStatus")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("EcalTriggerPrimitiveSample")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("HaloTowerStrip")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("HcalElectronicsId")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1AcceptBunchCrossing")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctEmCand")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctEtHad")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctEtMiss")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctEtTotal")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctHFBitCounts")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctHFRingEtSums")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctHtMiss")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctJetCand")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GctJetCounts")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GtFdlWord")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1GtPsbWord")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1MuGMTReadoutRecord")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("L1TriggerScalers")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("Level1TriggerScalers")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("LumiScalers")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("Measurement1DFloat")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("PixelFEDChannel")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("Run3ScoutingParticle")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("Run3ScoutingTrack")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("Run3ScoutingVertex")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("TotemFEDInfo")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("ZDCDataFrame")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("ZDCRecHit")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<CTPPSDiamondDigi>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<CTPPSDiamondLocalTrack>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<CTPPSDiamondRecHit>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<CTPPSPixelLocalTrack>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemRPCluster>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemRPDigi>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemRPLocalTrack>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemRPRecHit>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemRPUVPattern>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemTimingDigi>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemTimingLocalTrack>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::DetSet<TotemTimingRecHit>")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::HLTPathStatus")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::IndexIntoFile::RunOrLumiEntry")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::RefCoreWithIndex")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::StoredMergeableRunProductMetadata::SingleRunEntry")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::StoredMergeableRunProductMetadata::SingleRunEntryAndProcess")
+      ->GetStreamerInfo()
+      ->ForceWriteInfo(f);
+  TClass::GetClass("edm::StoredProductProvenance")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("edm::ThinnedAssociationBranches")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("l1t::CaloTower")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("l1t::RegionalMuonShower")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::DeDxData")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::ElectronSeed::PMVars")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::ForwardProton")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::JetID")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::MuonCosmicCompatibility")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::MuonGEMHitMatch")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::MuonMETCorrectionData")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::MuonRPCHitMatch")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::MuonTimeExtra")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::PhiWedge")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("reco::RecoEcalCandidate")->GetStreamerInfo()->ForceWriteInfo(f);
+  TClass::GetClass("trigger::TriggerObject")->GetStreamerInfo()->ForceWriteInfo(f);
+
+  TClass::GetClass("l1t::MuonShower")->GetStreamerInfo()->ForceWriteInfo(f);
+
+  delete f;
+}

IOPool/Input/src/FixMissingStreamerInfos.cc

@@ -0,0 +1,74 @@
+// -*- C++ -*-
+//
+// Package:     Services
+// Class  :     FixMissingStreamerInfos
+//
+// Implementation:
+
+/** \class edm::service::FixMissingStreamerInfos
+
+This service is used to open and close a ROOT file that contains
+StreamerInfo objects causing them to be saved in memory. It is
+used when reading a file written with a version of ROOT with a
+bug that caused it to fail to write out StreamerInfo objects.
+(see Issue 41246).
+
+CMSSW_13_0_0 had such a problem and files were written with
+this problem. When using this service to read files written
+with this release set the "fileInPath" parameter to the string
+"IOPool/Input/data/fileContainingStreamerInfos_13_0_0.root".
+This file is saved in the cms-data repository for IOPool/Input.
+Note that it was difficult to identify all the problem classes
+and we might have missed some. If there are additional problem
+classes a new version of this file can be generated with script
+IOPool/Input/scripts/makeFileContainingStreamerInfos.C. If the
+problem ever recurs in ROOT with a different release, one could
+use that script to generate a file containing StreamerInfos for
+other releases.
+
+    \author W. David Dagenhart, created 30 October, 2023
+
+*/
+
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
+#include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
+#include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
+#include "FWCore/Utilities/interface/EDMException.h"
+#include "FWCore/Utilities/interface/FileInPath.h"
+
+#include "TFile.h"
+
+namespace edm {
+  namespace service {
+
+    class FixMissingStreamerInfos {
+    public:
+      FixMissingStreamerInfos(ParameterSet const&, ActivityRegistry&);
+      static void fillDescriptions(ConfigurationDescriptions&);
+
+    private:
+      FileInPath fileInPath_;
+    };
+
+    FixMissingStreamerInfos::FixMissingStreamerInfos(ParameterSet const& pset, edm::ActivityRegistry&)
+        : fileInPath_(pset.getUntrackedParameter<FileInPath>("fileInPath")) {
+      auto tFile = TFile::Open(fileInPath_.fullPath().c_str());
+      if (!tFile || tFile->IsZombie()) {
+        throw cms::Exception("FixMissingStreamerInfo")
+            << "Failed opening file containing missing StreamerInfos: " << fileInPath_.fullPath();
+      }
+      tFile->Close();
+    }
+
+    void FixMissingStreamerInfos::fillDescriptions(ConfigurationDescriptions& descriptions) {
+      ParameterSetDescription desc;
+      desc.addUntracked<FileInPath>("fileInPath");
+      descriptions.add("FixMissingStreamerInfos", desc);
+    }
+  }  // namespace service
+}  // namespace edm
+
+using namespace edm::service;
+DEFINE_FWK_SERVICE(FixMissingStreamerInfos);

IOPool/Input/test/SchemaEvolution_test_read_cfg.py

@@ -1,3 +1,5 @@
+# This configuration is used to test ROOT schema evolution.
+
 import FWCore.ParameterSet.Config as cms
 import sys
 import argparse
@@ -6,10 +8,72 @@
 
 parser.add_argument("--inputFile", type=str, help="Input file name (default: SchemaEvolutionTest.root)", default="SchemaEvolutionTest.root")
 parser.add_argument("--outputFileName", type=str, help="Output file name (default: SchemaEvolutionTest2.root)", default="SchemaEvolutionTest2.root")
+parser.add_argument("--enableStreamerInfosFix", action="store_true", help="Enable service that fixes missing streamer infos")
 args = parser.parse_args()
 
 process = cms.Process("READ")
 
+# The service named FixMissingStreamerInfos is
+# tested when enabled.
+#
+# The version of ROOT associated with CMSSW_13_0_0
+# had a bug that caused some products to be written
+# to an output file with no StreamerInfo in the file.
+# At some future point, if the format of one of those
+# types changes then ROOT schema evolution fails.
+#
+# There is a workaround fix for this problem that
+# involves creating a standalone ROOT file that
+# contains the StreamerInfo's. Opening and closing
+# that file before reading the data files will
+# bring the StreamerInfos into memory and makes
+# the problem files readable even if the data formats
+# change.
+#
+# Create the "fixit" file as follows:
+#
+# Create a working area with release CMSSW_13_2_6
+# There is nothing special about that release. We could
+# have used another release for this. It was just the latest
+# at the time this was done (11/1/2023). I didn't want to use
+# an IB or pre-release.
+# Then add the package with the relevant class definitions and dictionaries:
+#     git cms-addpkg DataFormats/TestObjects
+#
+# Add the relevant files from the master branch:
+#    git checkout official-cmssw/master DataFormats/TestObjects/interface/SchemaEvolutionTestObjects.h
+#    git checkout official-cmssw/master DataFormats/TestObjects/interface/VectorVectorTop.h 
+#    git checkout official-cmssw/master DataFormats/TestObjects/src/VectorVectorTop.cc
+#    git checkout official-cmssw/master DataFormats/TestObjects/src/SchemaEvolutionTestObjects.cc
+#    git checkout official-cmssw/master DataFormats/TestObjects/src/classes_def.xml                 
+#    git checkout official-cmssw/master DataFormats/TestObjects/src/classes.h      
+#
+# Edit the files to use the older versions as described above and build.
+#
+# Note that if a release is available with the desired versions
+# (class definitions and classes_def.xml), then you don't need
+# to checkout the code and/or edit the code. You probably want
+# to use a release at or earlier than the release you will use
+# to read the file (this might not be necessary, depends on what
+# has changed in ROOT...).
+#
+# Start root, then give the following commands at the root prompt:
+#
+#    root [0] auto f = TFile::Open("fixitfile.root", "NEW");
+#    root [1] TClass::GetClass("edmtest::VectorVectorElementNonSplit")->GetStreamerInfo()->ForceWriteInfo(f);
+#    root [2] delete f
+#
+# rename the output file to "fixMissingStreamerInfosUnitTest.root" and add
+# to the cms-data repository for IOPool/Input.
+#
+# Note the test only needs the one class definition, but in real use
+# cases many different types of StreamerInfos might be needed.
+
+if args.enableStreamerInfosFix:
+    process.FixMissingStreamerInfos = cms.Service("FixMissingStreamerInfos",
+        fileInPath = cms.untracked.FileInPath("IOPool/Input/data/fixMissingStreamerInfosUnitTest.root")
+    )
+
 process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring("file:"+args.inputFile))
 
 process.schemaEvolutionTestRead = cms.EDAnalyzer("SchemaEvolutionTestRead",

IOPool/Input/test/testSchemaEvolution.sh

@@ -91,15 +91,11 @@ cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py --inputFile "$inputfil
 
 file=SchemaEvolutionTestOLD13_0_0.root
 inputfile=$(edmFileInPath IOPool/Input/data/$file) || die "Failure edmFileInPath IOPool/Input/data/$file" $?
-# These fail because there was a bug in the version of ROOT associated with CMSSW_13_0_0
-# The bug caused StreamerInfo objects to missing from the ROOT file. In this case,
-# schema evolution fails and also the testForStreamerInfo.C script will find
-# missing StreamerInfo objects.
-# Lets keep this code around because it may be useful if we need to
-# do additional work related to data files written using an executable
-# built from code having the bug.
-#cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py --inputFile "$inputfile" || die "Failed to read old file $file" $?
-#root.exe -b -l -q file:$inputfile "${LOCAL_TEST_DIR}/testForStreamerInfo.C(gFile)" | sort -u | grep Missing > testForStreamerInfo2.log
-#grep "Missing" testForStreamerInfo2.log && die "Missing nested streamer info" 1
+# The test below would fail without the "--enableStreamerInfosFix"
+# because there was a bug in the version of ROOT associated with CMSSW_13_0_0.
+# The bug caused StreamerInfo objects to be missing from the ROOT file. In this case,
+# schema evolution fails without the fix and also the testForStreamerInfo.C script will
+# find missing StreamerInfo objects.
+cmsRun ${LOCAL_TEST_DIR}/SchemaEvolution_test_read_cfg.py --inputFile $inputfile --enableStreamerInfosFix || die "Failed to read old file $file with fix" $?
 
 exit 0