for adding 2024 campaign autotest and an update and another request

.github/workflows/main.yml

@@ -127,4 +127,5 @@ jobs:
           python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TRK-Run3Summer23wmLHEGS-00002
           python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TSG-Run3Summer23BPixGS-00007
           python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 
-          python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid BPH-Run3Winter24GS-00001 
+          python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid BPH-Run3Winter24GS-00001
+          python3 bin/utils/request_fragment_check.py --bypass_status --develop --bypass_runcmsgrid_patch --local --prepid GEN-RunIII2024Summer24wmLHEGS-00001 

bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001

@@ -0,0 +1,40 @@
+import FWCore.ParameterSet.Config as cms
+
+externalLHEProducer = cms.EDProducer('ExternalLHEProducer',
+    args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/PdmV/RunIII2024Summer24/Powheg/TT/hvq_slc7_amd64_gcc10_CMSSW_12_4_8_TTtoLNu2Q_powheg-pythia8.tgz'),
+    nEvents = cms.untracked.uint32(5000),
+    numberOfParameters = cms.uint32(1),
+    outputFile = cms.string('cmsgrid_final.lhe'),
+    scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh'),
+    generateConcurrently = cms.untracked.bool(True),
+)
+
+from Configuration.Generator.Pythia8CommonSettings_cfi import *
+from Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import *
+from Configuration.Generator.Pythia8PowhegEmissionVetoSettings_cfi import *
+from Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *
+
+generator = cms.EDFilter("Pythia8ConcurrentHadronizerFilter",
+    PythiaParameters = cms.PSet(
+        pythia8CommonSettingsBlock,
+        pythia8CP5SettingsBlock,
+        pythia8PowhegEmissionVetoSettingsBlock,
+        pythia8PSweightsSettingsBlock,
+        processParameters = cms.vstring(
+        'POWHEG:nFinal = 2',
+        'TimeShower:mMaxGamma = 4.0'
+        ),
+        parameterSets = cms.vstring(
+            'pythia8CommonSettings',
+            'pythia8CP5Settings',
+            'pythia8PowhegEmissionVetoSettings',
+            'processParameters',
+            'pythia8PSweightsSettings',
+        )
+    ),
+    comEnergy = cms.double(13600),
+    maxEventsToPrint = cms.untracked.int32(1),
+    pythiaHepMCVerbosity = cms.untracked.bool(False),
+    pythiaPylistVerbosity = cms.untracked.int32(1),
+)
+

bin/utils/GEN-RunIII2024Summer24wmLHEGS-00001_get_test

@@ -0,0 +1,158 @@
+#!/bin/bash
+
+# Binds for singularity containers
+# Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd
+export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user'
+
+cat <<'EndOfGenScriptFile' > GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh
+#!/bin/bash
+
+echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64"
+python3 -m venv cms_gen_venv_GEN-RunIII2024Summer24wmLHEGS-00001 && source ./cms_gen_venv_GEN-RunIII2024Summer24wmLHEGS-00001/bin/activate
+
+# Install the PdmV REST client
+pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null
+
+echo "Packages installed"
+pip freeze
+echo ""
+
+# GEN Script begin
+rm -f request_fragment_check.py
+wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py
+chmod +x request_fragment_check.py
+
+./request_fragment_check.py --bypass_status --prepid GEN-RunIII2024Summer24wmLHEGS-00001
+
+# End of CMS GEN script file: GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh
+EndOfGenScriptFile
+chmod +x GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh
+
+# Run in singularity container
+singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/GEN-RunIII2024Summer24wmLHEGS-00001_gen_script.sh)
+
+GEN_ERR=$?
+if [ $GEN_ERR -ne 0 ]; then
+  echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors"
+  echo "Validation WILL NOT RUN"
+  echo "Please correct errors in the request and run validation again"
+  exit $GEN_ERR
+fi
+echo "Running VALIDATION. GEN Request Checking Script returned no errors"
+# GEN Script end
+
+# Download fragment from McM
+curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/GEN-RunIII2024Summer24wmLHEGS-00001 --retry 3 --create-dirs -o Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py
+[ -s Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py ] || exit $?;
+
+# Check if fragment contais gridpack path ant that it is in cvmfs
+if grep -q "gridpacks" Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py; then
+  if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py; then
+    echo "Gridpack inside fragment is not in cvmfs."
+    exit -1
+  fi
+fi
+
+# Dump actual test code to a GEN-RunIII2024Summer24wmLHEGS-00001_test.sh file that can be run in Singularity
+cat <<'EndOfTestFile' > GEN-RunIII2024Summer24wmLHEGS-00001_test.sh
+#!/bin/bash
+
+export SCRAM_ARCH=el8_amd64_gcc12
+
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+if [ -r CMSSW_14_0_18/src ] ; then
+  echo release CMSSW_14_0_18 already exists
+else
+  scram p CMSSW CMSSW_14_0_18
+fi
+cd CMSSW_14_0_18/src
+eval `scram runtime -sh`
+
+mv ../../Configuration .
+scram b
+cd ../..
+
+# Maximum validation duration: 28800s
+# Margin for validation duration: 30%
+# Validation duration with margin: 28800 * (1 - 0.30) = 20160s
+# Time per event for each sequence: 22.1832s
+# Threads for each sequence: 4
+# Time per event for single thread for each sequence: 4 * 22.1832s = 88.7329s
+# Which adds up to 88.7329s per event
+# Single core events that fit in validation duration: 20160s / 88.7329s = 227
+# Produced events limit in McM is 10000
+# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000
+# Take the minimum of 227 and 10000, but more than 0 -> 227
+# It is estimated that this validation will produce: 227 * 1.0000 = 227 events
+EVENTS=227
+
+# Random seed between 1 and 100 for externalLHEProducer
+SEED=$(($(date +%s) % 100 + 1))
+
+
+# cmsDriver command
+cmsDriver.py Configuration/GenProduction/python/GEN-RunIII2024Summer24wmLHEGS-00001-fragment.py --python_filename GEN-RunIII2024Summer24wmLHEGS-00001_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN-SIM,LHE --fileout file:GEN-RunIII2024Summer24wmLHEGS-00001.root --conditions 140X_mcRun3_2024_realistic_v26 --beamspot DBrealistic --customise_commands process.RandomNumberGeneratorService.externalLHEProducer.initialSeed="int(${SEED})"\\nprocess.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN,SIM --geometry DB:Extended --era Run3_2024 --no_exec --mc -n $EVENTS || exit $? ;
+
+# Run generated config
+REPORT_NAME=GEN-RunIII2024Summer24wmLHEGS-00001_report.xml
+# Run the cmsRun
+cmsRun -e -j $REPORT_NAME GEN-RunIII2024Summer24wmLHEGS-00001_1_cfg.py || exit $? ;
+
+# Parse values from GEN-RunIII2024Summer24wmLHEGS-00001_report.xml report
+processedEvents=$(grep -Po "(?<=<Metric Name=\"NumberEvents\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+producedEvents=$(grep -Po "(?<=<TotalEvents>)(\d*)(?=</TotalEvents>)" $REPORT_NAME | tail -n 1)
+threads=$(grep -Po "(?<=<Metric Name=\"NumberOfThreads\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+peakValueRss=$(grep -Po "(?<=<Metric Name=\"PeakValueRss\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+peakValueVsize=$(grep -Po "(?<=<Metric Name=\"PeakValueVsize\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalSize=$(grep -Po "(?<=<Metric Name=\"Timing-tstoragefile-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalSizeAlt=$(grep -Po "(?<=<Metric Name=\"Timing-file-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalJobTime=$(grep -Po "(?<=<Metric Name=\"TotalJobTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+totalJobCPU=$(grep -Po "(?<=<Metric Name=\"TotalJobCPU\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+eventThroughput=$(grep -Po "(?<=<Metric Name=\"EventThroughput\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+avgEventTime=$(grep -Po "(?<=<Metric Name=\"AvgEventTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1)
+if [ -z "$threads" ]; then
+  echo "Could not find NumberOfThreads in report, defaulting to 1"
+  threads=1
+fi
+if [ -z "$eventThroughput" ]; then
+  eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)")
+fi
+if [ -z "$totalSize" ]; then
+  totalSize=$totalSizeAlt
+fi
+if [ -z "$processedEvents" ]; then
+  processedEvents=$EVENTS
+fi
+echo "Validation report of GEN-RunIII2024Summer24wmLHEGS-00001 sequence 1/1"
+echo "Processed events: $processedEvents"
+echo "Produced events: $producedEvents"
+echo "Threads: $threads"
+echo "Peak value RSS: $peakValueRss MB"
+echo "Peak value Vsize: $peakValueVsize MB"
+echo "Total size: $totalSize MB"
+echo "Total job time: $totalJobTime s"
+echo "Total CPU time: $totalJobCPU s"
+echo "Event throughput: $eventThroughput"
+echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %"
+echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB"
+echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s"
+echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %"
+echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents")
+
+# End of GEN-RunIII2024Summer24wmLHEGS-00001_test.sh file
+EndOfTestFile
+
+# Make file executable
+chmod +x GEN-RunIII2024Summer24wmLHEGS-00001_test.sh
+
+if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:amd64" ]; then
+  CONTAINER_NAME="el8:amd64"
+elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:x86_64" ]; then
+  CONTAINER_NAME="el8:x86_64"
+else
+  echo "Could not find amd64 or x86_64 for el8"
+  exit 1
+fi
+# Run in singularity container
+export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity"
+singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/GEN-RunIII2024Summer24wmLHEGS-00001_test.sh)

bin/utils/request_GEN-RunIII2024Summer24wmLHEGS-00001

@@ -0,0 +1 @@
+{"results": {"validation": {"content": "all", "valid": false, "results": {"1": [{"peak_value_rss": 1353.18, "total_events": 227, "estimated_events_per_lumi": 2491.4303999999997, "cpu_efficiency": 0.9906964733938126, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 11.55962454339483, "expected_events": 227, "size_per_event": 894.145973568282}], "8": [{"peak_value_rss": 2046.88, "total_events": 227, "estimated_events_per_lumi": 16484.4288, "cpu_efficiency": 0.871975369494733, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 1.7471033027240834, "expected_events": 227, "size_per_event": 889.860511013216}], "2": [{"peak_value_rss": 1571.68, "total_events": 227, "estimated_events_per_lumi": 5081.8752, "cpu_efficiency": 0.9830421416875211, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 5.667199383408707, "expected_events": 227, "size_per_event": 846.7983788546255}], "4": [{"peak_value_rss": 1713.93, "total_events": 227, "estimated_events_per_lumi": 9597.456, "cpu_efficiency": 0.9382336828924716, "filter_efficiency": 1, "cpu_name": "AMD EPYC 7302 16-Core Processor ()", "time_per_event": 3.0007952107308435, "expected_events": 227, "size_per_event": 843.0452158590308}]}}, "total_events": 500000000, "config_id": [], "events_per_lumi": 0, "_rev": "11-95a509edc0601cffbf2683cc564e82e2", "mcdb_id": 0, "transient_output_modules": [[]], "sequences": [{"eventcontent": ["RAWSIM", "LHE"], "runsScenarioForMC": "", "donotDropOnInput": "", "extra": "", "slhc": "", "runsAndWeightsForMC": "", "outputCommand": "", "dropDescendant": false, "pileup": "", "hltProcess": "", "index": -1, "customise": "", "custom_conditions": "", "datatier": ["GEN-SIM", "LHE"], "processName": "", "inputCommands": "", "particle_table": "", "conditions": "140X_mcRun3_2024_realistic_v26", "harvesting": "", "triggerResultsProcess": "", "inputEventContent": "", "beamspot": "DBrealistic", "restoreRNDSeeds": "", "filtername": "", "repacked": "", "customise_commands": "", "step": ["LHE", "GEN", "SIM"], "gflash": "", "procModifiers": "", "magField": "", "himix": false, "nThreads": "4", "scenario": "", "geometry": "DB:Extended", "nStreams": 0, "datamix": "", "inline_custom": false, "era": "Run3_2024"}], "block_black_list": [], "block_white_list": [], "priority": 20000, "process_string": "", "ppd_tags": [], "fragment_tag": "", "generator_parameters": [{"submission_details": {"submission_date": "2023-09-19-20-03", "author_email": "[email protected]", "author_username": "cvicovil", "author_name": "Carlos Vico Villalba"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 0, "cross_section": 1, "filter_efficiency_error": 0}, {"submission_details": {"author_email": "[email protected]", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}, "match_efficiency_error": 0, "match_efficiency": 1, "filter_efficiency": 1, "negative_weights_fraction": 0, "version": 1, "cross_section": 1, "filter_efficiency_error": 0}], "flown_with": "", "interested_pwg": ["GEN"], "version": 0, "generators": ["Powheg", "Pythia8"], "memory": 8000, "pilot": false, "type": "LHE", "cmssw_release": "CMSSW_14_0_18", "status": "approved", "keep_output": [false], "energy": 13.6, "tags": [], "fragment": "import FWCore.ParameterSet.Config as cms\n\nexternalLHEProducer = cms.EDProducer('ExternalLHEProducer',\n    args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/PdmV/RunIII2024Summer24/Powheg/TT/hvq_slc7_amd64_gcc10_CMSSW_12_4_8_TTtoLNu2Q_powheg-pythia8.tgz'),\n    nEvents = cms.untracked.uint32(5000),\n    numberOfParameters = cms.uint32(1),\n    outputFile = cms.string('cmsgrid_final.lhe'),\n    scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh'),\n    generateConcurrently = cms.untracked.bool(True),\n)\n\nfrom Configuration.Generator.Pythia8CommonSettings_cfi import *\nfrom Configuration.Generator.MCTunesRun3ECM13p6TeV.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.Pythia8PowhegEmissionVetoSettings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n\ngenerator = cms.EDFilter(\"Pythia8ConcurrentHadronizerFilter\",\n    PythiaParameters = cms.PSet(\n        pythia8CommonSettingsBlock,\n        pythia8CP5SettingsBlock,\n        pythia8PowhegEmissionVetoSettingsBlock,\n        pythia8PSweightsSettingsBlock,\n        processParameters = cms.vstring(\n        'POWHEG:nFinal = 2',\n        'TimeShower:mMaxGamma = 4.0'\n        ),\n        parameterSets = cms.vstring(\n            'pythia8CommonSettings',\n            'pythia8CP5Settings',\n            'pythia8PowhegEmissionVetoSettings',\n            'processParameters',\n            'pythia8PSweightsSettings',\n        )\n    ),\n    comEnergy = cms.double(13600),\n    maxEventsToPrint = cms.untracked.int32(1),\n    pythiaHepMCVerbosity = cms.untracked.bool(False),\n    pythiaPylistVerbosity = cms.untracked.int32(1),\n)\n\n", "time_event": [22.18322274769774], "pwg": "GEN", "reqmgr_name": [], "approval": "approve", "name_of_fragment": "", "pileup_dataset_name": "", "analysis_id": [], "input_dataset": "", "member_of_chain": [], "prepid": "GEN-RunIII2024Summer24wmLHEGS-00001", "extension": 0, "size_event": [878], "notes": "", "completed_events": -1, "history": [{"action": "created", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "clone", "step": "GEN-Run3Summer23wmLHEGS-00009", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-15-15-58", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "step": "fragment, total_events", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-15-16-11", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-15-16-15", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "approve", "step": "validation", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-15-16-15", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "validation", "step": "failed", "updater": {"author_email": "", "submission_date": "2024-11-15-16-41", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "validation", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-19-20-06", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "validation", "step": "succeeded", "updater": {"author_email": "", "submission_date": "2024-11-19-22-01", "author_username": "automatic", "author_name": ""}}, {"action": "approve", "step": "define", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-20-16-08", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "set status", "step": "defined", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-20-16-08", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "update", "step": "total_events", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-20-16-09", "author_username": "shjeon", "author_name": "Si Hyun Jeon"}}, {"action": "approve", "step": "approve", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-20-16-37", "author_username": "lviliani", "author_name": "Lorenzo Viliani"}}, {"action": "set status", "step": "approved", "updater": {"author_email": "[email protected]", "submission_date": "2024-11-20-16-37", "author_username": "lviliani", "author_name": "Lorenzo Viliani"}}], "output_dataset": [], "member_of_campaign": "RunIII2024Summer24wmLHEGS", "_id": "GEN-RunIII2024Summer24wmLHEGS-00001", "dataset_name": "TTtoLNu2Q_TuneCP5_13p6TeV_powheg-pythia8", "cadi_line": ""}}