From 87dde1e3b95084488c023d368746a73e4c796b4e Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Fri, 7 Jun 2024 19:27:35 -0700 Subject: [PATCH 1/8] Changes to make standalone run also from within a CMSSW area --- .../LSTCore/src/alpaka/LSTESData.dev.cc | 10 +++++--- RecoTracker/LSTCore/standalone/setup.sh | 25 +++++++++++-------- RecoTracker/LSTCore/standalone/setup_hpg.sh | 18 ++++++++----- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc index 616c6f4bf3894..81a640cf15f26 100644 --- a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc +++ b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc @@ -17,9 +17,6 @@ namespace { std::string path_str; if (path_lst_base != nullptr) { path_str = path_lst_base; - } else if (path_tracklooperdir != nullptr) { - path_str = path_tracklooperdir; - path_str += "/../"; } else { std::stringstream search_path(std::getenv("CMSSW_SEARCH_PATH")); std::string path; @@ -29,7 +26,12 @@ namespace { break; } } - path_str += "/RecoTracker/LSTCore"; + if (path_str.empty()) { + path_str = path_tracklooperdir; + path_str += "/.."; + } else { + path_str += "/RecoTracker/LSTCore"; + } } return path_str; } diff --git a/RecoTracker/LSTCore/standalone/setup.sh b/RecoTracker/LSTCore/standalone/setup.sh index 1730e1b136b8b..37a58c50e9c35 100644 --- a/RecoTracker/LSTCore/standalone/setup.sh +++ b/RecoTracker/LSTCore/standalone/setup.sh @@ -7,18 +7,23 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh ARCH=$(uname -m) -if [[ $(hostname) == *lnx4555* ]]; then - export SCRAM_ARCH=el9_amd64_gcc12 -elif [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then - export SCRAM_ARCH=el9_aarch64_gcc12 +if [ -z ${CMSSW_SEARCH_PATH+x} ]; then + if [[ $(hostname) == *lnx4555* ]]; then + export SCRAM_ARCH=el9_amd64_gcc12 + elif [[ $ARCH == "aarch64" || $ARCH == "arm64" ]]; then + export SCRAM_ARCH=el9_aarch64_gcc12 + else + export SCRAM_ARCH=el8_amd64_gcc12 + fi + export CMSSW_VERSION=CMSSW_14_1_0_pre3 + + source /cvmfs/cms.cern.ch/cmsset_default.sh + cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src + eval `scramv1 runtime -sh` else - export SCRAM_ARCH=el8_amd64_gcc12 + cd $CMSSW_BASE/src + eval `scramv1 runtime -sh` fi -export CMSSW_VERSION=CMSSW_14_1_0_pre3 - -source /cvmfs/cms.cern.ch/cmsset_default.sh -cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src -eval `scramv1 runtime -sh` # Export paths to libraries we need export BOOST_ROOT=$(scram tool info boost | grep BOOST_BASE | cut -d'=' -f2) diff --git a/RecoTracker/LSTCore/standalone/setup_hpg.sh b/RecoTracker/LSTCore/standalone/setup_hpg.sh index 8b2bd7f623407..46eee895ca286 100644 --- a/RecoTracker/LSTCore/standalone/setup_hpg.sh +++ b/RecoTracker/LSTCore/standalone/setup_hpg.sh @@ -11,13 +11,19 @@ module load cuda/11.4.3 git DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh -export SCRAM_ARCH=el8_amd64_gcc12 -export CMSSW_VERSION=CMSSW_14_1_0_pre3 -export CUDA_HOME=${HPC_CUDA_DIR} +if [ -z ${CMSSW_SEARCH_PATH+x} ]; then + export SCRAM_ARCH=el8_amd64_gcc12 + export CMSSW_VERSION=CMSSW_14_1_0_pre3 + + source /cvmfs/cms.cern.ch/cmsset_default.sh + cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src + eval `scramv1 runtime -sh` +else + cd $CMSSW_BASE/src + eval `scramv1 runtime -sh` +fi -source /cvmfs/cms.cern.ch/cmsset_default.sh -cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src -eval `scramv1 runtime -sh` +export CUDA_HOME=${HPC_CUDA_DIR} # Export paths to libraries we need export BOOST_ROOT=$(scram tool info boost | grep BOOST_BASE | cut -d'=' -f2) From d1286b03a4adcbebab24ccb8a8667f6f0f8f1971 Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Mon, 10 Jun 2024 17:02:13 -0700 Subject: [PATCH 2/8] Removal of LST_BASE and redudant setup lines --- .../LSTCore/src/alpaka/LSTESData.dev.cc | 33 +++++++++---------- RecoTracker/LSTCore/standalone/setup.sh | 1 - RecoTracker/LSTCore/standalone/setup_hpg.sh | 1 - 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc index 81a640cf15f26..21f82c3e90578 100644 --- a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc +++ b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc @@ -12,27 +12,24 @@ namespace { std::string trackLooperDir() { - const char* path_lst_base = std::getenv("LST_BASE"); + std::string path_str, path; const char* path_tracklooperdir = std::getenv("TRACKLOOPERDIR"); - std::string path_str; - if (path_lst_base != nullptr) { - path_str = path_lst_base; - } else { - std::stringstream search_path(std::getenv("CMSSW_SEARCH_PATH")); - std::string path; - while (std::getline(search_path, path, ':')) { - if (std::filesystem::exists(path + "/RecoTracker/LSTCore/data")) { - path_str = path; - break; - } - } - if (path_str.empty()) { - path_str = path_tracklooperdir; - path_str += "/.."; - } else { - path_str += "/RecoTracker/LSTCore"; + std::stringstream search_path(std::getenv("CMSSW_SEARCH_PATH")); + + while (std::getline(search_path, path, ':')) { + if (std::filesystem::exists(path + "/RecoTracker/LSTCore/data")) { + path_str = path; + break; } } + + if (path_str.empty()) { + path_str = path_tracklooperdir; + path_str += "/.."; + } else { + path_str += "/RecoTracker/LSTCore"; + } + return path_str; } diff --git a/RecoTracker/LSTCore/standalone/setup.sh b/RecoTracker/LSTCore/standalone/setup.sh index 37a58c50e9c35..e114e7550a7bd 100644 --- a/RecoTracker/LSTCore/standalone/setup.sh +++ b/RecoTracker/LSTCore/standalone/setup.sh @@ -22,7 +22,6 @@ if [ -z ${CMSSW_SEARCH_PATH+x} ]; then eval `scramv1 runtime -sh` else cd $CMSSW_BASE/src - eval `scramv1 runtime -sh` fi # Export paths to libraries we need diff --git a/RecoTracker/LSTCore/standalone/setup_hpg.sh b/RecoTracker/LSTCore/standalone/setup_hpg.sh index 46eee895ca286..3a39e49edea35 100644 --- a/RecoTracker/LSTCore/standalone/setup_hpg.sh +++ b/RecoTracker/LSTCore/standalone/setup_hpg.sh @@ -20,7 +20,6 @@ if [ -z ${CMSSW_SEARCH_PATH+x} ]; then eval `scramv1 runtime -sh` else cd $CMSSW_BASE/src - eval `scramv1 runtime -sh` fi export CUDA_HOME=${HPC_CUDA_DIR} From f529d3f6d6e9b93b28a6f3aaaad46055a76f2b17 Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Tue, 11 Jun 2024 22:56:01 +0200 Subject: [PATCH 3/8] Update standalone README.md with up-to-date instructions --- RecoTracker/LSTCore/standalone/README.md | 272 ++++++++--------------- 1 file changed, 96 insertions(+), 176 deletions(-) diff --git a/RecoTracker/LSTCore/standalone/README.md b/RecoTracker/LSTCore/standalone/README.md index 02fbef943f697..0337f67f7fbfa 100644 --- a/RecoTracker/LSTCore/standalone/README.md +++ b/RecoTracker/LSTCore/standalone/README.md @@ -1,37 +1,99 @@ -# TrackLooper +# How to set up standalone LST - -## Quick Start - - -### Setting up LSTPerformanceWeb (only for lnx7188 and lnx4555) +## Setting up LSTPerformanceWeb (only for lnx7188 and lnx4555) For lnx7188 and lnx4555 this needs to be done once cd /cdat/tem/${USER}/ git clone git@github.com:SegmentLinking/LSTPerformanceWeb.git -### Setting up container (only for lnx7188) +## Setting up container (only for lnx7188) For lnx7188 this needs to be done before compiling or running the code: singularity shell --nv --bind /mnt/data1:/data --bind /data2/segmentlinking/ --bind /opt --bind /nfs --bind /mnt --bind /usr/local/cuda/bin/ --bind /cvmfs /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el8:x86_64 -### Setting up the code +## Setting up LST + +There are two way to set up LST as a standalone, either by setting up a full CMSSW area, which provides a unified setup for standalone and CMSSW tests, or by `sparse-checkout` only the relevant package and using them independent of CMSSW. A CVMFS-less setup is also provided for the second option. + +### Setting up LST within CMSSW (preferred option) + +```bash +CMSSW_VERSION=CMSSW_14_1_0_pre3 # Change with latest/preferred CMSSW version +LST_BRANCH=CMSSW_14_1_0_pre3_LST_X_LSTCore_realfiles # Change to the development branch +cmsrel ${CMSSW_VERSION} +cd ${CMSSW_VERSION}/src/ +cmsenv +git cms-init +git remote add SegLink git@github.com:SegmentLinking/cmssw.git +git fetch ${LST_BRANCH} +git checkout ${LST_BRANCH} +git cms-addpkg RecoTracker/LST RecoTracker/LSTCore +scram b -j 12 +cd RecoTracker/LSTCore/standalone +# Source one of the commands below, depending on the site +source setup.sh # if on UCSD or Cornell +source setup_hpg.sh # if on Florida +``` + +### Setting up LST outside of CMSSW + +For this setup, dependencies are still provided from CMSSW through CVMFS (see below for an even more independent setup) but no CMSSW area is setup. This is done by running the following commands. + +``` bash +LST_BRANCH=CMSSW_14_1_0_pre3_LST_X_LSTCore_realfiles # Change to the development branch +git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch ${LST_BRANCH} https://github.com/SegmentLinking/cmssw.git TrackLooper +cd TrackLooper +git sparse-checkout add HeterogeneousCore/AlpakaInterface RecoTracker/LSTCore +git checkout +cd RecoTracker/LSTCore/standalone/ +# Source one of the commands below, depending on the site +source setup.sh # if on UCSD or Cornell +source setup_hpg.sh # if on Florida +``` + +#### Running LST in a CVMFS-less setup + +The setup scripts included in this repository assume that the [CernVM File System (CVMFS)](https://cernvm.cern.ch/fs/) is installed. This provides a convenient way to fetch the required dependencies, but it is not necessary to run LST in standalone mode. Here, we briefly describe how to build and run it when CVMFS is not available. + +The necessary dependencies are CUDA, ROOT, the Boost libraries, Alpaka, and some CMSSW headers. CUDA, ROOT, and Boost, are fairly standard libraries and are available from multiple package managers. For the remaining necessary headers you will need to clone the [Alpaka](https://github.com/alpaka-group/alpaka) and [CMSSW](https://github.com/cms-sw/cmssw) repositories (since the CMSSW repository is extremely large, the instructions just above need to be followed to checkout the strictly required part of it). + +Then all that is left to do is set some environment variables. We give an example of how to do this in lnx7188/cgpu-1. - git clone git@github.com:SegmentLinking/TrackLooper.git - cd TrackLooper/ - # Source one of the commands below, depending on the site - source setup.sh # if on UCSD or Cornell - source setup_hpg.sh # if on Florida +```bash +# These two lines are only needed to set the right version of gcc and nvcc. They are not needed for standard installations. +export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/external/el8_amd64_gcc12/bin:$PATH +export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/biglib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/lib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/external/el8_amd64_gcc12/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib:$LD_LIBRARY_PATH -### Running the code +# These are the lines that you need to manually change for a CVMFS-less setup. +# In this example we use cvmfs paths since that is where the dependencies are in lnx7188/cgpu1, but they can point to local directories. +export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/boost/1.80.0-60a217837b5db1cff00c7d88ec42f53a +export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/alpaka/1.1.0-7d0324257db47fde2d27987e7ff98fb4 +export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/cuda/12.4.1-06cde0cd9f95a73a1ea05c8535f60bde +export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/lcg/root/6.30.07-21947a33e64ceb827a089697ad72e468 +export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3 + +# These lines are needed to account for some extra environment variables that are exported in the setup script. +export LD_LIBRARY_PATH=$PWD/SDL/cuda:$PWD/SDL/cpu:$PWD:$LD_LIBRARY_PATH +export PATH=$PWD/bin:$PATH +export PATH=$PWD/efficiency/bin:$PATH +export PATH=$PWD/efficiency/python:$PATH +export TRACKLOOPERDIR=$PWD +export TRACKINGNTUPLEDIR=/data2/segmentlinking/CMSSW_12_2_0_pre2/ +export LSTOUTPUTDIR=. +source $PWD/code/rooutil/thisrooutil.sh + +# After this, you can compile and run LST as usual, as described below. +``` + +## Running the code sdl_make_tracklooper -mc sdl_ -i PU200 -o LSTNtuple.root createPerfNumDenHists -i LSTNtuple.root -o LSTNumDen.root - lst_plot_performance.py LSTNumDen.root -t "myTag" - # python3 efficiency/python/lst_plot_performance.py LSTNumDen.root -t "myTag" # if you are on cgpu-1 or Cornell + lst_plot_performance.py LSTNumDen.root -t "myTag" # or + python3 efficiency/python/lst_plot_performance.py LSTNumDen.root -t "myTag" # if you are on cgpu-1 or Cornell The above can be even simplified @@ -41,7 +103,7 @@ The `-f` flag can be omitted when the code has already been compiled. If multipl sdl_run -b cpu -s PU200 -n -1 -t myTag -## Command explanations +### Command explanations Compile the code with option flags. If none of `C,G,R,A` are used, then it defaults to compiling for CUDA and CPU. @@ -106,181 +168,39 @@ Comparing two different runs -t "mywork" \ --compare -## CMSSW Integration -This is the a complete set of instruction on how the TrackLooper code -can be linked as an external tool in CMSSW: +## Run the LST reconstruction in CMSSW (read to the end, before running) -### Build TrackLooper -```bash -git clone git@github.com:SegmentLinking/TrackLooper.git -cd TrackLooper/ -# Source one of the commands below, depending on the site -source setup.sh # if on UCSD or Cornell -source setup_hpg.sh # if on Florida -sdl_make_tracklooper -mc -cd .. -``` +Two complete workflows have been implemented within CMSSW to run a two-iteration, tracking-only reconstruction with LST: + - 24834.703 (CPU) + - 24834.704 (GPU) -### Set up `TrackLooper` as an external -```bash -mkdir workingFolder # Create the folder you will be working in -cd workingFolder -cmsrel CMSSW_14_1_0_pre3 -cd CMSSW_14_1_0_pre3/src -cmsenv -git cms-init -git remote add SegLink git@github.com:SegmentLinking/cmssw.git -git fetch SegLink CMSSW_14_1_0_pre3_LST_X -git cms-addpkg RecoTracker Configuration -git checkout CMSSW_14_1_0_pre3_LST_X -#To include both the CPU library and GPU library into CMSSW, create 3 xml files (headers file has no library). -#Before writing the following xml file, check that libsdl_cpu.so and libsdl_gpu.so can be found under the ../../../TrackLooper/SDL/ folder. -cat <lst_headers.xml - - - - - - - -EOF -cat <lst_cpu.xml - - - - - - - - - -EOF -cat <lst_cuda.xml - - - - - - - - - -EOF -scram setup lst_headers.xml -scram setup lst_cpu.xml -scram setup lst_cuda.xml -cmsenv -git cms-checkdeps -a -A -scram b -j 12 -``` +We will use the second one in the example below. To get the commands of this workflow, one can run: -### Run the LST reconstruction in CMSSW -A simple test configuration of the LST reconstruction can be run with the command: -```bash -cmsRun RecoTracker/LST/test/LSTAlpakaTester.py -``` - -For a more complete workflow, one can run a modified version of the 21034.1 workflow. -To get the commands of this workflow, one can run: -```bash -runTheMatrix.py -w upgrade -n -e -l 21034.1 -``` + runTheMatrix.py -w upgrade -n -e -l 24834.704 For convenience, the workflow has been run for 100 events and the output is stored here: -```bash -/data2/segmentlinking/CMSSW_14_1_0_pre0/step2_21034.1_100Events.root -``` -For enabling the LST reconstruction in the CMSSW tracking workflow, a modified step3 needs to be run. -This is based on the step3 command of the 21034.1 workflow with the following changes: - - Remove the `--pileup_input` and `--pileup` flags. - - The number of threads and streams for the job can be optionally controlled by the `--nThreads` and `--nStreams` command line options respectively (`1` ends up being the actual default value for both, and more info can be found by running `cmsDriver.py --help`). - - Add at the end of the command: `--procModifiers gpu,trackingLST,trackingIters01 --no_exec` - -Run the command and modify the output configuration file with the following: - - If want to run a cpu version, remove the ```gpu``` in the line defining the `process` object: - ```python - process = cms.Process('RECO',...,gpu,...) - ``` - - Add the following lines below the part where the import of the standard configurations happens: - ```python - process.load('Configuration.StandardSequences.Accelerators_cff') - process.load("HeterogeneousCore.AlpakaCore.ProcessAcceleratorAlpaka_cfi") - ``` - - Modify the input and output file names accordingly, as well as the number of events. - -Then, run the configuration file with `cmsRun`. - -To get the DQM files, one would have to run step4 of the 21034.1 workflow with the following modifications: - - Add `--no_exec` to the end of command and then run it. - - Modify the output configuration file by changing the input file (the one containing `inDQM` from the previous step) and number of events accordingly. + /data2/segmentlinking/CMSSW_14_1_0_pre0/step2_21034.1_100Events.root + +The input files in each step may need to be properly adjusted to match the ones produced by the previous step/provided externally, hence it is better to run the commands with the `--no_exec` option included. Running the configuration file with `cmsRun`, the output file will have a name starting with `DQM`. The name is the same every time this step runs, -so it is good practice to rename the file, e.g. to `tracking_Iters01LST.root`. +so it is good practice to rename the file, e.g. to `step4_24834.704.root`. The MTV plots can be produced with the command: -```bash -makeTrackValidationPlots.py --extended tracking_Iters01LST.root -``` -Comparison plots can be made by including multiple ROOT files as arguments. -**Note:** In case one wants to run step2 as well, similar modifications as in step4 (`--no_exec` flag and input file/number of events) need to be applied. Moreover, the PU files have better be modified to point to local ones. This can be done by inserting a dummy file when running the command (set the argument of the `--pileup_input` flag to `file:file.root`), and then change the PU input files in the configuration to the following line (by means of replacing the corresponding line in the configuration): -```python -process.mix.input.fileNames = cms.untracked.vstring(['file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/066fc95d-1cef-4469-9e08-3913973cd4ce.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/07928a25-231b-450d-9d17-e20e751323a1.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/26bd8fb0-575e-4201-b657-94cdcb633045.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/4206a9c5-44c2-45a5-aab2-1a8a6043a08a.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/55a372bf-a234-4111-8ce0-ead6157a1810.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/59ad346c-f405-4288-96d7-795f81c43fe8.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/7280f5ec-b71d-4579-a730-7ce2de0ff906.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/b93adc85-715f-477a-afc9-65f3241933ee.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/c7a0aa46-f55c-4b01-977f-34a397b71fba.root', 'file:/data2/segmentlinking/PUSamplesForCMSSW1263/CMSSW_12_3_0_pre5/RelValMinBias_14TeV/GEN-SIM/123X_mcRun4_realistic_v4_2026D88noPU-v1/e77fa467-97cb-4943-884f-6965b4eb0390.root']) -``` - -### Inclusion of LST in other CMSSW packages -Including the line -``` - -``` -in the relevant package `BuildFile.xml` allows for -including our headers in the code of that package. + makeTrackValidationPlots.py --extended step4_24834.704.root -## Running LST in a CVMFS-less setup +Comparison plots can be made by including multiple ROOT files as arguments. -The setup scripts included in this repository assume that the [CernVM File System (CVMFS)](https://cernvm.cern.ch/fs/) is installed. This provides a convenient way to fetch the required dependencies, but it is not necessary to run LST in standalone mode. Here, we briefly describe how to build and run it when CVMFS is not available. +## Code formatting and checking -The necessary dependencies are CUDA, ROOT, the Boost libraries, Alpaka, and some CMSSW headers. CUDA, ROOT, and Boost, are fairly standard libraries and are available from multiple package managers. For the remaining necessary headers you will need to clone the [Alpaka](https://github.com/alpaka-group/alpaka) and [CMSSW](https://github.com/cms-sw/cmssw) repositories. The Alpaka repository is reasonably sized, but the CMSSW one extremely large, especially considering that we only need a tiny fraction of its files to build LST. We can get only the Alpaka interface headers from CMSSW by running the following commands. +Using the first setup option above, it is prefered to run the checks provided by CMSSW using the following commands. -``` bash -git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch CMSSW_14_1_X https://github.com/cms-sw/cmssw.git -cd cmssw -git sparse-checkout add HeterogeneousCore/AlpakaInterface -git checkout ``` - -Then all that is left to do is set some environment variables. We give an example of how to do this in lnx7188/cgpu-1. - -```bash -# These two lines are only needed to set the right version of gcc and nvcc. They are not needed for standard installations. -export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/external/el8_amd64_gcc12/bin:$PATH -export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/biglib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/lib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3/external/el8_amd64_gcc12/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib:$LD_LIBRARY_PATH - -# These are the lines that you need to manually change for a CVMFS-less setup. -# In this example we use cvmfs paths since that is where the dependencies are in lnx7188/cgpu1, but they can point to local directories. -export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/boost/1.80.0-60a217837b5db1cff00c7d88ec42f53a -export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/alpaka/1.1.0-7d0324257db47fde2d27987e7ff98fb4 -export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/cuda/12.4.1-06cde0cd9f95a73a1ea05c8535f60bde -export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/lcg/root/6.30.07-21947a33e64ceb827a089697ad72e468 -export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre3 - -# These lines are needed to account for some extra environment variables that are exported in the setup script. -export LD_LIBRARY_PATH=$PWD/SDL/cuda:$PWD/SDL/cpu:$PWD:$LD_LIBRARY_PATH -export PATH=$PWD/bin:$PATH -export PATH=$PWD/efficiency/bin:$PATH -export PATH=$PWD/efficiency/python:$PATH -export TRACKLOOPERDIR=$PWD -export TRACKINGNTUPLEDIR=/data2/segmentlinking/CMSSW_12_2_0_pre2/ -export LSTOUTPUTDIR=. -source $PWD/code/rooutil/thisrooutil.sh - -# After this, you can compile and run LST as usual. -sdl_run -f -mc -s PU200 -n -1 -t myTag +scram b -j 12 code-checks >& c.log && scram b -j 12 code-format >& f.log ``` -## Code formatting and checking - -The makefile in the `SDL` directory includes phony targets to run `clang-format` and `clang-tidy` on the code using the formatting and checks used in CMSSW. The following are the available commands. +In any case, the makefile in the `standalone` directory also includes phony targets to run `clang-format` and `clang-tidy` on the code using the formatting and checks used in CMSSW. The following are the available commands. - `make format` Formats the code in the `SDL` directory using `clang-format` following the rules specified in `.clang-format`. @@ -288,4 +208,4 @@ The makefile in the `SDL` directory includes phony targets to run `clang-format` Runs `clang-tidy` on the code in the `SDL` directory to performs the checks specified in `.clang-tidy`. - `make check-fix` Same as `make check`, but fixes the issues that it knows how to fix. - \ No newline at end of file + From 1d4b2aa42522ffb3f645a1aaac53bb11b27599bd Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Wed, 12 Jun 2024 11:31:25 +0200 Subject: [PATCH 4/8] Comment out the CVMFS-less setup (for now) --- RecoTracker/LSTCore/standalone/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RecoTracker/LSTCore/standalone/README.md b/RecoTracker/LSTCore/standalone/README.md index 0337f67f7fbfa..0b9031b8b3077 100644 --- a/RecoTracker/LSTCore/standalone/README.md +++ b/RecoTracker/LSTCore/standalone/README.md @@ -53,6 +53,7 @@ source setup.sh # if on UCSD or Cornell source setup_hpg.sh # if on Florida ``` + ## Running the code From 96ec1b8c003d99e171dbd8d81cb7c560ea95db82 Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Wed, 12 Jun 2024 15:51:20 +0200 Subject: [PATCH 5/8] Delete deprecated code checks and format from README --- RecoTracker/LSTCore/standalone/README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/RecoTracker/LSTCore/standalone/README.md b/RecoTracker/LSTCore/standalone/README.md index 0b9031b8b3077..279a7556d9f09 100644 --- a/RecoTracker/LSTCore/standalone/README.md +++ b/RecoTracker/LSTCore/standalone/README.md @@ -201,13 +201,3 @@ Using the first setup option above, it is prefered to run the checks provided by ``` scram b -j 12 code-checks >& c.log && scram b -j 12 code-format >& f.log ``` - -In any case, the makefile in the `standalone` directory also includes phony targets to run `clang-format` and `clang-tidy` on the code using the formatting and checks used in CMSSW. The following are the available commands. - -- `make format` - Formats the code in the `SDL` directory using `clang-format` following the rules specified in `.clang-format`. -- `make check` - Runs `clang-tidy` on the code in the `SDL` directory to performs the checks specified in `.clang-tidy`. -- `make check-fix` - Same as `make check`, but fixes the issues that it knows how to fix. - From 2e0c60881ce4eb98ad6db2f2e85decafe733d9d1 Mon Sep 17 00:00:00 2001 From: Manos Vourliotis Date: Wed, 12 Jun 2024 17:03:50 +0200 Subject: [PATCH 6/8] Include cloning of LST data files in README --- RecoTracker/LSTCore/standalone/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/RecoTracker/LSTCore/standalone/README.md b/RecoTracker/LSTCore/standalone/README.md index 279a7556d9f09..23ddb4d0a8a3f 100644 --- a/RecoTracker/LSTCore/standalone/README.md +++ b/RecoTracker/LSTCore/standalone/README.md @@ -37,6 +37,15 @@ source setup.sh # if on UCSD or Cornell source setup_hpg.sh # if on Florida ``` +Currently, the data files for LST need to be copied manually. This is done by running: + +```bash +mkdir -p $CMSSW_BASE/external/$SCRAM_ARCH/data/RecoTracker/LSTCore/ +cd $CMSSW_BASE/external/$SCRAM_ARCH/data/RecoTracker/LSTCore/ +git clone git@github.com:cms-data/RecoTracker-LSTCore.git data +cd - +``` + ### Setting up LST outside of CMSSW For this setup, dependencies are still provided from CMSSW through CVMFS (see below for an even more independent setup) but no CMSSW area is setup. This is done by running the following commands. @@ -53,6 +62,12 @@ source setup.sh # if on UCSD or Cornell source setup_hpg.sh # if on Florida ``` +Currently, the data files for LST need to be copied manually. This is done by running: + +```bash +git clone git@github.com:cms-data/RecoTracker-LSTCore.git data +``` +