Merge branch 'IB/CMSSW_14_1_X/master' into Rivet4

cms-sw · Apr 17, 2024 · 6863d1f · 6863d1f
2 parents 256c88c + dcee9d7
commit 6863d1f
Show file tree

Hide file tree

Showing 24 changed files with 135 additions and 39 deletions.
diff --git a/SCRAMV1.spec b/SCRAMV1.spec
@@ -1,8 +1,8 @@
-### RPM lcg SCRAMV1 V3_00_67
+### RPM lcg SCRAMV1 V3_00_68
 ## NOCOMPILER
 ## NO_VERSION_SUFFIX
 
-%define tag 85a81e2bc84e2af73ae747a89a514b541e14a0ca
+%define tag 15a8797db3180ba786e6e131a9c4e3011b53bc68
 %define branch SCRAMV3
 %define github_user cms-sw
 Source: git+https://github.com/%{github_user}/SCRAM.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz

diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec
@@ -184,6 +184,7 @@ Requires: xgboost
 Requires: pytorch
 Requires: pytorch-custom-ops
 
+## INCLUDE tfaot-models
 ## INCLUDE cmssw-vectorization
 ## INCLUDE cmssw-drop-tools
 ## INCLUDE scram-tool-conf
diff --git a/crab-dev.spec b/crab-dev.spec
@@ -2,9 +2,9 @@
 #For new crabclient_version, set the version_suffix to 00
 #For any other change, increment version_suffix
 ##########################################
-%define version_suffix 01
-%define crabclient_version v3.240110
+%define version_suffix 00
+%define crabclient_version v3.240416
 ### RPM cms crab-dev %{crabclient_version}.%{version_suffix}
-%define crabserver_version v3.231215
+%define crabserver_version v3.240325
 
 ## IMPORT crab-build
diff --git a/crab-pre.spec b/crab-pre.spec
@@ -2,9 +2,9 @@
 #For new crabclient_version, set the version_suffix to 00
 #For any other change, increment version_suffix
 ##########################################
-%define version_suffix 01
-%define crabclient_version v3.231010
+%define version_suffix 00
+%define crabclient_version v3.240110
 ### RPM cms crab-pre %{crabclient_version}.%{version_suffix}
-%define crabserver_version v3.231006
+%define crabserver_version v3.231215
 
 ## IMPORT crab-build
diff --git a/crab-prod.spec b/crab-prod.spec
@@ -2,9 +2,9 @@
 #For new crabclient_version, set the version_suffix to 00
 #For any other change, increment version_suffix
 ##########################################
-%define version_suffix 01
-%define crabclient_version v3.240110
+%define version_suffix 00
+%define crabclient_version v3.240404
 ### RPM cms crab-prod %{crabclient_version}.%{version_suffix}
-%define crabserver_version v3.231215
+%define crabserver_version v3.240325
 
 ## IMPORT crab-build
diff --git a/cuda.spec b/cuda.spec
@@ -1,8 +1,8 @@
-### RPM external cuda 12.4.0
+### RPM external cuda 12.4.1
 ## INITENV +PATH LD_LIBRARY_PATH %i/lib64
 
 %define runpath_opts -m compute-sanitizer -m drivers -m nvvm
-%define driversversion 550.54.14
+%define driversversion 550.54.15
 
 %ifarch x86_64
 Source0: https://developer.download.nvidia.com/compute/cuda/%{realversion}/local_installers/%{n}_%{realversion}_%{driversversion}_linux.run

diff --git a/data/cmsswdata.txt b/data/cmsswdata.txt
@@ -3,9 +3,10 @@
 #Once a non-default section is empty then cleanup that section and remove its cmsdist/${PACKAGE_TYPE}.file
 #If there is no customization for the packae then remove its .spec and .file
 [default]
-RecoEgamma-PhotonIdentification=V01-07-00
+RecoBTag-Combined=V01-22-00
+RecoEgamma-PhotonIdentification=V01-08-00
+PhysicsTools-NanoAOD=V01-04-00
 Validation-HGCalValidation=V00-06-00
-RecoBTag-Combined=V01-21-00
 L1Trigger-L1TMuonEndCapPhase2=V00-01-00
 L1Trigger-L1TTrackMatch=V00-01-00
 L1Trigger-VertexFinder=V00-01-00
@@ -47,7 +48,6 @@ DataFormats-Common=V00-01-00
 DQM-EcalMonitorClient=V00-03-00
 CondTools-Hcal=V00-01-00
 RecoTracker-DisplacedRegionalTracking=V00-01-00
-PhysicsTools-NanoAOD=V01-03-00
 CalibTracker-SiStripDCS=V01-01-00
 L1Trigger-TrackFindingTracklet=V00-03-00
 Alignment-OfflineValidation=V00-03-00

diff --git a/pip/cms-tfaot.file b/pip/cms-tfaot.file
@@ -0,0 +1,12 @@
+BuildRequires: py3-pip py3-setuptools py3-wheel
+Requires: py3-PyYAML py3-cmsml
+
+%define github_user cms-externals
+%define tag df4bfd35dfed51d7c6ff0367ebc78093dfcc7d1d
+%define branch master
+%define source0 git+https://github.com/%{github_user}/cms-tfaot.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz
+
+# copy test models
+%define PipPostInstall \
+  mkdir -p %{i}/share; \
+  cp -r cmsdist-tmp/pip-req-build-*/test_models %{i}/share/
diff --git a/pip/requirements.txt b/pip/requirements.txt
@@ -56,7 +56,8 @@ charset-normalizer==3.1.0
 cleo==2.0.1
 click==8.1.3
 clikit==0.6.2
-cmsml==0.2.2
+cmsml==0.2.5
+cms-tfaot==1.0.1
 contourpy==1.0.7
 correctionlib==2.2.2
 crashtest==0.4.1

diff --git a/python_tools.spec b/python_tools.spec
@@ -12,6 +12,7 @@ Requires: py3-keras
 Requires: py3-scikit-learn
 #save for the end
 Requires: py3-tensorflow
+Requires: py3-cms-tfaot
 Requires: py3-cmsml
 Requires: py3-law
 Requires: py3-protobuf

diff --git a/rivet.spec b/rivet.spec
@@ -3,6 +3,7 @@
 ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
 ## OLD GENSER Source: http://cern.ch/service-spi/external/MCGenerators/distribution/rivet/rivet-%{realversion}-src.tgz
 Source: git+https://gitlab.com/hepcedar/rivet.git?obj=master/%{n}-%{realversion}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz
+Source99: scram-tools.file/tools/eigen/env
 Patch0: rivet-duplicate-libs
 
 Requires: hepmc3 fastjet fastjet-contrib yoda hdf5 highfive onnxruntime
@@ -13,6 +14,8 @@ BuildRequires: python3 py3-cython autotools
 %setup -n %{n}-%{realversion}
 %patch0 -p1
 
+%build
+source %{_sourcedir}/env
 # Update config.{guess,sub} to detect aarch64 and ppc64le
 rm -f %{_tmppath}/config.{sub,guess}
 %get_config_guess %{_tmppath}/config.guess
@@ -36,7 +39,7 @@ autoreconf -fiv
 %ifarch aarch64
 sed -i -e 's|^ax_openmp_flags=".*"|ax_openmp_flags="none"|' ./configure
 %endif
-CXXFLAGS="-std=c++%{cms_cxx_standard}"
+CXXFLAGS="-std=c++%{cms_cxx_standard} $CMS_EIGEN_CXX_FLAGS"
 %ifarch x86_64
     CXXFLAGS="${CXXFLAGS} -msse3"
 %endif
@@ -51,8 +54,8 @@ PYTHON=$(which python3) \
 # The following hack insures that the bins with the library linked explicitly
 # rather than indirectly, as required by the gold linker
 perl -p -i -e "s|LIBS = $|LIBS = -lHepMC3|g" bin/Makefile
-%build
-make %{makeprocesses} all 
+make %{makeprocesses} all
+
 %install
 make install 
 sed -i -e 's|^#!.*python.*|#!/usr/bin/env python3|' %{i}/bin/*

diff --git a/root.spec b/root.spec
@@ -1,10 +1,10 @@
-### RPM lcg root 6.30.05
+### RPM lcg root 6.30.07
 ## INITENV +PATH PYTHON3PATH %{i}/lib
 ## INITENV SET ROOTSYS %{i}
 ## INCLUDE compilation_flags
 ## INCLUDE cpp-standard
-%define tag e642ea24e23b7e0c168dfa4dbca3ed6bf4fb8273
-%define branch cms/v6-30-00-patches/bd1b2a6bc8
+%define tag 5ccdd10f224eef86cef23d2c0fcad6f723bbddba
+%define branch cms/v6-30-00-patches/9be8ca1756
 
 %define github_user cms-sw
 Source: git+https://github.com/%{github_user}/root.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz

diff --git a/scram-project-build.file b/scram-project-build.file
@@ -55,7 +55,7 @@ BuildRequires: dwz
 %endif
 
 %if "%{?configtag:set}" != "set"
-%define configtag       V09-03-05
+%define configtag       V09-04-01
 %endif
 
 %if "%{?buildarch:set}" != "set"

diff --git a/scram-tools.file/tools/alpaka/alpaka-cuda.xml b/scram-tools.file/tools/alpaka/alpaka-cuda.xml
@@ -2,8 +2,8 @@
   <use name="alpaka"/>
   <use name="cuda"/>
   <!-- host comiplation should run with ALPAKA_HOST_ONLY defined -->
-  <flags CXXFLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -DALPAKA_HOST_ONLY"/>
-  <flags GENREFLEX_CPPFLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -DALPAKA_HOST_ONLY"/>
+  <flags CXXFLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -DALPAKA_ACC_GPU_CUDA_ONLY_MODE -DALPAKA_HOST_ONLY"/>
+  <flags GENREFLEX_CPPFLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -DALPAKA_ACC_GPU_CUDA_ONLY_MODE -DALPAKA_HOST_ONLY"/>
   <!-- device comiplation should run without ALPAKA_HOST_ONLY defined -->
-  <flags CUDA_FLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -UALPAKA_HOST_ONLY"/>
+  <flags CUDA_FLAGS="-DALPAKA_ACC_GPU_CUDA_ENABLED -DALPAKA_ACC_GPU_CUDA_ONLY_MODE -UALPAKA_HOST_ONLY"/>
 </tool>
diff --git a/scram-tools.file/tools/alpaka/alpaka-rocm.xml b/scram-tools.file/tools/alpaka/alpaka-rocm.xml
@@ -2,8 +2,8 @@
   <use name="alpaka"/>
   <use name="rocm"/>
   <!-- host comiplation should run with ALPAKA_HOST_ONLY defined -->
-  <flags CXXFLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_HOST_ONLY"/>
-  <flags GENREFLEX_CPPFLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_HOST_ONLY"/>
+  <flags CXXFLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_ACC_GPU_HIP_ONLY_MODE -DALPAKA_HOST_ONLY"/>
+  <flags GENREFLEX_CPPFLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_ACC_GPU_HIP_ONLY_MODE -DALPAKA_HOST_ONLY"/>
   <!-- device comiplation should run without ALPAKA_HOST_ONLY defined -->
-  <flags ROCM_FLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -UALPAKA_HOST_ONLY"/>
+  <flags ROCM_FLAGS="-DALPAKA_ACC_GPU_HIP_ENABLED -DALPAKA_ACC_GPU_HIP_ONLY_MODE -UALPAKA_HOST_ONLY"/>
 </tool>
diff --git a/scram-tools.file/tools/cmssw/cmssw.xml b/scram-tools.file/tools/cmssw/cmssw.xml
@@ -9,6 +9,7 @@
   </client>
   <runtime name="@OS_RUNTIME_LDPATH_NAME@" value="$CMSSW_BASE/biglib/$SCRAM_ARCH" type="path"/>
   <runtime name="@OS_RUNTIME_LDPATH_NAME@" value="$CMSSW_BASE/lib/$SCRAM_ARCH" type="path"/>
+  <runtime name="RIVET_ANALYSIS_PATH"      value="$LOCALTOP/lib/$SCRAM_ARCH" type="path"/>
   <runtime name="PATH"       value="$CMSSW_BINDIR" type="path"/>
   <runtime name="PYTHON3PATH" value="$CMSSW_BASE/lib/$SCRAM_ARCH" type="path"/>
   <runtime name="ROOT_INCLUDE_PATH" value="$CMSSW_BASE/src" type="path"/>

diff --git a/scram-tools.file/tools/rivet/rivet.xml b/scram-tools.file/tools/rivet/rivet.xml
@@ -6,7 +6,6 @@
 <environment name="INCLUDE" default="$RIVET_BASE/include"/>
 </client>
 <runtime name="PATH" value="$RIVET_BASE/bin" type="path"/>
-<runtime name="RIVET_ANALYSIS_PATH" value="$RIVET_BASE/lib/Rivet" type="path"/>
 <runtime name="RIVET_DATA_PATH" value="$RIVET_BASE/share/Rivet" type="path"/>
 <runtime name="PDFPATH" default="$RIVET_BASE/share" type="path"/>
 <runtime name="ROOT_INCLUDE_PATH" value="$INCLUDE" type="path"/>

diff --git a/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml b/scram-tools.file/tools/tensorflow-xla-runtime/tensorflow-xla-runtime.xml
@@ -1,9 +1,10 @@
 <tool name="tensorflow-xla-runtime" version="@TOOL_VERSION@">
   <client>
     <environment name="TENSORFLOW_XLA_RUNTIME_BASE" default="@TOOL_ROOT@"/>
-    <environment name="LIBDIR" default="$TENSORFLOW_XLA_RUNTIME_BASE/lib/archive"/>
+    <environment name="LIBDIR" default="$TENSORFLOW_XLA_RUNTIME_BASE/lib"/>
   </client>
-  <lib name="tf_xla_runtime-static"/>
+  <lib name="tf_xla_runtime"/>
+
   <use name="eigen"/>
   <use name="tensorflow-includes"/>
 </tool>
diff --git a/tensorflow-xla-runtime-absl.patch b/tensorflow-xla-runtime-absl.patch
@@ -0,0 +1,12 @@
+--- tensorflow/xla_aot_runtime_src/CMakeLists.txt    2024-03-24 08:28:34.000000000 +0100
++++ tensorflow/xla_aot_runtime_src/CMakeLists.txt      2024-03-25 11:17:58.108587945 +0100
+@@ -14,6 +14,8 @@
+   -Wno-sign-compare
+ )
+
+-add_library(tf_xla_runtime STATIC
++find_package(absl REQUIRED)
++add_library(tf_xla_runtime SHARED
+   $<TARGET_OBJECTS:tf_xla_runtime_objects>
+ )
++target_link_libraries(tf_xla_runtime absl::strings absl::str_format_internal)
diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec
@@ -4,28 +4,39 @@
 
 Source99: scram-tools.file/tools/eigen/env
 
-Requires: eigen py3-tensorflow
+Patch0: tensorflow-xla-runtime-absl
+
+Requires: eigen py3-tensorflow abseil-cpp
 BuildRequires: cmake
 
 %prep
 
 cp -r ${PY3_TENSORFLOW_ROOT}/lib/python%{cms_python3_major_minor_version}/site-packages/tensorflow .
+%patch -p0
 
 %build
+
 source %{_sourcedir}/env
 export CPATH="${CPATH}:${EIGEN_ROOT}/include/eigen3"
 
-CXXFLAGS="-fPIC %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}"
+CXXFLAGS="-fPIC -Wl,-z,defs %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS}"
 %ifarch x86_64
-    CXXFLAGS="${CXXFLAGS} -msse3"
+  CXXFLAGS="${CXXFLAGS} -msse3"
 %endif
 
 pushd tensorflow/xla_aot_runtime_src
-  cmake . -DCMAKE_CXX_FLAGS="${CXXFLAGS}" -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} -DBUILD_SHARED_LIBS=OFF
+  # remove unnecessary implementations that use symbols that are not even existing
+  find . -type f -path '*/service/cpu/runtime_fork_join.cc' | xargs rm -f
+
+  cmake . \
+    -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \
+    -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} \
+    -DCMAKE_PREFIX_PATH=${ABSEIL_CPP_ROOT} \
+    -DBUILD_SHARED_LIBS=ON
   make %{makeprocesses}
 popd
 
 %install
 
-mkdir -p %{i}/lib/archive
-mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.a %{i}/lib/archive/libtf_xla_runtime-static.a
+mkdir -p %{i}/lib
+mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.so %{i}/lib/
diff --git a/tfaot-compile.file b/tfaot-compile.file
@@ -0,0 +1,41 @@
+## tfaot common compilation and requirement file
+## specs including this file should provide:
+## 1. a variable %{aot_config}, pointing to the aot config file of the model to compile (required)
+## 2. a variable %{aot_source}, referring to a fetched source to unpack during %prep (optional)
+##    (in this case a "Source" should be defined and %{aot_source} is likely %{n}-%{realversion})
+
+BuildRequires: py3-cms-tfaot
+Requires: tensorflow-xla-runtime
+
+%ifarch ppc64le
+%define build_arch powerpc64le-unknown-linux-gnu
+%else
+%define build_arch %{_arch}-unknown-linux-gnu
+%endif
+
+%prep
+%if "%{?aot_source}"
+%setup -n %{aot_source}
+%endif
+
+%build
+cms_tfaot_compile \
+  --aot-config "%{aot_config}" \
+  --tool-name "%{n}" \
+  --tool-base "%{i}" \
+  --output-directory compiled_model \
+  --additional-flags="--target_triple %{build_arch}"
+
+%install
+mkdir -p %{i}/lib
+mv compiled_model/*.o %{i}/lib/
+
+mkdir -p %{i}/include/%{n}
+mv compiled_model/*.h %{i}/include/%{n}
+
+mkdir -p %{i}/etc/scram.d
+mv compiled_model/%{n}.xml %{i}/etc/scram.d/
+
+%post
+%{relocateConfig}etc/scram.d/%{n}.xml
+%relocateConfigAll include/%{n} *.h
diff --git a/tfaot-model-test-multi.spec b/tfaot-model-test-multi.spec
@@ -0,0 +1,5 @@
+### RPM external tfaot-model-test-multi 1.0.1
+
+%define aot_config $PY3_CMS_TFAOT_ROOT/share/test_models/multi/aot_config.yaml
+
+## INCLUDE tfaot-compile
diff --git a/tfaot-model-test-simple.spec b/tfaot-model-test-simple.spec
@@ -0,0 +1,5 @@
+### RPM external tfaot-model-test-simple 1.0.1
+
+%define aot_config $PY3_CMS_TFAOT_ROOT/share/test_models/simple/aot_config.yaml
+
+## INCLUDE tfaot-compile
diff --git a/tfaot-models.file b/tfaot-models.file
@@ -0,0 +1,3 @@
+# test models needed by unit tests in PhysicsTools/TensorFlowAOT
+Requires: tfaot-model-test-simple
+Requires: tfaot-model-test-multi