[PGO] Initial changes to support PGO builds

cmssw-patch-build.file

@@ -12,20 +12,16 @@
 %define baserelqueue	      %(echo %realversion | sed -e "s|_[[:alnum:]]*patch.*||")
 %endif
 
-%if "%{?cvstag:set}" != "set"
-%define cvstag		      %realversion
-%endif
-
 %if "%{?prebuildtarget}" == "gindices"
 %define prebuildtarget        %{nil}
 %endif
 
-%define source1               cmstc://?tag=%{cvstag}&module=CMSSW&export=src&baserelver=%{baserelver}&baserel=%{baserelqueue}&output=/src.tar.gz
+%define source1               cmstc://?tag=%{realversion}&module=CMSSW&export=src&baserelver=%{baserelver}&baserel=%{baserelqueue}&output=/src.tar.gz
 
 %define baserelverX           %(echo %{pkgreqs} | sed -e 's|.*/%baserelname/||' | awk '{print $1}')
 %define patchsrc9 \
   if [ "X%{baserelverX}" != "X%{baserelver}" ] ; then \
-    echo "ERROR: Please define valid baserel_postfix e.g. -cms, which was added by cmsBuild for %baserelname base release, in %n.spec file.";\
+    echo "ERROR: Please define valid baserel_postfix e.g. -cms, which was added by cmsBuild for %baserelname base release, in %{n}.spec file.";\
     false ;\
   fi
 

cmssw-tool-conf.spec

@@ -1,4 +1,4 @@
-### RPM cms cmssw-tool-conf 56.0
+### RPM cms cmssw-tool-conf 57.0
 # With cmsBuild, change the above version only when a new tool is added
 
 ## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools SCRAMV2
@@ -121,7 +121,6 @@ Requires: opencl-cpp
 Requires: qd
 Requires: blackhat
 Requires: sherpa
-Requires: geant4-parfullcms
 Requires: fasthadd
 Requires: eigen
 Requires: gdb

compilation_flags_pgo.file

@@ -0,0 +1,21 @@
+#####################
+#PGO flags
+########################################################
+#This file should not be included directly by spec files.
+# It is automatically added for selected packages via pgo/cmsdist_packages.py
+########################################################
+# Following are set by pgo/cmsdist_packages.py
+# pgo_path_prefix: path prefix where package was build.
+# pgo_package_name: Original name of the package e.g. geant4, vecgeom , cmssw etc.
+# pgo_generate: Set if package is build in pgo generate mode
+# pgo_use: Set if package is build in pgo use mode
+
+%define cmsdist_package_initenv export CMSSW_PGO_DIRECTORY=%{cmsroot}/%{tempprefix}
+%define pgo_common -fprofile-prefix-path=%{pgo_path_prefix} -fprofile-update=prefer-atomic -fprofile-correction
+%if "%{?pgo_generate:set}" == "set"
+%define pgo_build_flags %{pgo_common} -fprofile-generate -fprofile-dir=%%q{CMSSW_PGO_DIRECTORY}/pgo/%%p/%{pgo_package_name}
+%endif
+%if "%{?pgo_use:set}" == "set"
+%define pgo_build_flags %{pgo_common} -fprofile-use -fprofile-partial-training -fprofile-dir=%%q{CMSSW_PGO_DIRECTORY}/pgo/%{pgo_package_name}
+%endif
+

dd4hep.spec

@@ -11,11 +11,7 @@ Source: git+https://github.com/%{github_user}/DD4hep.git?obj=%{branch}/%{tag}&ex
 BuildRequires: cmake
 Requires: root boost clhep xerces-c geant4
 
-%if "%{?arch_build_flags}"
-%define build_flags -fPIC %{arch_build_flags} %{lto_build_flags}
-%else
-%define build_flags -fPIC %{lto_build_flags}
-%endif
+%define build_flags -fPIC %{?arch_build_flags} %{?lto_build_flags} %{?pgo_build_flags}
 
 %define cmake_fixed_args \\\
   -DCMAKE_INSTALL_PREFIX='%{i}' \\\

g4hepem.spec

@@ -1,5 +1,6 @@
 ### RPM external g4hepem 20230309
 ## INCLUDE compilation_flags
+## INCLUDE compilation_flags_lto
 %define tag %{realversion}
 %define branch master
 %define github_user mnovak42
@@ -10,6 +11,7 @@ BuildRequires: cmake gmake
 Requires: geant4
 
 %define keep_archives true
+%define build_flags -fPIC %{?arch_build_flags} %{?lto_build_flags} %{?pgo_build_flags}
 
 %prep
 %setup -n %{n}.%{realversion}
@@ -22,11 +24,7 @@ cd ../build
 
 cmake ../%{n}.%{realversion} \
   -DCMAKE_CXX_COMPILER="g++" \
-%if "%{?arch_build_flags}"
-  -DCMAKE_CXX_FLAGS="-fPIC %{arch_build_flags}" \
-%else
-  -DCMAKE_CXX_FLAGS="-fPIC" \
-%endif
+  -DCMAKE_CXX_FLAGS="%{build_flags}" \
   -DCMAKE_AR=$(which gcc-ar) \
   -DCMAKE_RANLIB=$(which gcc-ranlib) \
   -DCMAKE_INSTALL_PREFIX:PATH="%i" \

geant4.spec

@@ -18,11 +18,7 @@ Requires: vecgeom
 Requires: zlib
 
 %define keep_archives true
-%if "%{?arch_build_flags}"
-%define build_flags -fPIC %{arch_build_flags} %{lto_build_flags}
-%else
-%define build_flags -fPIC %{lto_build_flags}
-%endif
+%define build_flags -fPIC %{?arch_build_flags} %{?lto_build_flags} %{?pgo_build_flags}
 
 %prep
 %setup -n %{n}.%{realversion}
@@ -80,6 +76,10 @@ find %i/lib64 -name "*.a" -exec gcc-ar x {} \;
 gcc-ar rcs libgeant4-static.a *.o
 find . -name "*.o" -delete
 
+%if "%{?pgo_build_flags}"
+sed -ire 's| +(-fprofile-[^ ]+ )+||' %{i}/lib64/Geant4-*/Geant4Config.cmake %{i}/bin/geant4-config
+%endif
+
 %post
 %{relocateCmsFiles} $(find $RPM_INSTALL_PREFIX/%{pkgrel} -name '*.cmake')
 %{relocateConfig}bin/geant4-config

pgo/cmsdist_packages.py

@@ -0,0 +1,17 @@
+from os.path import dirname, join
+pgo_packages = ["geant4", "vecgeom", "g4hepem", "dd4hep", "cmssw", "cmssw-patch", "cmssw-tool-conf"]
+
+def packages(virtual_packages, *args):
+  opts = args[0].options
+  try:
+    if (not opts.PGOGenerate) and (not opts.PGOUse): return
+  except:
+    return
+  for pkg in pgo_packages:
+    spec  = "  echo '%%define pgo_path_prefix  %s'" % ('@LOCALTOP@' if pkg in ["cmssw-tool-conf"] else '%{_builddir}')
+    spec += "; echo '%%define pgo_package_name %s'" % ('cmssw' if pkg in ["cmssw-tool-conf"] else pkg)
+    spec += "; echo '%%define %s 1'" % ('pgo_generate' if opts.PGOGenerate else 'pgo_use')
+    spec += "; echo '## INCLUDE compilation_flags_pgo'"
+    spec += "; cat %s/%s.spec" % (opts.cmsdist, pkg)
+    virtual_packages[pkg] = spec
+  return

rpm-preamble.file

@@ -66,7 +66,7 @@
 %define cmsbuildApiVersion 0
 %endif
 
-%define pkgdir		%{pkgcategory}/%{n}/%{v}
+%define pkgdir		%{pkgcategory}/%{pkgname}/%{pkgversion}
 
 # Architecture comes first by default.  Run with "--define 'archfirst no'"
 # to change this.
@@ -164,13 +164,14 @@
 %define drop_bootstrap_lib  export %{dynamic_path_var}=$(echo $%{dynamic_path_var} | tr ':' '\\n' | grep -v '/external/bootstrap-bundle/' | tr '\\n' ':' | sed 's|:*$||')
 %define disable_recursive_env %{drop_bootstrap_path}; %{drop_bootstrap_lib}; export _CMSBUILD_BUILD_ENV_=1
 %if "%archfirst" == "yes"
-%define initenv_all	%{disable_recursive_env} ; for x in %{allpkgreqs}                          .; do i=%{cmsroot}/%{cmsplatf}/$x/etc/profile.d/init.sh; [ -f $i ] && . $i; done
-%define initenv_direct	%{disable_recursive_env} ; for x in %{directpkgreqs} %{builddirectpkgreqs} .; do i=%{cmsroot}/%{cmsplatf}/$x/etc/profile.d/init.sh; [ -f $i ] && . $i; done
+%define initenv_all     %{disable_recursive_env} ; for x in %{allpkgreqs}                          .; do i=%{cmsroot}/%{cmsplatf}/$x/etc/profile.d/init.sh; [ -f $i ] && . $i; done
+%define initenv_direct1 %{disable_recursive_env} ; for x in %{directpkgreqs} %{builddirectpkgreqs} .; do i=%{cmsroot}/%{cmsplatf}/$x/etc/profile.d/init.sh; [ -f $i ] && . $i; done
 %else
-%define initenv_all	%{disable_recursive_env} ; for x in %{allpkgreqs}                          .; do i=%{cmsroot}/$x/%{cmsplatf}/etc/profile.d/init.sh; [ -f $i ] && . $i; done
-%define initenv_direct	%{disable_recursive_env} ; for x in %{directpkgreqs} %{builddirectpkgreqs} .; do i=%{cmsroot}/$x/%{cmsplatf}/etc/profile.d/init.sh; [ -f $i ] && . $i; done
+%define initenv_all     %{disable_recursive_env} ; for x in %{allpkgreqs}                          .; do i=%{cmsroot}/$x/%{cmsplatf}/etc/profile.d/init.sh; [ -f $i ] && . $i; done
+%define initenv_direct1 %{disable_recursive_env} ; for x in %{directpkgreqs} %{builddirectpkgreqs} .; do i=%{cmsroot}/$x/%{cmsplatf}/etc/profile.d/init.sh; [ -f $i ] && . $i; done
 %endif
-%define initenv		%initenv_all
+%define initenv_direct  %{initenv_direct1} %{?cmsdist_package_initenv: ; %{cmsdist_package_initenv}}
+%define initenv         %{initenv_all}     %{?cmsdist_package_initenv: ; %{cmsdist_package_initenv}}
 
 %if "%{?compiling_processes:set}" == "set"
 %define makeprocesses -j %compiling_processes 
@@ -279,8 +280,8 @@ fi
 %define check_version_suffix echo "%%allow_version_suffix defined, skipping version suffix check."
 %else
 %define check_version_suffix \
-  if [ "%{v}" != "%{realversion}" ] ; then \
-    echo "ERROR: %{v} not same as %{realversion}. Automatic version suffix was added, please update the version explicitly for %{n}.spec to avoid this error." ;\
+  if [ "%{pkgversion}" != "%{realversion}" ] ; then \
+    echo "ERROR: %{pkgversion} not same as %{realversion}. Automatic version suffix was added, please update the version explicitly for %{pkgname}.spec to avoid this error." ;\
     exit 1 ;\
   fi
 %endif

scram-project-build.file

@@ -11,6 +11,11 @@
 %define vectorized_build yes
 %endif
 %endif
+%if "%{?pgo_generate}"
+%undefine runGlimpse
+%undefine saveDeps
+%undefine subpackageDebug
+%endif
 
 Requires: SCRAMV1
 BuildRequires: gcc
@@ -32,10 +37,6 @@ BuildRequires: dwz
 %define buildtarget	release-build
 %endif
 
-%if "%{?cvstag:set}" != "set"
-%define cvstag          %realversion
-%endif
-
 %if "%{?scram_compiler:set}" != "set"
 %define scram_compiler          gcc
 %endif
@@ -54,27 +55,19 @@ BuildRequires: dwz
 %endif
 
 %if "%{?configtag:set}" != "set"
-%define configtag       V07-09-01
-%endif
-
-%if "%{?cvssrc:set}" != "set"
-%define cvssrc		%(echo %n | sed -e "s|-patch||"| tr 'a-z' 'A-Z')
+%define configtag       V08-00-00
 %endif
 
 %if "%{?buildarch:set}" != "set"
 %define buildarch	:
 %endif
 
-%if "%{?cmsroot:set}" != "set"
-%define cmsroot       %instroot
-%endif
-
 %if "%{?ucprojtype:set}" != "set"
-%define ucprojtype      %(echo %n | sed -e "s|-patch||" | tr 'a-z' 'A-Z')
+%define ucprojtype      %(echo %{n} | sed -e "s|-patch||" | tr 'a-z' 'A-Z')
 %endif
 %define lcprojtype      %(echo %ucprojtype | tr 'A-Z' 'a-z')
 %if "%{?toolconf:set}" != "set"
-%define toolconf        %(echo %n | sed "s|-|_|g" | tr 'a-z' 'A-Z')_TOOL_CONF_ROOT
+%define toolconf        %(echo %{n} | sed "s|-|_|g" | tr 'a-z' 'A-Z')_TOOL_CONF_ROOT
 %endif
 
 %define source0 git+https://github.com/cms-sw/cmssw-config.git?obj=master/%{configtag}&export=config&output=/cmssw-config-%{configtag}.tgz
@@ -121,9 +114,14 @@ echo %{configtag} > %_builddir/config/config_tag
   --keys SCRAM_COMPILER=%{scram_compiler} \
   --keys ENABLE_LTO=%{enable_lto} \
 %if "%{?gitcommit:set}" == "set"
-  --keys PROJECT_GIT_HASH=%{gitcommit}
+  --keys PROJECT_GIT_HASH=%{gitcommit} \
+%else
+  --keys PROJECT_GIT_HASH=%{realversion} \
+%endif
+%if "%{?pgo_build_flags}"
+  --keys ENABLE_PGO=1
 %else
-  --keys PROJECT_GIT_HASH=%{realversion}
+  --keys ENABLE_PGO=0
 %endif
 %if "%{?vectorized_build:set}" == "set"
 sed -i -e 's| SCRAM_TARGETS=.*"| SCRAM_TARGETS="%{package_vectorization}"|' %_builddir/config/Self.xml
@@ -141,6 +139,9 @@ rm -rf %i
 mkdir -p $(dirname %i)
 %{?buildarch:%buildarch}
 %scramcmd project -d $(dirname %i) -b %{bootstrapfile}
+%if "%{?pgo_build_flags}"
+sed -i -e 's|@LOCALTOP@|%{i}|' %i/config/toolbox/%{cmsplatf}/tools/selected/gcc-cxxcompiler.xml
+%endif
 
 %build
 
@@ -183,7 +184,9 @@ export SCRAM_NOSYMCHECK=true
 %scramcmd b --verbose -f %{prebuildtarget} </dev/null
 %endif
 
-case %n in (cmssw|cmssw-patch) %scramcmd b -f -k %{makeprocesses} llvm-ccdb </dev/null || true ;; esac
+%if "%{?pgo_generate:set}" != "set"
+case %{n} in (cmssw|cmssw-patch) %scramcmd b -f -k %{makeprocesses} llvm-ccdb </dev/null || true ;; esac
+%endif
 %if "%{?vectorized_build:set}" == "set"
 touch %{i}/.SCRAM/%{cmsplatf}/multi-targets
 %endif

scram-tools.file/tool-env.file

@@ -3,10 +3,8 @@
 ## INCLUDE cuda-flags
 
 export ROOT_CXXMODULES="0"
-%if "%{?package_vectorization}" != ""
-export PKG_VECTORIZATION=`echo %package_vectorization`
-%endif
-export CMSDIST_DIR=%cmsdist_directory
+export PKG_VECTORIZATION=%{?package_vectorization}
+export CMSDIST_DIR=%{cmsdist_directory}
 
 %ifarch x86_64
 export COMPILER_CXXFLAGS="$(%{cmsdist_directory}/vectorization/cmsdist_packages.py)"
@@ -17,8 +15,7 @@ export COMPILER_CXXFLAGS="%{arch_build_flags}"
 export ORACLE_ENV_ROOT=""
 export CUDA_FLAGS="%{nvcc_cuda_flags}"
 export CUDA_HOST_CXXFLAGS="%{nvcc_stdcxx}"
-%if "%{?lto_build_flags}"
-export LTO_FLAGS="%{lto_build_flags}"
-%endif
+export LTO_FLAGS="%{?lto_build_flags}"
+export PGO_FLAGS="%{?pgo_build_flags}"
 if [ $(echo ' %requiredtools ' | grep ' python3 ' |wc -l) -gt 0 ] ; then export PYTHON3_LIB_SITE_PACKAGES ; fi
 if [ $(echo ' %requiredtools ' | grep ' python ' |wc -l) -gt 0 ] ;  then export PYTHON_LIB_SITE_PACKAGES ; fi

scram-tools.file/tools/gcc/gcc-cxxcompiler.xml

@@ -24,6 +24,7 @@
     <flags CXXFLAGS="-Werror=switch -fdiagnostics-show-option"/>
     <flags CXXFLAGS="-Wno-unused-local-typedefs -Wno-attributes -Wno-psabi"/>
     <flags LTO_FLAGS="@LTO_FLAGS@"/>
+    <flags PGO_FLAGS="@PGO_FLAGS@"/>
     <flags LDFLAGS="@GCC_LDFLAGS@"/>
     <flags CXXSHAREDFLAGS="@GCC_SHAREDFLAGS@"/>
     <flags LD_UNIT="@GCC_LD_UNIT@"/>

scram-tools.file/tools/geant4/env.sh

@@ -1,5 +1,5 @@
 GEANT4_VECGEOM=""
-if grep VECGEOM_ROOT ${TOOL_ROOT}/etc/profile.d/dependencies-setup.sh >/dev/null 2>&1  ; then
+if grep VECGEOM_ ${TOOL_ROOT}/etc/profile.d/dependencies-setup.sh >/dev/null 2>&1  ; then
   GEANT4_VECGEOM='<use name="vecgeom"/>'
 fi
 export GEANT4_VECGEOM

scram-tools.file/tools/llvm/llvm-cxxcompiler.xml

@@ -19,6 +19,7 @@
     <flags REM_CXXFLAGS="-funroll-all-loops"/>
     <flags REM_LTO_FLAGS="-fipa-icf"/>
     <flags REM_LTO_FLAGS="-flto-odr-type-merging"/>
+    <flags REM_PGO_FLAGS="-fprofile%"/>
     <flags CXXFLAGS="-Wno-c99-extensions"/>
     <flags CXXFLAGS="-Wno-c++11-narrowing"/>
     <flags CXXFLAGS="-D__STRICT_ANSI__"/>

vecgeom.spec

@@ -8,6 +8,8 @@ BuildRequires: cmake gmake
 
 Patch0: vecgeom-fix-vector
 
+%define build_flags %{?arch_build_flags} %{?lto_build_flags} %{?pgo_build_flags}
+
 %prep
 %setup -n %{n}-%{realversion}
 
@@ -32,11 +34,10 @@ cmake ../%{n}-%{realversion} \
 %endif
   -DCMAKE_VERBOSE_MAKEFILE=TRUE \
   -DCMAKE_CXX_STANDARD=17 \
-%if "%{?arch_build_flags}"
-  -DCMAKE_CXX_FLAGS="%{arch_build_flags} %{lto_build_flags}" \
-%else
-  -DCMAKE_CXX_FLAGS="%{lto_build_flags}" \
-%endif
+  -DCMAKE_STATIC_LIBRARY_CXX_FLAGS="%{build_flags}" \
+  -DCMAKE_STATIC_LIBRARY_C_FLAGS="%{build_flags}" \
+  -DCMAKE_CXX_FLAGS="%{build_flags}" \
+  -DCMAKE_C_FLAGS="%{build_flags}" \
   -DGEANT4=OFF
 
 make %{makeprocesses} VERBOSE=1