Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

scram tool clean; support for building stack without an option tool #9218

Merged
merged 6 commits into from
Jun 6, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bootstrap-bundle.spec
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
### RPM external bootstrap-bundle 3.0
## NO_AUTO_DEPENDENCY
## NOCOMPILER

BuildRequires: gcc
BuildRequires: db6-bootstrap lua-bootstrap file-bootstrap
BuildRequires: xz-bootstrap libarchive-bootstrap
Expand Down Expand Up @@ -30,6 +29,7 @@ mkdir %{i}/share/misc
cp ${FILE_BOOTSTRAP_ROOT}/share/misc/magic.mgc %{i}/share/misc/magic.mgc
rm -f %{i}/bin/xml2-config %{i}/lib/xml2Conf.sh

%if 0%{!?use_system_gcc:1}
#Bundle libstd and libgcc_s and libelf
%ifos darwin
cp -P $GCC_ROOT/lib/lib{stdc++,gcc_s}*.%{soname} %{i}/lib
Expand All @@ -41,6 +41,7 @@ cp -P $GCC_ROOT/lib/libdw.%{soname}* %{i}/lib
cp -P $GCC_ROOT/lib/libdw-*.%{soname} %{i}/lib
cp -P $GCC_ROOT/bin/readelf %{i}/bin
%endif
%endif

find %{i}/bin -type f -writable -exec %{strip} {} \;
# Do not strip archives, otherwise index of contents will be lost on newer binutils
Expand Down
9 changes: 9 additions & 0 deletions bootstrap-driver.spec
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ platformSeeds+=" libaio"
platformSeeds+=" libxcrypt perl-libs"
%endif

platformSeeds+=" %{?rhel:libgcc}"
%if 0%{?fedora:1}
platformSeeds+=" libgcc libstdc++ elfutils-libelf libgfortran"
#Needed by millepede
platformSeeds+=" libgomp"
#Needed by gdb
platformSeeds+=" elfutils-debuginfod-client gmp mpfr"
%endif

defaultPkgs="cms+cms-common+1.0 cms+fakesystem+1.0"

mkdir -p %{i}/etc/profile.d
Expand Down
4 changes: 2 additions & 2 deletions cmssw-patch-tool-conf.spec → cmssw-patch-tools.spec
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
### RPM cms cmssw-patch-tool-conf 5.0
### RPM cms cmssw-patch-tools 1.0

## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes

Requires: cmssw

## INCLUDE cmssw-vectorization
## INCLUDE cmssw-drop-tools
## INCLUDE scram-tool-conf
## INCLUDE scram/tool-conf-src
17 changes: 5 additions & 12 deletions cmssw-tool-conf.spec → cmssw-tools.spec
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
### RPM cms cmssw-tool-conf 62.0
### RPM cms cmssw-tools 1.0
# With cmsBuild, change the above version only when a new tool is added

## INSTALL_DEPENDENCIES cmsLHEtoEOSManager gcc-fixincludes cmssw-osenv cms-git-tools SCRAMV2
Expand Down Expand Up @@ -76,7 +76,6 @@ Requires: sherpa
Requires: libpciaccess
Requires: numactl
Requires: hwloc
Requires: gdrcopy
Requires: rdma-core
Requires: ucx
Requires: openmpi
Expand Down Expand Up @@ -153,14 +152,9 @@ Requires: rivet
%ifos linux
Requires: openldap
Requires: gperftools
Requires: cuda
Requires: cuda-compatible-runtime
Requires: alpaka

%if "%{cmsos}" != "slc7_aarch64"
Requires: cudnn
%endif
%{!?without_cuda:Requires: cuda cuda-compatible-runtime gdrcopy cudnn}

Requires: alpaka
Requires: libunwind
%ifnarch ppc64le
Requires: igprof
Expand All @@ -175,8 +169,7 @@ Requires: oracle
Requires: icc
Requires: icx
Requires: intel-vtune
Requires: rocm
Requires: rocm-rocrand
%{!?without_rocm:Requires: rocm rocm-rocrand}
Requires: cmsmon-tools
Requires: dip
%else
Expand All @@ -194,4 +187,4 @@ Requires: pytorch-custom-ops
## INCLUDE tfaot-models
## INCLUDE cmssw-vectorization
## INCLUDE cmssw-drop-tools
## INCLUDE scram-tool-conf
## INCLUDE scram/tool-conf-src
4 changes: 2 additions & 2 deletions coral-tool-conf.spec → coral-tools.spec
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
### RPM cms coral-tool-conf 10.0
### RPM cms coral-tools 1.0

Requires: pcre
Requires: python3
Expand All @@ -18,4 +18,4 @@ Requires: oracle

%define skipreqtools jcompiler

## INCLUDE scram-tool-conf
## INCLUDE scram/tool-conf-src
2 changes: 1 addition & 1 deletion cpu_features.spec
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
### RPM external cpu_features 0.7.0
### RPM external cpu_features 0.9.0

Source: https://github.com/google/cpu_features/archive/refs/tags/v%{realversion}.tar.gz
BuildRequires: gmake cmake
Expand Down
13 changes: 5 additions & 8 deletions fwlite-tool-conf.spec → fwlite-tools.spec
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
### RPM cms fwlite-tool-conf 20.0
### RPM cms fwlite-tools 1.0
# with cmsBuild, change the above version only when a new
# tool is added
## INITENV SET CMSSW_TOOL_CONF_ROOT $FWLITE_TOOL_CONF_ROOT

%ifarch x86_64
Requires: rocm
Requires: rocm-rocrand
%endif
%{!?without_rocm:Requires: rocm rocm-rocrand}
%{!?without_cuda:Requires: cuda cuda-compatible-runtime}

Requires: alpaka
Requires: cuda-compatible-runtime
Requires: cuda
Requires: eigen
Requires: fmt
Requires: tbb
Expand Down Expand Up @@ -62,4 +59,4 @@ Requires: freetype
%endif

## INCLUDE cmssw-drop-tools
## INCLUDE scram-tool-conf
## INCLUDE scram/tool-conf-src
23 changes: 9 additions & 14 deletions hwloc.spec
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
Source: https://download.open-mpi.org/release/%{n}/v2.10/%{n}-%{realversion}.tar.bz2

BuildRequires: autotools
Requires: cuda libpciaccess libxml2 numactl
%ifarch x86_64
Requires: rocm
%endif
Requires: libpciaccess libxml2 numactl
%{!?without_rocm:Requires: rocm}
%{!?without_cuda:Requires: cuda}


%prep
%setup -n %{n}-%{realversion}

%build
./configure \
--prefix %{i} \
--enable-shared \
Expand All @@ -20,15 +21,10 @@ Requires: rocm
--disable-cairo \
--disable-doxygen \
--disable-opencl \
--with-cuda=$CUDA_ROOT \
--enable-cuda \
--enable-nvml \
%ifarch x86_64
--with-rocm=$ROCM_ROOT \
--enable-rsmi \
--enable-plugins=cuda,nvml,rsmi \
%else
--enable-plugins=cuda,nvml \
%{!?without_cuda:--with-cuda=$CUDA_ROOT --enable-cuda --enable-nvml} \
%{!?without_rocm:--with-rocm=$ROCM_ROOT --enable-rsmi} \
%if 0%{!?without_cuda:1}%{!?without_rocm:1}
--enable-plugins=$(echo %{!?without_cuda:cuda,nvml,}%{!?without_rocm:rsmi} | sed 's|,$||') \
%endif
--with-pic \
--with-gnu-ld \
Expand All @@ -40,7 +36,6 @@ Requires: rocm
HWLOC_NUMA_CFLAGS="-I$NUMACTL_ROOT/include" \
HWLOC_NUMA_LIBS="-L$NUMACTL_ROOT/lib -lnuma"

%build
make %{makeprocesses}

%install
Expand Down
4 changes: 2 additions & 2 deletions java-env.spec
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
### RPM external java-env 1.1
### RPM external java-env 11.0
## INITENV SET JAVA_HOME %{java_home}
## INITENV +PATH PATH %{java_home}/bin
## INITENV +PATH LD_LIBRARY_PATH %{java_home}/lib
## NOCOMPILER

%if %{rhel} < 9
%if 0%{?rhel} >= 9
# /usr/lib/jvm/java on EL8 contains java 1.8, and Bazel requires java 11
%define java_home /usr/lib/jvm/java-11
%else
Expand Down
2 changes: 1 addition & 1 deletion jemalloc-common.file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ BuildRequires: autotools gmake

%if "%{n}" != "jemalloc"
%ifarch x86_64
%if "%{rhel}" == "7"
%if 0%{?rhel} == 7
BuildRequires: patchelf
%endif
%endif
Expand Down
6 changes: 4 additions & 2 deletions llvm.spec
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

BuildRequires: cmake ninja
Requires: gcc zlib python3
Requires: cuda
%{!?without_cuda:Requires: cuda}

%define llvmCommit 83204dfcd4277154e46a5c6094aee389a7f260e8
%define llvmBranch cms/release/17.x/afbe354
Expand Down Expand Up @@ -49,9 +49,11 @@ cmake %{_builddir}/llvm-%{realversion}-%{llvmCommit}/llvm \
-DLLVM_ENABLE_PIC:BOOL=ON \
-DLLVM_ENABLE_RTTI:BOOL=ON \
-DLLVM_HOST_TRIPLE=$(gcc -dumpmachine) \
-DLLVM_TARGETS_TO_BUILD:STRING="X86;PowerPC;AArch64;NVPTX" \
-DLLVM_TARGETS_TO_BUILD:STRING="X86;PowerPC;AArch64%{!?without_cuda:;NVPTX}" \
%if 0%{!?without_cuda:1}
-DLIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER=/usr/bin/gcc \
-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES="%omptarget_cuda_archs" \
%endif
-DCMAKE_REQUIRED_INCLUDES="${ZLIB_ROOT}/include" \
-DCMAKE_PREFIX_PATH="${ZLIB_ROOT}"

Expand Down
18 changes: 9 additions & 9 deletions onnxruntime.spec
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz

BuildRequires: cmake ninja
Requires: protobuf py3-numpy py3-wheel py3-onnx zlib libpng py3-pybind11 cuda re2
%if "%{cmsos}" != "slc7_aarch64"
Requires: cudnn
%endif
Requires: protobuf py3-numpy py3-wheel py3-onnx zlib libpng py3-pybind11 re2
%{!?without_cuda:Requires: cuda cudnn}

%prep
%setup -q -n %{n}-%{realversion}
Expand All @@ -21,7 +19,7 @@ rm -rf ../build; mkdir ../build; cd ../build
USE_CUDA=OFF
%if "%{cmsos}" != "slc7_aarch64"
if [ "%{cuda_gcc_support}" = "true" ] ; then
USE_CUDA=ON
USE_CUDA=%{!?without_cuda:ON}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@smuzaffar Sholdn't %{cuda_gcc_support} evaluate to false if we don't have cuda?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cuda_gcc_support is when we have cuda i.e. we can run but we do not have a working cuda/gcc combination

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and

cmsdist/rpm-preamble.file

Lines 259 to 270 in 6a36618

%if 0%{!?without_cuda:1}
%define cuda_gcc_support $(\
if [ ! -f %{_tmppath}/cache/cuda_gcc_supported.txt ] ; then \
touch %{_builddir}/cuda_gcc_supported.cu &&\
echo true > %{_builddir}/cuda_gcc_supported.txt &&\
(nvcc -dc %{_builddir}/cuda_gcc_supported.cu -o %{_builddir}/cuda_gcc_supported.cu.o 2>&1 || echo 'unsupported GCC version') > %{_builddir}/cuda_gcc_supported.log &&\
if [ $(grep -i 'unsupported ' %{_builddir}/cuda_gcc_supported.log | wc -l) -gt 0 ] ; then echo false > %{_builddir}/cuda_gcc_supported.txt ; fi;\
mv %{_builddir}/cuda_gcc_supported.txt %{_tmppath}/cache/cuda_gcc_supported.txt ;\
fi; cat %{_tmppath}/cache/cuda_gcc_supported.txt)
%else
%define cuda_gcc_support false
%endif
chage is going to force set cuda_gcc_support to false is sw stack is build without cuda

Copy link
Contributor

@iarspider iarspider Jun 3, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation. Will this work in a hypotetical situation where we have CUDA support, but don't build with CUDA (i.e. won't a line USE_CUDA=, where nothing is assigned to USE_CUDA, break anything?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in the case it will work as USE_CUDA=%{!?without_cuda:ON} will never be executed when we build without_cuda

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is true that in this case USE_CUDA=%{!?without_cuda:ON} change is not needed as for without_cuda this blockof code is not executed

fi
%endif

Expand All @@ -33,9 +31,15 @@ cmake ../%{n}-%{realversion}/cmake -GNinja \
-Donnxruntime_ENABLE_PYTHON=ON \
-Donnxruntime_BUILD_SHARED_LIB=ON \
-Donnxruntime_USE_CUDA=${USE_CUDA} \
%if 0%{!?without_cuda:1}
-Donnxruntime_CUDA_HOME="${CUDA_ROOT}" \
-Donnxruntime_CUDNN_HOME="${CUDNN_ROOT}" \
-Donnxruntime_NVCC_THREADS=0 \
-DCMAKE_CUDA_ARCHITECTURES=$(echo %{cuda_arch} | tr ' ' ';' | sed 's|;;*|;|') \
-DCMAKE_CUDA_FLAGS="-cudart shared" \
-DCMAKE_CUDA_RUNTIME_LIBRARY=Shared \
-DCMAKE_TRY_COMPILE_PLATFORM_VARIABLES="CMAKE_CUDA_RUNTIME_LIBRARY" \
%endif
-Donnxruntime_BUILD_CSHARP=OFF \
-Donnxruntime_USE_OPENMP=OFF \
-Donnxruntime_USE_TVM=OFF \
Expand All @@ -48,10 +52,6 @@ cmake ../%{n}-%{realversion}/cmake -GNinja \
-Donnxruntime_DISABLE_CONTRIB_OPS=OFF \
-Donnxruntime_PREFER_SYSTEM_LIB=ON \
-Donnxruntime_BUILD_UNIT_TESTS=OFF \
-DCMAKE_CUDA_ARCHITECTURES=$(echo %{cuda_arch} | tr ' ' ';' | sed 's|;;*|;|') \
-DCMAKE_CUDA_FLAGS="-cudart shared" \
-DCMAKE_CUDA_RUNTIME_LIBRARY=Shared \
-DCMAKE_TRY_COMPILE_PLATFORM_VARIABLES="CMAKE_CUDA_RUNTIME_LIBRARY" \
-DCMAKE_PREFIX_PATH="${ZLIB_ROOT};${LIBPNG_ROOT};${PROTOBUF_ROOT};${PY3_PYBIND11_ROOT};${RE2_ROOT}" \
-DRE2_INCLUDE_DIR="${RE2_ROOT}/include" \
-DCMAKE_CXX_FLAGS="-Wno-error=stringop-overflow -Wno-error=maybe-uninitialized"
Expand Down
4 changes: 2 additions & 2 deletions openmpi.spec
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
## INITENV SET OPAL_PREFIX %{i}
Source: https://download.open-mpi.org/release/open-mpi/v4.1/%{n}-%{realversion}.tar.bz2
BuildRequires: autotools
Requires: cuda
%{!?without_cuda:Requires: cuda}
Requires: hwloc
Requires: rdma-core
Requires: xpmem
Expand Down Expand Up @@ -31,7 +31,7 @@ Requires: zlib
--disable-mpi-java \
--enable-openib-rdmacm-ibaddr \
--with-zlib=$ZLIB_ROOT \
--with-cuda=$CUDA_ROOT \
%{!?without_cuda:--with-cuda=$CUDA_ROOT} \
--with-hwloc=$HWLOC_ROOT \
--without-ofi \
--without-portals4 \
Expand Down
2 changes: 1 addition & 1 deletion python_tools.spec
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ Requires: py3-importlib-resources
Requires: py3-smmap
Requires: py3-zipp

Requires: py3-pycuda
%{!?without_cuda:Requires: py3-pycuda}

Requires: py3-boost-histogram
Requires: py3-hist
Expand Down
2 changes: 1 addition & 1 deletion pytorch-cluster.spec
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
Source: git+https://github.com/%{github_user}/pytorch_cluster.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz

BuildRequires: cmake
Requires: pytorch cudnn
Requires: pytorch
%define build_flags -Wall -Wextra -pedantic %{?arch_build_flags}

%prep
Expand Down
2 changes: 1 addition & 1 deletion pytorch-scatter.spec
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
Source: git+https://github.com/%{github_user}/pytorch_scatter.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz

BuildRequires: cmake
Requires: pytorch cudnn
Requires: pytorch
%define build_flags -Wall -Wextra -pedantic %{?arch_build_flags}

%prep
Expand Down
2 changes: 1 addition & 1 deletion pytorch-sparse.spec
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
Source: git+https://github.com/%{github_user}/pytorch_sparse.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz

BuildRequires: cmake
Requires: pytorch cudnn
Requires: pytorch
%define build_flags -Wall -Wextra -pedantic %{?arch_build_flags}

%prep
Expand Down
7 changes: 5 additions & 2 deletions pytorch.spec
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ Patch2: pytorch-system-fmt

BuildRequires: cmake ninja
Requires: eigen fxdiv numactl openmpi protobuf psimd python3 py3-PyYAML
Requires: cuda cudnn OpenBLAS zlib protobuf fmt py3-pybind11
Requires: OpenBLAS zlib protobuf fmt py3-pybind11
%{!?without_cuda:Requires: cuda cudnn}

%prep
%setup -n %{n}-%{realversion}
Expand All @@ -32,7 +33,7 @@ source %{_sourcedir}/env
USE_CUDA=OFF
%if "%{cmsos}" != "slc7_aarch64"
if [ "%{cuda_gcc_support}" = "true" ] ; then
USE_CUDA=ON
USE_CUDA=%{!?without_cuda:ON}
fi
%endif

Expand All @@ -43,10 +44,12 @@ cmake ../%{n}-%{realversion} \
-DBUILD_TEST=OFF \
-DBUILD_BINARY=OFF \
-DBUILD_PYTHON=OFF \
%if 0%{!?without_cuda:1}
-DUSE_CUDA=${USE_CUDA} \
-DTORCH_CUDA_ARCH_LIST="%{cuda_arch_float}" \
-DCUDNN_INCLUDE_DIR=${CUDNN_ROOT}/include \
-DCUDNN_LIBRARY=${CUDNN_ROOT}/lib64/libcudnn.so \
%endif
-DUSE_NCCL=OFF \
-DUSE_FBGEMM=OFF \
-DUSE_KINETO=OFF \
Expand Down
2 changes: 1 addition & 1 deletion rocm-rocrand.spec
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
### RPM external rocm-rocrand 5.6.1
## NOCOMPILER

%if "%{rhel}" == "7"
%if 0%{?rhel} == 7
# allow rpm2cpio dependency on the bootstrap bundle
%undefine drop_bootstrap_lib
%define drop_bootstrap_lib true
Expand Down
2 changes: 1 addition & 1 deletion rocm.spec
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
### RPM external rocm 5.6.1

%if "%{rhel}" == "7"
%if 0%{?rhel} == 7
# allow rpm2cpio dependency on the bootstrap bundle
%undefine drop_bootstrap_lib
%define drop_bootstrap_lib true
Expand Down
Loading