From 903a4ab360be737160d5c3a3c284baaa274e4d8e Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Wed, 7 Aug 2024 13:52:13 -0600 Subject: [PATCH] DAOS-16305 build: Simplify hermetic builds (#14874) Move URLs for repos to the config file allowing one to replace the config file. Signed-off-by: Jeff Olivier --- site_scons/components/__init__.py | 22 ++++++++++----------- site_scons/prereq_tools/base.py | 10 ++++++---- utils/build.config | 32 +++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index 8ae538f9624..ed50b80461b 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2016-2023 Intel Corporation +# Copyright 2016-2024 Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -127,7 +127,7 @@ def define_mercury(reqs): ofi_build.append('--disable-debug') reqs.define('ofi', - retriever=GitRepoRetriever('https://github.com/ofiwg/libfabric'), + retriever=GitRepoRetriever(), commands=[['./autogen.sh'], ofi_build, ['make'], @@ -152,7 +152,7 @@ def define_mercury(reqs): ucx_configure.extend(['--disable-debug', '--disable-logging']) reqs.define('ucx', - retriever=GitRepoRetriever('https://github.com/openucx/ucx.git'), + retriever=GitRepoRetriever(), libs=['ucs', 'ucp', 'uct'], functions={'ucs': ['ucs_debug_disable_signal']}, headers=['uct/api/uct.h'], @@ -189,7 +189,7 @@ def define_mercury(reqs): mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=OFF') reqs.define('mercury', - retriever=GitRepoRetriever('https://github.com/mercury-hpc/mercury.git', True), + retriever=GitRepoRetriever(True), commands=[mercury_build, ['make'], ['make', 'install']], @@ -246,14 +246,14 @@ def define_components(reqs): define_ompi(reqs) reqs.define('isal', - retriever=GitRepoRetriever('https://github.com/intel/isa-l.git'), + retriever=GitRepoRetriever(), commands=[['./autogen.sh'], ['./configure', '--prefix=$ISAL_PREFIX', '--libdir=$ISAL_PREFIX/lib'], ['make'], ['make', 'install']], libs=['isal']) reqs.define('isal_crypto', - retriever=GitRepoRetriever('https://github.com/intel/isa-l_crypto'), + retriever=GitRepoRetriever(), commands=[['./autogen.sh'], ['./configure', '--prefix=$ISAL_CRYPTO_PREFIX', @@ -263,7 +263,7 @@ def define_components(reqs): libs=['isal_crypto']) reqs.define('pmdk', - retriever=GitRepoRetriever('https://github.com/pmem/pmdk.git'), + retriever=GitRepoRetriever(), commands=[['make', 'all', 'NDCTL_ENABLE=n', @@ -288,7 +288,7 @@ def define_components(reqs): abt_build.append('--enable-valgrind') reqs.define('argobots', - retriever=GitRepoRetriever('https://github.com/pmodels/argobots.git', True), + retriever=GitRepoRetriever(True), commands=[['git', 'clean', '-dxf'], ['./autogen.sh'], abt_build, @@ -299,7 +299,7 @@ def define_components(reqs): headers=['abt.h']) reqs.define('fuse', libs=['fuse3'], defines=['FUSE_USE_VERSION=35'], - retriever=GitRepoRetriever('https://github.com/libfuse/libfuse.git'), + retriever=GitRepoRetriever(), commands=[['meson', 'setup', '--prefix=$FUSE_PREFIX', '-Ddisable-mtab=True', '-Dudevrulesdir=$FUSE_PREFIX/udev', '-Dutils=False', '--default-library', 'both', '../fuse'], @@ -328,7 +328,7 @@ def define_components(reqs): spdk_arch = 'haswell' reqs.define('spdk', - retriever=GitRepoRetriever('https://github.com/spdk/spdk.git', True), + retriever=GitRepoRetriever(True), commands=[['./configure', '--prefix=$SPDK_PREFIX', '--disable-tests', @@ -357,7 +357,7 @@ def define_components(reqs): patch_rpath=['lib', 'bin']) reqs.define('protobufc', - retriever=GitRepoRetriever('https://github.com/protobuf-c/protobuf-c.git'), + retriever=GitRepoRetriever(), commands=[['./autogen.sh'], ['./configure', '--prefix=$PROTOBUFC_PREFIX', '--disable-protoc'], ['make'], diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index d3446b6893b..74c39680466 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -239,9 +239,9 @@ def default_libpath(): class GitRepoRetriever(): """Identify a git repository from which to download sources""" - def __init__(self, url, has_submodules=False, branch=None): + def __init__(self, has_submodules=False, branch=None): - self.url = url + self.url = None self.has_submodules = has_submodules self.branch = branch self.commit_sha = None @@ -272,9 +272,10 @@ def _update_submodules(self, subdir): if not RUNNER.run_commands(commands, subdir=subdir): raise DownloadFailure(self.url, subdir) - def get(self, subdir, **kw): + def get(self, subdir, repo, **kw): """Downloads sources from a git repository into subdir""" # Now checkout the commit_sha if specified + self.url = repo passed_commit_sha = kw.get("commit_sha", None) if passed_commit_sha is None: comp = os.path.basename(subdir) @@ -1043,6 +1044,7 @@ def get(self): return branch = self.prereqs.get_config("branches", self.name) commit_sha = self.prereqs.get_config("commit_versions", self.name) + repo = self.prereqs.get_config("repos", self.name) if not self.retriever: print(f'Using installed version of {self.name}') @@ -1055,7 +1057,7 @@ def get(self): print(f'Downloading source for {self.name}') patches = self._resolve_patches() - self.retriever.get(self.src_path, commit_sha=commit_sha, + self.retriever.get(self.src_path, repo, commit_sha=commit_sha, patches=patches, branch=branch) def _has_missing_system_deps(self, env): diff --git a/utils/build.config b/utils/build.config index 2abb16d3f9a..3748bbd6e4e 100644 --- a/utils/build.config +++ b/utils/build.config @@ -2,16 +2,28 @@ component=daos [commit_versions] -ARGOBOTS = v1.1 -FUSE = fuse-3.16.2 -PMDK = 2.0.0 -ISAL = v2.30.0 -ISAL_CRYPTO = v2.23.0 -SPDK = v22.01.2 -OFI = v1.19.1 -MERCURY = v2.4.0rc4 -PROTOBUFC = v1.3.3 -UCX=v1.14.1 +argobots=v1.1 +fuse=fuse-3.16.2 +pmdk=2.0.0 +isal=v2.30.0 +isal_crypto=v2.23.0 +spdk=v22.01.2 +ofi=v1.19.1 +mercury=v2.4.0rc4 +protobufc=v1.3.3 +ucx=v1.14.1 + +[repos] +argobots=https://github.com/pmodels/argobots.git +fuse=https://github.com/libfuse/libfuse.git +pmdk=https://github.com/pmem/pmdk.git +isal=https://github.com/intel/isa-l.git +isal_crypto=https://github.com/intel/isa-l_crypto.git +spdk=https://github.com/spdk/spdk.git +ofi=https://github.com/ofiwg/libfabric.git +mercury=https://github.com/mercury-hpc/mercury.git +protobufc=https://github.com/protobuf-c/protobuf-c.git +ucx=https://github.com/openucx/ucx.git [patch_versions] spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff