DAOS-16305 build: Simplify hermetic builds (#14874)

Move URLs for repos to the config file allowing one to replace the config file. Signed-off-by: Jeff Olivier <[email protected]>
daos-stack · Aug 8, 2024 · 903a4ab · 903a4ab
1 parent 3f0d0d1
commit 903a4ab
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 25 deletions.
diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2016-2023 Intel Corporation
+# Copyright 2016-2024 Intel Corporation
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -127,7 +127,7 @@ def define_mercury(reqs):
         ofi_build.append('--disable-debug')
 
     reqs.define('ofi',
-                retriever=GitRepoRetriever('https://github.com/ofiwg/libfabric'),
+                retriever=GitRepoRetriever(),
                 commands=[['./autogen.sh'],
                           ofi_build,
                           ['make'],
@@ -152,7 +152,7 @@ def define_mercury(reqs):
         ucx_configure.extend(['--disable-debug', '--disable-logging'])
 
     reqs.define('ucx',
-                retriever=GitRepoRetriever('https://github.com/openucx/ucx.git'),
+                retriever=GitRepoRetriever(),
                 libs=['ucs', 'ucp', 'uct'],
                 functions={'ucs': ['ucs_debug_disable_signal']},
                 headers=['uct/api/uct.h'],
@@ -189,7 +189,7 @@ def define_mercury(reqs):
         mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=OFF')
 
     reqs.define('mercury',
-                retriever=GitRepoRetriever('https://github.com/mercury-hpc/mercury.git', True),
+                retriever=GitRepoRetriever(True),
                 commands=[mercury_build,
                           ['make'],
                           ['make', 'install']],
@@ -246,14 +246,14 @@ def define_components(reqs):
     define_ompi(reqs)
 
     reqs.define('isal',
-                retriever=GitRepoRetriever('https://github.com/intel/isa-l.git'),
+                retriever=GitRepoRetriever(),
                 commands=[['./autogen.sh'],
                           ['./configure', '--prefix=$ISAL_PREFIX', '--libdir=$ISAL_PREFIX/lib'],
                           ['make'],
                           ['make', 'install']],
                 libs=['isal'])
     reqs.define('isal_crypto',
-                retriever=GitRepoRetriever('https://github.com/intel/isa-l_crypto'),
+                retriever=GitRepoRetriever(),
                 commands=[['./autogen.sh'],
                           ['./configure',
                            '--prefix=$ISAL_CRYPTO_PREFIX',
@@ -263,7 +263,7 @@ def define_components(reqs):
                 libs=['isal_crypto'])
 
     reqs.define('pmdk',
-                retriever=GitRepoRetriever('https://github.com/pmem/pmdk.git'),
+                retriever=GitRepoRetriever(),
                 commands=[['make',
                            'all',
                            'NDCTL_ENABLE=n',
@@ -288,7 +288,7 @@ def define_components(reqs):
         abt_build.append('--enable-valgrind')
 
     reqs.define('argobots',
-                retriever=GitRepoRetriever('https://github.com/pmodels/argobots.git', True),
+                retriever=GitRepoRetriever(True),
                 commands=[['git', 'clean', '-dxf'],
                           ['./autogen.sh'],
                           abt_build,
@@ -299,7 +299,7 @@ def define_components(reqs):
                 headers=['abt.h'])
 
     reqs.define('fuse', libs=['fuse3'], defines=['FUSE_USE_VERSION=35'],
-                retriever=GitRepoRetriever('https://github.com/libfuse/libfuse.git'),
+                retriever=GitRepoRetriever(),
                 commands=[['meson', 'setup', '--prefix=$FUSE_PREFIX', '-Ddisable-mtab=True',
                            '-Dudevrulesdir=$FUSE_PREFIX/udev', '-Dutils=False',
                            '--default-library', 'both', '../fuse'],
@@ -328,7 +328,7 @@ def define_components(reqs):
         spdk_arch = 'haswell'
 
     reqs.define('spdk',
-                retriever=GitRepoRetriever('https://github.com/spdk/spdk.git', True),
+                retriever=GitRepoRetriever(True),
                 commands=[['./configure',
                            '--prefix=$SPDK_PREFIX',
                            '--disable-tests',
@@ -357,7 +357,7 @@ def define_components(reqs):
                 patch_rpath=['lib', 'bin'])
 
     reqs.define('protobufc',
-                retriever=GitRepoRetriever('https://github.com/protobuf-c/protobuf-c.git'),
+                retriever=GitRepoRetriever(),
                 commands=[['./autogen.sh'],
                           ['./configure', '--prefix=$PROTOBUFC_PREFIX', '--disable-protoc'],
                           ['make'],

diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py
@@ -239,9 +239,9 @@ def default_libpath():
 class GitRepoRetriever():
     """Identify a git repository from which to download sources"""
 
-    def __init__(self, url, has_submodules=False, branch=None):
+    def __init__(self, has_submodules=False, branch=None):
 
-        self.url = url
+        self.url = None
         self.has_submodules = has_submodules
         self.branch = branch
         self.commit_sha = None
@@ -272,9 +272,10 @@ def _update_submodules(self, subdir):
             if not RUNNER.run_commands(commands, subdir=subdir):
                 raise DownloadFailure(self.url, subdir)
 
-    def get(self, subdir, **kw):
+    def get(self, subdir, repo, **kw):
         """Downloads sources from a git repository into subdir"""
         # Now checkout the commit_sha if specified
+        self.url = repo
         passed_commit_sha = kw.get("commit_sha", None)
         if passed_commit_sha is None:
             comp = os.path.basename(subdir)
@@ -1043,6 +1044,7 @@ def get(self):
             return
         branch = self.prereqs.get_config("branches", self.name)
         commit_sha = self.prereqs.get_config("commit_versions", self.name)
+        repo = self.prereqs.get_config("repos", self.name)
 
         if not self.retriever:
             print(f'Using installed version of {self.name}')
@@ -1055,7 +1057,7 @@ def get(self):
 
         print(f'Downloading source for {self.name}')
         patches = self._resolve_patches()
-        self.retriever.get(self.src_path, commit_sha=commit_sha,
+        self.retriever.get(self.src_path, repo, commit_sha=commit_sha,
                            patches=patches, branch=branch)
 
     def _has_missing_system_deps(self, env):

diff --git a/utils/build.config b/utils/build.config
@@ -2,16 +2,28 @@
 component=daos
 
 [commit_versions]
-ARGOBOTS = v1.1
-FUSE = fuse-3.16.2
-PMDK = 2.0.0
-ISAL = v2.30.0
-ISAL_CRYPTO = v2.23.0
-SPDK = v22.01.2
-OFI = v1.19.1
-MERCURY = v2.4.0rc4
-PROTOBUFC = v1.3.3
-UCX=v1.14.1
+argobots=v1.1
+fuse=fuse-3.16.2
+pmdk=2.0.0
+isal=v2.30.0
+isal_crypto=v2.23.0
+spdk=v22.01.2
+ofi=v1.19.1
+mercury=v2.4.0rc4
+protobufc=v1.3.3
+ucx=v1.14.1
+
+[repos]
+argobots=https://github.com/pmodels/argobots.git
+fuse=https://github.com/libfuse/libfuse.git
+pmdk=https://github.com/pmem/pmdk.git
+isal=https://github.com/intel/isa-l.git
+isal_crypto=https://github.com/intel/isa-l_crypto.git
+spdk=https://github.com/spdk/spdk.git
+ofi=https://github.com/ofiwg/libfabric.git
+mercury=https://github.com/mercury-hpc/mercury.git
+protobufc=https://github.com/protobuf-c/protobuf-c.git
+ucx=https://github.com/openucx/ucx.git
 
 [patch_versions]
 spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff