Merge branch 'master' into jvolivie/hermetic

Required-githooks: true

Signed-off-by: Jeff Olivier <[email protected]>

SConstruct

@@ -60,9 +60,15 @@ def add_command_line_options():
     AddOption('--build-deps',
               dest='build_deps',
               type='choice',
-              choices=['yes', 'no', 'only', 'build-only'],
+              choices=['fetch', 'yes', 'no', 'only'],
               default='no',
-              help="Automatically download and build sources.  (yes|no|only|build-only) [no]")
+              help="Automatically download and build sources.  (fetch|yes|no|only) [no]")
+
+    AddOption('--skip-download',
+              dest='skip_download',
+              action='store_true',
+              default=False,
+              help="Assume the source for prerequisites is already downloaded")
 
     # We want to be able to check what dependencies are needed without
     # doing a build, similar to --dry-run.  We can not use --dry-run
@@ -372,6 +378,8 @@ def scons():
     check_for_release_target()
 
     deps_env = Environment()
+    # Ensure 'install-sandbox' option is defined early
+    deps_env.Tool('install')
 
     # Silence deprecation warning so it doesn't fail the build
     SetOption('warn', ['no-python-version'])

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.7.100-4) unstable; urgency=medium
+  [ Jerome Soumagne ]
+  * Bump mercury version to 2.4.0rc4
+
+ -- Jerome Soumagne <[email protected]>  Mon, 05 Aug 2024 12:00:00 -0500
+
 daos (2.7.100-3) unstable; urgency=medium
   [ Dalton Bohning ]
   * Add pciutils-devel build dep for client-tests package

debian/control

@@ -10,7 +10,7 @@ Build-Depends: debhelper (>= 10),
                libopenmpi-dev,
                libssl-dev,
                libyaml-dev,
-               libmercury-dev (>= 2.3.1-1),
+               libmercury-dev (>= 2.4),
                scons,
                uuid-dev,
                pkg-config,

site_scons/prereq_tools/base.py

@@ -407,13 +407,15 @@ class PreReqComponent():
 
     def __init__(self, env, opts):
         self.__defined = {}
+        self.__ = {}
         self.__required = {}
         self.__errors = {}
         self.__env = env
         self.__dry_run = GetOption('no_exec')
         self.__require_optional = GetOption('require_optional')
         self._has_icx = False
         self.download_deps = False
+        self.fetch_only = False
         self.build_deps = False
         self.__parse_build_deps()
         self._replace_env(LIBTOOLIZE='libtoolize')
@@ -436,7 +438,7 @@ def __init__(self, env, opts):
         RUNNER.initialize(self.__env)
 
         opts.Add(PathVariable('PREFIX', 'Installation path', install_dir,
-                              PathVariable.PathIsDirCreate))
+                              PathVariable.PathAccept))
         opts.Add('ALT_PREFIX', f'Specifies {os.pathsep} separated list of alternative paths to add',
                  None)
         opts.Add(PathVariable('BUILD_ROOT', 'Alternative build root directory', "build",
@@ -488,6 +490,8 @@ def __init__(self, env, opts):
         self._setup_path_var('GOPATH')
         self.__build_info.update("PREFIX", self.__env.subst("$PREFIX"))
         self.prereq_prefix = self.__env.subst("$PREFIX/prereq/$TTYPE_REAL")
+        if GetOption('install_sandbox'):
+            self.prereq_prefix = f"{GetOption('install_sandbox')}/{self.prereq_prefix}"
 
         if config_file is not None:
             self._configs = configparser.ConfigParser()
@@ -549,7 +553,14 @@ def run_build(self, opts):
 
         # Go ahead and prebuild some components
         for comp in reqs:
-            self.__env.Clone().require(comp)
+            if self.fetch_only:
+                self.download(comp)
+            else:
+                self.__env.Clone().require(comp)
+
+        if self.fetch_only:
+            print("--build-deps=fetch was set, so exiting...")
+            sys.exit(0)
 
     def _setup_build_type(self):
         """Set build type"""
@@ -676,11 +687,13 @@ def save_build_info(self):
     def __parse_build_deps(self):
         """Parse the build dependencies command line flag"""
         build_deps = GetOption('build_deps')
-        if build_deps in ('yes', 'only'):
-            self.download_deps = True
-            self.build_deps = True
-        elif build_deps == 'build-only':
+        skip_download = GetOption('skip_download')
+        if build_deps in ('fetch',):
+            self.fetch_only = True
+        elif build_deps in ('yes', 'only'):
             self.build_deps = True
+            if not skip_download:
+                self.download_deps = True
 
     def _sub_path(self, path):
         """Resolve the real path"""
@@ -747,6 +760,18 @@ def _modify_prefix(self, comp_def):
            not os.path.exists(self.__env.get(f'{comp_def.name.upper()}_PREFIX')):
             self._save_component_prefix(f'{comp_def.name.upper()}_PREFIX', '/usr')
 
+    def download(self, *comps):
+        """Ensure all components are downloaded"""
+
+        for comp in comps:
+            if comp not in self.__defined:
+                raise MissingDefinition(comp)
+            if comp in self.__errors:
+                raise self.__errors[comp]
+            comp_def = self.__defined[comp]
+            comp_def.configure()
+            comp_def.get()
+
     def require(self, env, *comps, **kw):
         """Ensure a component is built.
 
@@ -861,18 +886,20 @@ def get_prebuilt_path(self, comp, name):
             lpath = None
             for lib in ['lib64', 'lib']:
                 lpath = os.path.join(path, lib)
-                if not os.path.exists(lpath):
-                    lpath = None
+                if os.path.exists(lpath):
+                    break
+                lpath = None
             if ipath is None and lpath is None:
                 continue
             env = self.__env.Clone()
             if ipath:
                 env.AppendUnique(CPPPATH=[ipath])
             if lpath:
                 env.AppendUnique(LIBPATH=[lpath])
-            if not comp.has_missing_targets(env):
-                self.__prebuilt_path[name] = path
-                return path
+            realpath = os.path.realpath(path)
+            if not comp.has_missing_targets(env, realpath):
+                self.__prebuilt_path[name] = realpath
+                return self.__prebuilt_path[name]
 
         self.__prebuilt_path[name] = None
 
@@ -1051,9 +1078,10 @@ def get(self):
             return
 
         # Source code is retrieved using retriever
-
-        if not self.prereqs.download_deps:
-            raise DownloadRequired(self.name)
+        if not (self.prereqs.download_deps or self.prereqs.fetch_only):
+            if self.prereqs.build_deps:
+                print("Assuming sources have been downloaded already")
+                return
 
         print(f'Downloading source for {self.name}')
         patches = self._resolve_patches()
@@ -1095,18 +1123,23 @@ def _has_missing_system_deps(self, env):
             env.SetOption('no_exec', True)
         return False
 
-    def _parse_config(self, env, opts):
+    def _parse_config(self, env, opts, comp_path=None):
         """Parse a pkg-config file"""
         if self.pkgconfig is None:
             return
+
+        real_comp_path = self.component_prefix
+        if comp_path:
+            real_comp_path = comp_path
+
         path = os.environ.get("PKG_CONFIG_PATH", None)
         if path and "PKG_CONFIG_PATH" not in env["ENV"]:
             env["ENV"]["PKG_CONFIG_PATH"] = path
-        if (not self.use_installed and self.component_prefix is not None
-           and not self.component_prefix == "/usr"):
+        if (not self.use_installed and real_comp_path is not None
+           and not real_comp_path == "/usr"):
             path_found = False
             for path in ["lib", "lib64"]:
-                config = os.path.join(self.component_prefix, path, "pkgconfig")
+                config = os.path.join(real_comp_path, path, "pkgconfig")
                 if not os.path.exists(config):
                     continue
                 path_found = True
@@ -1126,7 +1159,7 @@ def _print(self, msg):
             return
         print(msg)
 
-    def has_missing_targets(self, env):
+    def has_missing_targets(self, env, comp_path=None):
         """Check for expected build targets (e.g. libraries or headers)"""
         # pylint: disable=too-many-return-statements
         if self.targets_found:
@@ -1146,7 +1179,10 @@ def has_missing_targets(self, env):
             return True
 
         # No need to fail here if we can't find the config, it may not always be generated
-        self._parse_config(env, "--cflags")
+        self._parse_config(env, "--cflags --libs-only-L", comp_path=comp_path)
+
+        for define in self.defines:
+            env.AppendUnique(CPPDEFINES=[define])
 
         if GetOption('help'):
             print('help set')

src/cart/crt_context.c

@@ -416,6 +416,9 @@ crt_iface_name2idx(const char *iface_name, int *idx)
 	for (i = 0; i < num_ifaces; i++) {
 		name = crt_provider_iface_str_get(true, crt_gdata.cg_primary_prov, i);
 
+		if (!name)
+			return -DER_INVAL;
+
 		if (strcmp(name, iface_name) == 0) {
 			*idx = i;
 			return DER_SUCCESS;

src/cart/crt_hg.c

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -488,14 +488,6 @@ crt_provider_ctx0_port_get(bool primary, crt_provider_t provider)
 	return prov_data->cpg_na_config.noc_port;
 }
 
-static char*
-crt_provider_domain_get(bool primary, crt_provider_t provider)
-{
-	struct crt_prov_gdata *prov_data = crt_get_prov_gdata(primary, provider);
-
-	return prov_data->cpg_na_config.noc_domain;
-}
-
 static struct crt_na_dict *
 crt_get_na_dict_entry(crt_provider_t provider)
 {
@@ -518,6 +510,19 @@ crt_provider_name_get(crt_provider_t provider)
 	return entry ? entry->nad_str : NULL;
 }
 
+char *
+crt_provider_domain_str_get(bool primary, crt_provider_t provider, int idx)
+{
+	struct crt_prov_gdata *prov_data = crt_get_prov_gdata(primary, provider);
+
+	/* If the domain was not specified, return NULL */
+	if (prov_data->cpg_na_config.noc_domain == NULL)
+		return NULL;
+
+	D_ASSERTF(idx < prov_data->cpg_na_config.noc_domain_total, "Bad idx=%d\n", idx);
+	return prov_data->cpg_na_config.noc_domain_str[idx];
+}
+
 char*
 crt_provider_iface_str_get(bool primary, crt_provider_t provider, int iface_idx)
 {
@@ -527,16 +532,8 @@ crt_provider_iface_str_get(bool primary, crt_provider_t provider, int iface_idx)
 	if (prov_data->cpg_na_config.noc_interface == NULL)
 		return NULL;
 
-	/*
- 	 * CXI provider requires domain names instead of interfaces.
- 	 * Returning NULL here will cause crt_get_info_string() to use domain names instead
-	 * */
-	if (provider == CRT_PROV_OFI_CXI)
-		return NULL;
-
-	D_ASSERTF(iface_idx < prov_data->cpg_na_config.noc_iface_total,
-		  "Bad iface_idx=%d\n", iface_idx);
-
+	D_ASSERTF(iface_idx < prov_data->cpg_na_config.noc_iface_total, "Bad iface_idx=%d\n",
+		  iface_idx);
 	return prov_data->cpg_na_config.noc_iface_str[iface_idx];
 }
 
@@ -712,12 +709,18 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx,
 	int	 start_port;
 	char	*domain_str;
 	char	*iface_str;
+	bool     no_iface, no_domain;
 	int	rc = 0;
 
 	provider_str = crt_provider_name_get(provider);
 	start_port = crt_provider_ctx0_port_get(primary, provider);
-	domain_str = crt_provider_domain_get(primary, provider);
-	iface_str = crt_provider_iface_str_get(primary, provider, iface_idx);
+	domain_str   = crt_provider_domain_str_get(primary, provider, iface_idx);
+
+	/* CXI provider uses domain names for info string */
+	if (provider == CRT_PROV_OFI_CXI)
+		iface_str = NULL;
+	else
+		iface_str = crt_provider_iface_str_get(primary, provider, iface_idx);
 
 	if (provider == CRT_PROV_SM) {
 		D_ASPRINTF(*string, "%s://", provider_str);
@@ -731,42 +734,45 @@ crt_get_info_string(bool primary, crt_provider_t provider, int iface_idx,
 		D_GOTO(out, rc);
 	}
 
+	/* treat not set and set to empty as the same */
+	no_iface  = (iface_str == NULL || *iface_str == '\0') ? true : false;
+	no_domain = (domain_str == NULL || *domain_str == '\0') ? true : false;
+
 	/* TODO: for now pass same info for all providers including CXI */
 	if (crt_provider_is_contig_ep(provider) && start_port != -1) {
-
-		if (iface_str == NULL) {
-			if (domain_str)
-				D_ASPRINTF(*string, "%s://%s:%d",
-					   provider_str, domain_str, start_port + ctx_idx);
+		if (no_iface) {
+			if (no_domain)
+				D_ASPRINTF(*string, "%s://:%d", provider_str, start_port + ctx_idx);
 			else
-				D_ASPRINTF(*string, "%s://:%d",
-					   provider_str, start_port + ctx_idx);
-		} else {
-			if (domain_str)
-				D_ASPRINTF(*string, "%s://%s/%s:%d",
-					   provider_str, domain_str, iface_str,
+				D_ASPRINTF(*string, "%s://%s:%d", provider_str, domain_str,
 					   start_port + ctx_idx);
-			else
+		} else {
+			if (no_domain)
 				D_ASPRINTF(*string, "%s://%s:%d",
 					   provider_str, iface_str,
 					   start_port + ctx_idx);
+			else
+				D_ASPRINTF(*string, "%s://%s/%s:%d", provider_str, domain_str,
+					   iface_str, start_port + ctx_idx);
 		}
 	} else {
-		if (iface_str == NULL) {
-			if (domain_str)
-				D_ASPRINTF(*string, "%s://%s",
-					   provider_str, domain_str);
-			else
+		if (no_iface) {
+			if (no_domain)
 				D_ASPRINTF(*string, "%s://", provider_str);
-		} else {
-			if (domain_str)
-				D_ASPRINTF(*string, "%s://%s/%s",
-					   provider_str, domain_str, iface_str);
 			else
+				D_ASPRINTF(*string, "%s://%s", provider_str, domain_str);
+		} else {
+			if (no_domain)
 				D_ASPRINTF(*string, "%s://%s", provider_str, iface_str);
+			else
+				D_ASPRINTF(*string, "%s://%s/%s", provider_str, domain_str,
+					   iface_str);
 		}
 	}
 
+	D_DEBUG(DB_ALL, "iface_idx:%d context:%d domain_str=%s iface_str=%s info_str=%s\n",
+		iface_idx, ctx_idx, domain_str ? domain_str : "none",
+		iface_str ? iface_str : "none", *string);
 out:
 	if (rc == DER_SUCCESS && *string == NULL)
 		return -DER_NOMEM;