clp-package: Include build-platform's /etc/os-release and determine…

… `execution_container` from it. (#322)

Taskfile.yml

@@ -78,6 +78,7 @@ tasks:
       - task: "clean-package"
       - "mkdir -p '{{.OUTPUT_DIR}}'"
       - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'"
+      - "rsync --copy-links /etc/os-release '{{.OUTPUT_DIR}}/etc/os-release'"
       - "mkdir -p '{{.OUTPUT_DIR}}/lib/python3/site-packages'"
       - |-
         . "{{.PACKAGE_VENV_DIR}}/bin/activate"
@@ -116,6 +117,7 @@ tasks:
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clp-s"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "/etc/os-release"
       - "components/clp-package-utils/dist/*.whl"
       - "components/clp-py-utils/dist/*.whl"
       - "components/job-orchestration/dist/*.whl"
@@ -141,6 +143,7 @@ tasks:
       - "{{.SRC_DIR}}/CMakeLists.txt"
       - "{{.SRC_DIR}}/src/**/*"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "/etc/os-release"
     generates:
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
@@ -272,6 +275,7 @@ tasks:
     sources:
       - "{{.TASKFILE_DIR}}/requirements.txt"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "/etc/os-release"
     status:
       - "test -f '{{.CHECKSUM_FILE}}'"
       - >-
@@ -306,6 +310,7 @@ tasks:
       - "{{.PACKAGE}}/**/*"
       - "{{.TASKFILE_DIR}}/requirements.txt"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "/etc/os-release"
       - "pyproject.toml"
     generates:
       - "dist/*.whl"
@@ -331,6 +336,7 @@ tasks:
     sources:
       - "{{.TASKFILE_DIR}}/requirements.txt"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "/etc/os-release"
       - "pyproject.toml"
     status:
       - "test -f '{{.CHECKSUM_FILE}}'"

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -785,6 +785,8 @@ def main(argv):
             config_file_path, default_config_file_path, clp_home
         )
 
+        clp_config.load_execution_container_name()
+
         # Validate and load necessary credentials
         if target in (
             ALL_TARGET_NAME,

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -2,7 +2,8 @@
 import typing
 from enum import auto
 
-from pydantic import BaseModel, validator
+from dotenv import dotenv_values
+from pydantic import BaseModel, PrivateAttr, validator
 from strenum import KebabCaseStrEnum
 
 from .clp_logging import get_valid_logging_level, is_valid_logging_level
@@ -33,6 +34,8 @@
 COMPRESSION_JOBS_TABLE_NAME = "compression_jobs"
 COMPRESSION_TASKS_TABLE_NAME = "compression_tasks"
 
+OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release"
+
 CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml"
 CLP_METADATA_TABLE_PREFIX = "clp_"
 
@@ -301,7 +304,7 @@ def validate_logging_level(cls, field):
 
 
 class CLPConfig(BaseModel):
-    execution_container: str = "ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main"
+    execution_container: typing.Optional[str]
 
     input_logs_directory: pathlib.Path = pathlib.Path("/")
 
@@ -321,12 +324,15 @@ class CLPConfig(BaseModel):
     data_directory: pathlib.Path = pathlib.Path("var") / "data"
     logs_directory: pathlib.Path = pathlib.Path("var") / "log"
 
+    _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH)
+
     def make_config_paths_absolute(self, clp_home: pathlib.Path):
         self.input_logs_directory = make_config_path_absolute(clp_home, self.input_logs_directory)
         self.credentials_file_path = make_config_path_absolute(clp_home, self.credentials_file_path)
         self.archive_output.make_config_paths_absolute(clp_home)
         self.data_directory = make_config_path_absolute(clp_home, self.data_directory)
         self.logs_directory = make_config_path_absolute(clp_home, self.logs_directory)
+        self._os_release_file_path = make_config_path_absolute(clp_home, self._os_release_file_path)
 
     def validate_input_logs_dir(self):
         # NOTE: This can't be a pydantic validator since input_logs_dir might be a package-relative
@@ -355,6 +361,23 @@ def validate_logs_dir(self):
         except ValueError as ex:
             raise ValueError(f"logs_directory is invalid: {ex}")
 
+    def load_execution_container_name(self):
+        if self.execution_container is not None:
+            # Accept configured value for debug purposes
+            return
+
+        os_release = dotenv_values(self._os_release_file_path)
+        if "ubuntu" == os_release["ID"]:
+            self.execution_container = (
+                f"clp-execution-x86-{os_release['ID']}-{os_release['VERSION_CODENAME']}:main"
+            )
+        else:
+            raise NotImplementedError(
+                f"Unsupported OS {os_release['ID']} in {OS_RELEASE_FILE_PATH}"
+            )
+
+        self.execution_container = "ghcr.io/y-scope/clp/" + self.execution_container
+
     def load_database_credentials_from_file(self):
         config = read_yaml_config_file(self.credentials_file_path)
         if config is None: