From 7e3b26adffb1c0014ecef2ba109caaae644340e7 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 19 Feb 2024 16:44:18 +0000 Subject: [PATCH 01/13] Sanity check for python configuration --- easybuild/easyblocks/generic/cmakemake.py | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 3074193df0..9465f792d1 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -72,6 +72,22 @@ def det_cmake_version(): return cmake_version + ####Sanity check to verify which Python executable is picked up by CMake + cmake_python_check_cmd = ' cmake -E env python --version' + python_version_output, _ = run_cmd(cmake_python_check_cmd, log_all=True, simple=False) + + ####Extract Python from the output + python_version = None + if 'Python ' in python_version_output: + python_version = python_version_output.split('Python ')[1].strip() + + #####Log the Python version detected by CMake + if python_version: + self.log.info("Python version detected by CMake: %s", python_version) + else: + self.log.warning("Failed to determine Python version detected by CMake.") + + def setup_cmake_env(tc): """Setup env variables that cmake needs in an EasyBuild context.""" @@ -278,6 +294,29 @@ def configure_step(self, srcdir=None, builddir=None): if LooseVersion(self.cmake_version) >= '3.15': options['CMAKE_POLICY_DEFAULT_CMP0094'] = 'NEW' + #####Set Python_FIND_STRATEGY and Python_ROOT_DIR based on CMake and Python versions + if LooseVersion(self.cmake_version) >= '3.12': + # For CMake version 3.12 and above, use the new variables + options['Python_FIND_STRATEGY'] = 'LOCATION' + + # Check if Python version is detected by CMake and update Python_ROOT_DIR accordingly + if python_version is not None: + options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', f'python{python_version}') + else: + # Use the default Python version + options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', 'python') + + elif LooseVersion(self.cmake_version) < '3.12': + options['Python_FIND_STRATEGY'] = 'LOCATION' + + # Check if Python version is detected by CMake and update Python_ROOT_DIR accordingly + if python_version is not None: + options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', f'python{python_version}') + else: + # Use the default Python version + options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROPYTHON', ''), 'bin', 'python') + ######### + # show what CMake is doing by default options['CMAKE_VERBOSE_MAKEFILE'] = 'ON' From 40986c8120776c82ebb579506ac83f3744032b31 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 26 Feb 2024 16:49:28 +0000 Subject: [PATCH 02/13] Sanity check for Python --- easybuild/easyblocks/generic/cmakemake.py | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 9465f792d1..d3dc9520c2 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -357,8 +357,31 @@ def configure_step(self, srcdir=None, builddir=None): (out, _) = run_cmd(command, log_all=True, simple=False) - return out +def check_python_paths(file_path): + with open(file_path, 'r') as file: + lines = file.readlines() + + # Search for Python paths in each line + for line in lines: + if line.startswith('Python3_EXECUTABLE:FILEPATH='): + exec_path = line.split('=')[1].strip() + elif line.startswith('Python3_INCLUDE_DIR:FILEPATH='): + include_path = line.split('=')[1].strip() + elif line.startswith('Python3_LIBRARY:FILEPATH='): + library_path = line.split('=')[1].strip() + + # Check if paths include EBROOTPYTHON + ebrootpython_path = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/skylake_avx512/software/Python/3.11.3-GCCcore-12.3.0" + if all(path and ebrootpython_path in path for path in [exec_path, include_path, library_path]): + print("Paths include EBROOTPYTHON.") + else: + print("Paths do not include EBROOTPYTHON.") + + cmake_cache_path = 'CMakeCache.txt' + check_python_paths(cmake_cache_path) + return out + def test_step(self): """CMake specific test setup""" # When using ctest for tests (default) then show verbose output if a test fails From 133e3b88176b3f769fa1193a97d7261c885c5137 Mon Sep 17 00:00:00 2001 From: Danilo Gonzalez Date: Tue, 27 Feb 2024 11:48:12 +0100 Subject: [PATCH 03/13] Remove legacy code, integrate python sanity check into configure section --- easybuild/easyblocks/generic/cmakemake.py | 87 +++++++---------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index d3dc9520c2..bf93848aa6 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -72,22 +72,6 @@ def det_cmake_version(): return cmake_version - ####Sanity check to verify which Python executable is picked up by CMake - cmake_python_check_cmd = ' cmake -E env python --version' - python_version_output, _ = run_cmd(cmake_python_check_cmd, log_all=True, simple=False) - - ####Extract Python from the output - python_version = None - if 'Python ' in python_version_output: - python_version = python_version_output.split('Python ')[1].strip() - - #####Log the Python version detected by CMake - if python_version: - self.log.info("Python version detected by CMake: %s", python_version) - else: - self.log.warning("Failed to determine Python version detected by CMake.") - - def setup_cmake_env(tc): """Setup env variables that cmake needs in an EasyBuild context.""" @@ -294,29 +278,6 @@ def configure_step(self, srcdir=None, builddir=None): if LooseVersion(self.cmake_version) >= '3.15': options['CMAKE_POLICY_DEFAULT_CMP0094'] = 'NEW' - #####Set Python_FIND_STRATEGY and Python_ROOT_DIR based on CMake and Python versions - if LooseVersion(self.cmake_version) >= '3.12': - # For CMake version 3.12 and above, use the new variables - options['Python_FIND_STRATEGY'] = 'LOCATION' - - # Check if Python version is detected by CMake and update Python_ROOT_DIR accordingly - if python_version is not None: - options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', f'python{python_version}') - else: - # Use the default Python version - options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', 'python') - - elif LooseVersion(self.cmake_version) < '3.12': - options['Python_FIND_STRATEGY'] = 'LOCATION' - - # Check if Python version is detected by CMake and update Python_ROOT_DIR accordingly - if python_version is not None: - options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROTPYTHON', ''), 'bin', f'python{python_version}') - else: - # Use the default Python version - options['Python_ROOT_DIR'] = os.path.join(os.environ.get('EBROPYTHON', ''), 'bin', 'python') - ######### - # show what CMake is doing by default options['CMAKE_VERBOSE_MAKEFILE'] = 'ON' @@ -357,31 +318,31 @@ def configure_step(self, srcdir=None, builddir=None): (out, _) = run_cmd(command, log_all=True, simple=False) -def check_python_paths(file_path): - with open(file_path, 'r') as file: - lines = file.readlines() - - # Search for Python paths in each line - for line in lines: - if line.startswith('Python3_EXECUTABLE:FILEPATH='): - exec_path = line.split('=')[1].strip() - elif line.startswith('Python3_INCLUDE_DIR:FILEPATH='): - include_path = line.split('=')[1].strip() - elif line.startswith('Python3_LIBRARY:FILEPATH='): - library_path = line.split('=')[1].strip() - - # Check if paths include EBROOTPYTHON - ebrootpython_path = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/skylake_avx512/software/Python/3.11.3-GCCcore-12.3.0" - if all(path and ebrootpython_path in path for path in [exec_path, include_path, library_path]): - print("Paths include EBROOTPYTHON.") - else: - print("Paths do not include EBROOTPYTHON.") - - cmake_cache_path = 'CMakeCache.txt' - check_python_paths(cmake_cache_path) - - return out + # sanitycheck for the configuration step + self.log.info("Checking python paths") + with open('CMakeCache.txt', 'r') as file: + lines = file.readlines() + + # Search for Python paths in each line + for line in lines: + if line.startswith('Python3_EXECUTABLE:FILEPATH='): + pythonExecPath = line.split('=')[1].strip() + elif line.startswith('Python3_INCLUDE_DIR:FILEPATH='): + pythonIncludePath = line.split('=')[1].strip() + elif line.startswith('Python3_LIBRARY:FILEPATH='): + pythonLibraryPath = line.split('=')[1].strip() + self.log.info("Python executable path: %s",pythonExecPath) + self.log.info("Python include path: %s",pythonIncludePath) + self.log.info("Python library path: %s",pythonLibraryPath) + # Check if paths include EBROOTPYTHON + ebrootpython_path = os.getenv('EBROOTPYTHON') + if all(path and ebrootpython_path in path for path in [pythonExecPath,pythonIncludePath,pythonLibraryPath]): + self.log.info("Python related paths configured correctly.") + return out + else: + raise EasyBuildError("Python related paths do not include EBROOTPYTHON, check log") + def test_step(self): """CMake specific test setup""" # When using ctest for tests (default) then show verbose output if a test fails From 775dde0cf514ec4364dc74e237cd1410b0838dd8 Mon Sep 17 00:00:00 2001 From: MartinsNadia <153844104+MartinsNadia@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:42:35 +0000 Subject: [PATCH 04/13] Update cmakemake.py removing/adding blank spaces --- easybuild/easyblocks/generic/cmakemake.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index bf93848aa6..9425d86d01 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -331,18 +331,16 @@ def configure_step(self, srcdir=None, builddir=None): pythonIncludePath = line.split('=')[1].strip() elif line.startswith('Python3_LIBRARY:FILEPATH='): pythonLibraryPath = line.split('=')[1].strip() - self.log.info("Python executable path: %s",pythonExecPath) - self.log.info("Python include path: %s",pythonIncludePath) - self.log.info("Python library path: %s",pythonLibraryPath) - + self.log.info("Python executable path: %s", pythonExecPath) + self.log.info("Python include path: %s", pythonIncludePath) + self.log.info("Python library path: %s", pythonLibraryPath) # Check if paths include EBROOTPYTHON ebrootpython_path = os.getenv('EBROOTPYTHON') - if all(path and ebrootpython_path in path for path in [pythonExecPath,pythonIncludePath,pythonLibraryPath]): + if all(path and ebrootpython_path in path for path in [pythonExecPath, pythonIncludePath, pythonLibraryPath]): self.log.info("Python related paths configured correctly.") return out else: raise EasyBuildError("Python related paths do not include EBROOTPYTHON, check log") - def test_step(self): """CMake specific test setup""" # When using ctest for tests (default) then show verbose output if a test fails From 067c4cde605f90a59c8e6a41c8806d313991d842 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 18 Mar 2024 15:30:43 +0000 Subject: [PATCH 05/13] teste --- easybuild/easyblocks/generic/cmakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 9425d86d01..0f28f5b7d1 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -258,7 +258,7 @@ def configure_step(self, srcdir=None, builddir=None): 'CXXFLAGS': 'CMAKE_CXX_FLAGS', 'FFLAGS': 'CMAKE_Fortran_FLAGS', }) - +#teste for env_name, option in env_to_options.items(): value = os.getenv(env_name) if value is not None: From 16b6465fe54c6c9795ed74b6d0f30a1a21c05865 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 18 Mar 2024 15:32:36 +0000 Subject: [PATCH 06/13] Implementation of path validation for sanitycheck --- easybuild/easyblocks/generic/cmakemake.py | 85 +++++++++++++++++------ 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0f28f5b7d1..4a9dacb0a7 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -160,6 +160,7 @@ def prepend_config_opts(self, config_opts): if '-D%s=' % key not in cfg_configopts) self.cfg['configopts'] = ' '.join([new_opts, cfg_configopts]) + def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" @@ -258,7 +259,7 @@ def configure_step(self, srcdir=None, builddir=None): 'CXXFLAGS': 'CMAKE_CXX_FLAGS', 'FFLAGS': 'CMAKE_Fortran_FLAGS', }) -#teste + for env_name, option in env_to_options.items(): value = os.getenv(env_name) if value is not None: @@ -318,29 +319,71 @@ def configure_step(self, srcdir=None, builddir=None): (out, _) = run_cmd(command, log_all=True, simple=False) - # sanitycheck for the configuration step - self.log.info("Checking python paths") +def sanitycheck_for_configuration(self, out): + self.log.info("Checking Python paths") + + if LooseVersion(self.cmake_version) >= '3.16' + try: with open('CMakeCache.txt', 'r') as file: lines = file.readlines() + # The case where CMakeCache.txt is not found + except FileNotFoundError: + self.log.warning("CMakeCache.txt not founf. Python paths checks skipped.") + return + # EBROOTPYTHON check + if not os.getenv('EBROOTPYTHON'): + self.log.warning("EBROOTPYTHON is not set.") + return + else: + self.log.info ("Skipping Python path checks as CMake version might be lower than 3.16.") + return + + # Initialize empty lists to store paths + python_exec_path = [] + python_include_path = [] + python_library_path = [] + + # Define a regular expression to match all relevant prefixes + python_prefixes = r"(Python|Python2|Python3)_" + + for line in lines: + if line.startswith(python_prefixes + r"EXECUTABLE:FILEPATH="): + python_exec_paths.append(line.split('=')[1].strip()) + self.log.info("Python executable path: %s", python_exec_path[-1]) + elif line.startswith(python_prefixes + r"INCLUDE_DIR:FILEPATH="): + python_include_paths.append(line.split('=')[1].strip()) + self.log.info("Python include path: %s", python_include_path[-1]) + elif line.startswith(python_prefixes + r"LIBRARY:FILEPATH="): + python_library_paths.append(line.split('=')[1].strip()) + self.log.info("Python library path: %s", python_library_path[-1]) + + # Check if paths are found and validate paths for symlinks + for path_list in [python_exec_path, python_include_path, python_library_path]: + for path in path_list: + if path and not os.path.exists(path): + raise EasyBuildError(f"Path does not exist: {path}") + + # Check if paths are found and handle EBROOTPYTHON cases + if any(path for path in [python_exec_path, python_include_path, python_library_path]): + if not os.getenv('EBROOTPYTHON'): + self.log.error("EBROOTPYTHON is not set and Python related paths are found.") + elif not all(path and os.getenv('EBROOTPYTHON') in path for path in [python_exec_path, python_include_path, python_library_path]): + self.log.error("Python related paths do not include EBROOTPYTHON.") + else: + self.log.info("Python related paths configured correctly.") + else: + # Handle cases where no Python paths are found + self.log.error("No Python related paths found in CMakeCache.txt.") + raise EasyBuildError("Python related paths not configured correctly.") + + ebrootpython_path = get_software_root("Python") + + # Validate that Python paths are consistent with EBROOTPYTHON + if python_exec_path and not ebrootpython_path: + raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") + + return out - # Search for Python paths in each line - for line in lines: - if line.startswith('Python3_EXECUTABLE:FILEPATH='): - pythonExecPath = line.split('=')[1].strip() - elif line.startswith('Python3_INCLUDE_DIR:FILEPATH='): - pythonIncludePath = line.split('=')[1].strip() - elif line.startswith('Python3_LIBRARY:FILEPATH='): - pythonLibraryPath = line.split('=')[1].strip() - self.log.info("Python executable path: %s", pythonExecPath) - self.log.info("Python include path: %s", pythonIncludePath) - self.log.info("Python library path: %s", pythonLibraryPath) - # Check if paths include EBROOTPYTHON - ebrootpython_path = os.getenv('EBROOTPYTHON') - if all(path and ebrootpython_path in path for path in [pythonExecPath, pythonIncludePath, pythonLibraryPath]): - self.log.info("Python related paths configured correctly.") - return out - else: - raise EasyBuildError("Python related paths do not include EBROOTPYTHON, check log") def test_step(self): """CMake specific test setup""" # When using ctest for tests (default) then show verbose output if a test fails From cfbe5e598d30fec5e51ec7ea118498c36f0eacc1 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Fri, 22 Mar 2024 14:36:47 +0000 Subject: [PATCH 07/13] Fix identation --- easybuild/easyblocks/generic/cmakemake.py | 42 +++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 4a9dacb0a7..b486d1088f 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -319,12 +319,12 @@ def configure_step(self, srcdir=None, builddir=None): (out, _) = run_cmd(command, log_all=True, simple=False) -def sanitycheck_for_configuration(self, out): - self.log.info("Checking Python paths") + def sanitycheck_for_configuration(self, out): + self.log.info("Checking Python paths") - if LooseVersion(self.cmake_version) >= '3.16' - try: - with open('CMakeCache.txt', 'r') as file: + if LooseVersion(self.cmake_version) >= '3.16' + try: + with open('CMakeCache.txt', 'r') as file: lines = file.readlines() # The case where CMakeCache.txt is not found except FileNotFoundError: @@ -334,9 +334,9 @@ def sanitycheck_for_configuration(self, out): if not os.getenv('EBROOTPYTHON'): self.log.warning("EBROOTPYTHON is not set.") return - else: - self.log.info ("Skipping Python path checks as CMake version might be lower than 3.16.") - return + else: + self.log.info ("Skipping Python path checks as CMake version might be lower than 3.16.") + return # Initialize empty lists to store paths python_exec_path = [] @@ -359,29 +359,29 @@ def sanitycheck_for_configuration(self, out): # Check if paths are found and validate paths for symlinks for path_list in [python_exec_path, python_include_path, python_library_path]: - for path in path_list: - if path and not os.path.exists(path): - raise EasyBuildError(f"Path does not exist: {path}") + for path in path_list: + if path and not os.path.exists(path): + raise EasyBuildError(f"Path does not exist: {path}") # Check if paths are found and handle EBROOTPYTHON cases if any(path for path in [python_exec_path, python_include_path, python_library_path]): - if not os.getenv('EBROOTPYTHON'): - self.log.error("EBROOTPYTHON is not set and Python related paths are found.") - elif not all(path and os.getenv('EBROOTPYTHON') in path for path in [python_exec_path, python_include_path, python_library_path]): - self.log.error("Python related paths do not include EBROOTPYTHON.") - else: - self.log.info("Python related paths configured correctly.") + if not os.getenv('EBROOTPYTHON'): + self.log.error("EBROOTPYTHON is not set and Python related paths are found.") + elif not all(path and os.getenv('EBROOTPYTHON') in path for path in [python_exec_path, python_include_path, python_library_path]): + self.log.error("Python related paths do not include EBROOTPYTHON.") + else: + self.log.info("Python related paths configured correctly.") else: # Handle cases where no Python paths are found - self.log.error("No Python related paths found in CMakeCache.txt.") - raise EasyBuildError("Python related paths not configured correctly.") + self.log.error("No Python related paths found in CMakeCache.txt.") + raise EasyBuildError("Python related paths not configured correctly.") ebrootpython_path = get_software_root("Python") # Validate that Python paths are consistent with EBROOTPYTHON if python_exec_path and not ebrootpython_path: - raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") - + raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") + return out def test_step(self): From 3d827873a3c0b9f956262df711047b0fa2860cdf Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Fri, 22 Mar 2024 15:42:10 +0000 Subject: [PATCH 08/13] Formatting improvements --- easybuild/easyblocks/generic/cmakemake.py | 114 +++++++++++----------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index b486d1088f..444f5b6f78 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -322,67 +322,67 @@ def configure_step(self, srcdir=None, builddir=None): def sanitycheck_for_configuration(self, out): self.log.info("Checking Python paths") - if LooseVersion(self.cmake_version) >= '3.16' - try: - with open('CMakeCache.txt', 'r') as file: - lines = file.readlines() - # The case where CMakeCache.txt is not found - except FileNotFoundError: - self.log.warning("CMakeCache.txt not founf. Python paths checks skipped.") - return - # EBROOTPYTHON check - if not os.getenv('EBROOTPYTHON'): - self.log.warning("EBROOTPYTHON is not set.") - return - else: - self.log.info ("Skipping Python path checks as CMake version might be lower than 3.16.") - return - - # Initialize empty lists to store paths - python_exec_path = [] - python_include_path = [] - python_library_path = [] - - # Define a regular expression to match all relevant prefixes - python_prefixes = r"(Python|Python2|Python3)_" - - for line in lines: - if line.startswith(python_prefixes + r"EXECUTABLE:FILEPATH="): - python_exec_paths.append(line.split('=')[1].strip()) - self.log.info("Python executable path: %s", python_exec_path[-1]) - elif line.startswith(python_prefixes + r"INCLUDE_DIR:FILEPATH="): - python_include_paths.append(line.split('=')[1].strip()) - self.log.info("Python include path: %s", python_include_path[-1]) - elif line.startswith(python_prefixes + r"LIBRARY:FILEPATH="): - python_library_paths.append(line.split('=')[1].strip()) - self.log.info("Python library path: %s", python_library_path[-1]) + if LooseVersion(self.cmake_version) >= '3.16': + try: + with open('CMakeCache.txt', 'r') as file: + lines = file.readlines() + except FileNotFoundError: + self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") + return + + if not os.getenv('EBROOTPYTHON'): + self.log.warning("EBROOTPYTHON is not set.") + return + else: + self.log.info("Skipping Python path checks as CMake version might be lower than 3.16.") + return + + python_exec_path = [] + python_include_path = [] + python_library_path = [] + + # Define a regular expression to match all relevant prefixes + python_prefixes = r"(Python|Python2|Python3)_" + + for line in lines: + if line.startswith(python_prefixes + r"EXECUTABLE:FILEPATH="): + python_exec_path.append(line.split('=')[1].strip()) + self.log.info("Python executable path: %s", python_exec_path[-1]) + elif line.startswith(python_prefixes + r"INCLUDE_DIR:FILEPATH="): + python_include_path.append(line.split('=')[1].strip()) + self.log.info("Python include path: %s", python_include_path[-1]) + elif line.startswith(python_prefixes + r"LIBRARY:FILEPATH="): + python_library_path.append(line.split('=')[1].strip()) + self.log.info("Python library path: %s", python_library_path[-1]) - # Check if paths are found and validate paths for symlinks - for path_list in [python_exec_path, python_include_path, python_library_path]: - for path in path_list: - if path and not os.path.exists(path): - raise EasyBuildError(f"Path does not exist: {path}") - - # Check if paths are found and handle EBROOTPYTHON cases - if any(path for path in [python_exec_path, python_include_path, python_library_path]): - if not os.getenv('EBROOTPYTHON'): - self.log.error("EBROOTPYTHON is not set and Python related paths are found.") - elif not all(path and os.getenv('EBROOTPYTHON') in path for path in [python_exec_path, python_include_path, python_library_path]): - self.log.error("Python related paths do not include EBROOTPYTHON.") - else: - self.log.info("Python related paths configured correctly.") - else: - # Handle cases where no Python paths are found - self.log.error("No Python related paths found in CMakeCache.txt.") - raise EasyBuildError("Python related paths not configured correctly.") + # Check if paths are found and validate paths for symlinks + for path_list in [python_exec_path, python_include_path, python_library_path]: + for path in path_list: + if path and not os.path.exists(path): + raise EasyBuildError(f"Path does not exist: {path}") + + # Check if paths are found and handle EBROOTPYTHON cases + if any(path for path in [python_exec_path, python_include_path, python_library_path]): + if not os.getenv('EBROOTPYTHON'): + self.log.error( + "EBROOTPYTHON is not set and Python related paths are found." + ) + elif not all(path and os.getenv('EBROOTPYTHON') in path + for path in [python_exec_path, python_include_path, python_library_path]): + self.log.error("Python related paths do not include EBROOTPYTHON.") + else: + self.log.info("Python related paths configured correctly.") + else: + self.log.error("No Python related paths found in CMakeCache.txt.") + raise EasyBuildError("Python related paths not configured correctly.") - ebrootpython_path = get_software_root("Python") + ebrootpython_path = get_software_root("Python") - # Validate that Python paths are consistent with EBROOTPYTHON - if python_exec_path and not ebrootpython_path: - raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") + # Validate that Python paths are consistent with EBROOTPYTHON + if python_exec_path and not ebrootpython_path: + raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") - return out + return out def test_step(self): """CMake specific test setup""" From c0d5e86456f9bd1e2f0b537df902503deb184d74 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 25 Mar 2024 09:40:47 +0000 Subject: [PATCH 09/13] Fixing whitespace --- easybuild/easyblocks/generic/cmakemake.py | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 444f5b6f78..4f316b20b4 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -321,19 +321,19 @@ def configure_step(self, srcdir=None, builddir=None): def sanitycheck_for_configuration(self, out): self.log.info("Checking Python paths") - + if LooseVersion(self.cmake_version) >= '3.16': try: with open('CMakeCache.txt', 'r') as file: lines = file.readlines() except FileNotFoundError: self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") - return - + return + if not os.getenv('EBROOTPYTHON'): self.log.warning("EBROOTPYTHON is not set.") - return - else: + return + else: self.log.info("Skipping Python path checks as CMake version might be lower than 3.16.") return @@ -341,7 +341,6 @@ def sanitycheck_for_configuration(self, out): python_include_path = [] python_library_path = [] - # Define a regular expression to match all relevant prefixes python_prefixes = r"(Python|Python2|Python3)_" for line in lines: @@ -354,11 +353,11 @@ def sanitycheck_for_configuration(self, out): elif line.startswith(python_prefixes + r"LIBRARY:FILEPATH="): python_library_path.append(line.split('=')[1].strip()) self.log.info("Python library path: %s", python_library_path[-1]) - + # Check if paths are found and validate paths for symlinks - for path_list in [python_exec_path, python_include_path, python_library_path]: + for path_list in [python_exec_path, python_include_path, python_library_path]: for path in path_list: - if path and not os.path.exists(path): + if path and not os.path.exists(path): raise EasyBuildError(f"Path does not exist: {path}") # Check if paths are found and handle EBROOTPYTHON cases @@ -367,21 +366,26 @@ def sanitycheck_for_configuration(self, out): self.log.error( "EBROOTPYTHON is not set and Python related paths are found." ) - elif not all(path and os.getenv('EBROOTPYTHON') in path - for path in [python_exec_path, python_include_path, python_library_path]): + elif not all( + (path and os.getenv('EBROOTPYTHON') in path) + for path in [python_exec_path, python_include_path, python_library_path] + ): self.log.error("Python related paths do not include EBROOTPYTHON.") else: self.log.info("Python related paths configured correctly.") else: self.log.error("No Python related paths found in CMakeCache.txt.") - raise EasyBuildError("Python related paths not configured correctly.") + raise EasyBuildError("Python related paths not configured correctly.") ebrootpython_path = get_software_root("Python") # Validate that Python paths are consistent with EBROOTPYTHON if python_exec_path and not ebrootpython_path: - raise EasyBuildError(f"Python executable path found ({python_exec_path}) but no Python dependency included. Check log.") - + raise EasyBuildError( + f"Python executable path found ({python_exec_path})" + f" but no Python dependency included. Check log." + ) + return out def test_step(self): From f260f7300c2cdf0bf526aa5dd4f562189a529ca0 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Tue, 30 Apr 2024 10:27:19 +0100 Subject: [PATCH 10/13] Revised version --- easybuild/easyblocks/generic/cmakemake.py | 90 +++++++++-------------- 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 4f316b20b4..e86c5e1ae9 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -49,10 +49,9 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub - +print ("Using Nadia EasyBlock") DEFAULT_CONFIGURE_CMD = 'cmake' - def det_cmake_version(): """ Determine active CMake version. @@ -319,7 +318,6 @@ def configure_step(self, srcdir=None, builddir=None): (out, _) = run_cmd(command, log_all=True, simple=False) - def sanitycheck_for_configuration(self, out): self.log.info("Checking Python paths") if LooseVersion(self.cmake_version) >= '3.16': @@ -328,64 +326,48 @@ def sanitycheck_for_configuration(self, out): lines = file.readlines() except FileNotFoundError: self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") - return if not os.getenv('EBROOTPYTHON'): self.log.warning("EBROOTPYTHON is not set.") - return - else: - self.log.info("Skipping Python path checks as CMake version might be lower than 3.16.") - return - - python_exec_path = [] - python_include_path = [] - python_library_path = [] + else: + self.log.info("Skipping Python path checks as CMake version might be lower than 3.16.") - python_prefixes = r"(Python|Python2|Python3)_" + python_paths = { + "executable": [], + "include": [], + "library": [], + } + errors = [] # List to store all encountered errors for line in lines: - if line.startswith(python_prefixes + r"EXECUTABLE:FILEPATH="): - python_exec_path.append(line.split('=')[1].strip()) - self.log.info("Python executable path: %s", python_exec_path[-1]) - elif line.startswith(python_prefixes + r"INCLUDE_DIR:FILEPATH="): - python_include_path.append(line.split('=')[1].strip()) - self.log.info("Python include path: %s", python_include_path[-1]) - elif line.startswith(python_prefixes + r"LIBRARY:FILEPATH="): - python_library_path.append(line.split('=')[1].strip()) - self.log.info("Python library path: %s", python_library_path[-1]) - - # Check if paths are found and validate paths for symlinks - for path_list in [python_exec_path, python_include_path, python_library_path]: - for path in path_list: - if path and not os.path.exists(path): - raise EasyBuildError(f"Path does not exist: {path}") - - # Check if paths are found and handle EBROOTPYTHON cases - if any(path for path in [python_exec_path, python_include_path, python_library_path]): - if not os.getenv('EBROOTPYTHON'): - self.log.error( - "EBROOTPYTHON is not set and Python related paths are found." - ) - elif not all( - (path and os.getenv('EBROOTPYTHON') in path) - for path in [python_exec_path, python_include_path, python_library_path] - ): - self.log.error("Python related paths do not include EBROOTPYTHON.") - else: - self.log.info("Python related paths configured correctly.") - else: - self.log.error("No Python related paths found in CMakeCache.txt.") - raise EasyBuildError("Python related paths not configured correctly.") - + match = re.match(r"(?i)_Python(\d+)_((?:EXECUTABLE|INCLUDE_DIR|LIBRARY)):.*?=(.*)", line) + if match: + prefix, path_type, path = match.groups() + if path_type == "INCLUDE_DIR": + python_paths["include"].append(path.strip()) + self.log.info(f"Python {path_type} path: {path}") + elif path_type == "EXECUTABLE": + python_paths["executable"].append(path.strip()) + self.log.info(f"Python {path_type} path: {path}") + elif path_type == "LIBRARY": + python_paths["library"].append(path.strip()) + self.log.info(f"Python {path_type} path: {path}") + + # Validate path and handle EBROOTHPYTHON ebrootpython_path = get_software_root("Python") - - # Validate that Python paths are consistent with EBROOTPYTHON - if python_exec_path and not ebrootpython_path: - raise EasyBuildError( - f"Python executable path found ({python_exec_path})" - f" but no Python dependency included. Check log." - ) - + for path_type, paths in python_paths.items(): + for path in paths: + if not os.path.exists(path): + errors.append(f"Python does not exist: {path}") + + if ebrootpython_path and not path.startswith(ebrootpython_path): + errors.append(f"Python {path_type} path '{path}' is outside EBROOTPYTHON ({ebrootpython_path})") + + if errors: + # Combine all errors into a single message + error_message = "\n".join(errors) + raise EasyBuildError(f"Python path errors:\n{error_message}") + self.log.info("Python check successful") return out def test_step(self): From b60ecd673ec68b77a12ca9205f0b1acf5283300b Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Tue, 30 Apr 2024 10:31:04 +0100 Subject: [PATCH 11/13] Revised version - whitespace --- easybuild/easyblocks/generic/cmakemake.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index e86c5e1ae9..07f1623029 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -49,7 +49,6 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub -print ("Using Nadia EasyBlock") DEFAULT_CONFIGURE_CMD = 'cmake' def det_cmake_version(): @@ -337,7 +336,7 @@ def configure_step(self, srcdir=None, builddir=None): "include": [], "library": [], } - errors = [] # List to store all encountered errors + errors = [] for line in lines: match = re.match(r"(?i)_Python(\d+)_((?:EXECUTABLE|INCLUDE_DIR|LIBRARY)):.*?=(.*)", line) From 77884b98aaf464e66dd362ff9f72dc93a9ae6268 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 30 Apr 2024 14:06:16 +0200 Subject: [PATCH 12/13] Improve Python check in cmakemake.py - Reintroduce function such that missing early returns can be used - Fix regexp (Superflous group, optional stuff) - Handle the case where the cache path is not found - Handle $EBROOTPYTHON not set and comparing it in the presence of symlinks --- easybuild/easyblocks/generic/cmakemake.py | 70 ++++++++++++----------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 07f1623029..8ffea41290 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -42,15 +42,17 @@ from easybuild.framework.easyconfig import BUILD, CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which +from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, read_file, which from easybuild.tools.environment import setvar from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub +import easybuild.tools.toolchain as toolchain DEFAULT_CONFIGURE_CMD = 'cmake' + def det_cmake_version(): """ Determine active CMake version. @@ -158,7 +160,6 @@ def prepend_config_opts(self, config_opts): if '-D%s=' % key not in cfg_configopts) self.cfg['configopts'] = ' '.join([new_opts, cfg_configopts]) - def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" @@ -316,50 +317,52 @@ def configure_step(self, srcdir=None, builddir=None): self.cfg['configopts']]) (out, _) = run_cmd(command, log_all=True, simple=False) + self.check_python_paths() - self.log.info("Checking Python paths") + return out - if LooseVersion(self.cmake_version) >= '3.16': - try: - with open('CMakeCache.txt', 'r') as file: - lines = file.readlines() - except FileNotFoundError: - self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") + def check_python_paths(self): + """Check that there are no detected Python paths outside the EB installed PythonF""" + if not os.path.exists('CMakeCache.txt'): + self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") + return + cmake_cache = read_file('CMakeCache.txt') + if not cmake_cache: + self.log.warning("CMake Cache could not be read. Python paths checks skipped.") + return - if not os.getenv('EBROOTPYTHON'): - self.log.warning("EBROOTPYTHON is not set.") - else: - self.log.info("Skipping Python path checks as CMake version might be lower than 3.16.") + self.log.info("Checking Python paths") python_paths = { "executable": [], - "include": [], + "include_dir": [], "library": [], } - errors = [] - - for line in lines: - match = re.match(r"(?i)_Python(\d+)_((?:EXECUTABLE|INCLUDE_DIR|LIBRARY)):.*?=(.*)", line) + PYTHON_RE = re.compile(r"_?(?:Python|PYTHON)\d_(EXECUTABLE|INCLUDE_DIR|LIBRARY)[^=]*=(.*)") + for line in cmake_cache.splitlines(): + match = PYTHON_RE.match(line) if match: - prefix, path_type, path = match.groups() - if path_type == "INCLUDE_DIR": - python_paths["include"].append(path.strip()) - self.log.info(f"Python {path_type} path: {path}") - elif path_type == "EXECUTABLE": - python_paths["executable"].append(path.strip()) - self.log.info(f"Python {path_type} path: {path}") - elif path_type == "LIBRARY": - python_paths["library"].append(path.strip()) - self.log.info(f"Python {path_type} path: {path}") - - # Validate path and handle EBROOTHPYTHON + path_type = match[1].lower() + path = match[2].strip() + self.log.info(f"Python {path_type} path: {path}") + python_paths[path_type].append(path) + ebrootpython_path = get_software_root("Python") + if not ebrootpython_path: + if any(python_paths.values()) and not self.toolchain.comp_family() == toolchain.SYSTEM: + self.log.warning("Found Python paths in CMake cache but Python is not a dependency") + # Can't do the check + return + ebrootpython_path = os.path.realpath(ebrootpython_path) + + errors = [] for path_type, paths in python_paths.items(): for path in paths: + if path.endswith('-NOTFOUND'): + continue if not os.path.exists(path): - errors.append(f"Python does not exist: {path}") - - if ebrootpython_path and not path.startswith(ebrootpython_path): + errors.append(f"Python {path_type} does not exist: {path}") + elif not os.path.realpath(path).startswith(ebrootpython_path): errors.append(f"Python {path_type} path '{path}' is outside EBROOTPYTHON ({ebrootpython_path})") if errors: @@ -367,7 +370,6 @@ def configure_step(self, srcdir=None, builddir=None): error_message = "\n".join(errors) raise EasyBuildError(f"Python path errors:\n{error_message}") self.log.info("Python check successful") - return out def test_step(self): """CMake specific test setup""" From c1f005a50f0581c536b0bdef22d66fd89bfbf976 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 25 Jun 2024 10:49:54 +0200 Subject: [PATCH 13/13] Address review --- easybuild/easyblocks/generic/cmakemake.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 8ffea41290..5b58594d6e 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -322,7 +322,7 @@ def configure_step(self, srcdir=None, builddir=None): return out def check_python_paths(self): - """Check that there are no detected Python paths outside the EB installed PythonF""" + """Check that there are no detected Python paths outside the Python dependency provided by EasyBuild""" if not os.path.exists('CMakeCache.txt'): self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") return @@ -338,13 +338,13 @@ def check_python_paths(self): "include_dir": [], "library": [], } - PYTHON_RE = re.compile(r"_?(?:Python|PYTHON)\d_(EXECUTABLE|INCLUDE_DIR|LIBRARY)[^=]*=(.*)") + python_regex = re.compile(r"_?(?:Python|PYTHON)\d?_(EXECUTABLE|INCLUDE_DIR|LIBRARY)[^=]*=(.*)") for line in cmake_cache.splitlines(): - match = PYTHON_RE.match(line) + match = python_regex.match(line) if match: path_type = match[1].lower() path = match[2].strip() - self.log.info(f"Python {path_type} path: {path}") + self.log.info("Python %s path: %s", path_type, path) python_paths[path_type].append(path) ebrootpython_path = get_software_root("Python") @@ -361,14 +361,15 @@ def check_python_paths(self): if path.endswith('-NOTFOUND'): continue if not os.path.exists(path): - errors.append(f"Python {path_type} does not exist: {path}") + errors.append("Python %s does not exist: %s" % (path_type, path)) elif not os.path.realpath(path).startswith(ebrootpython_path): - errors.append(f"Python {path_type} path '{path}' is outside EBROOTPYTHON ({ebrootpython_path})") + errors.append("Python %s path '%s' is outside EBROOTPYTHON (%s)" % + (path_type, path, ebrootpython_path)) if errors: # Combine all errors into a single message error_message = "\n".join(errors) - raise EasyBuildError(f"Python path errors:\n{error_message}") + raise EasyBuildError("Python path errors:\n" + error_message) self.log.info("Python check successful") def test_step(self):