diff --git a/README.md b/README.md index 22a1277..1b9bdbe 100644 --- a/README.md +++ b/README.md @@ -15,14 +15,14 @@ In cases where more dynamic customization is sensible, suitable environment vari Any option without a default is required. -| Option | Definition | Type | Default | Supports dynamic modification | -|-----------------------|---------------------------------------------------------------------|----------------|---------------------|-------------------------------| -| `build-backend` | The wrapped build backend (e.g. `setuptools.build_meta`) | string | | N | -| `commit-file` | The file in which to write the git commit hash | string | "" (No file) | N | -| `dependencies-file` | The path to the `dependencies.yaml` file to use | string | "dependencies.yaml" | Y | -| `matrix-entry` | A `;`-separated list of `=`-delimited key/value pairs | string | "" | Y | -| `require-cuda` | If false, builds will succeed even if nvcc is not available | bool | true | Y | -| `requires` | List of build requirements (in addition to `build-system.requires`) | list[str] | [] | N | +| Option | Definition | Type | Default | Supports dynamic modification | +|-----------------------|--------------------------------------------------------------------------------------------------|----------------|---------------------|-------------------------------| +| `build-backend` | The wrapped build backend (e.g. `setuptools.build_meta`) | string | | N | +| `commit-file` | The file in which to write the git commit hash | string | "" (No file) | N | +| `dependencies-file` | The path to the `dependencies.yaml` file to use | string | "dependencies.yaml" | Y | +| `disable-cuda` | If true, CUDA version in build environment is ignored when setting package name and dependencies | bool | false | Y | +| `matrix-entry` | A `;`-separated list of `=`-delimited key/value pairs | string | "" | Y | +| `requires` | List of build requirements (in addition to `build-system.requires`) | list[str] | [] | N | ## Outstanding questions diff --git a/dependencies.yaml b/dependencies.yaml index b945ac6..9357526 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -77,8 +77,7 @@ dependencies: - PyYAML - packaging - rapids-dependency-file-generator>=1.13.3,<2.0.dev0 - - tomli - - tomli-w + - tomlkit test: common: - output_types: [conda, requirements, pyproject] diff --git a/pyproject.toml b/pyproject.toml index b57f161..abaf7f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,7 @@ dependencies = [ "PyYAML", "packaging", "rapids-dependency-file-generator>=1.13.3,<2.0.dev0", - "tomli", - "tomli-w", + "tomlkit", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. license = { text = "Apache 2.0" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/rapids_build_backend/config.py b/rapids_build_backend/config.py index 4775dfb..14dee20 100644 --- a/rapids_build_backend/config.py +++ b/rapids_build_backend/config.py @@ -30,8 +30,8 @@ class Config: "build-backend": (None, False), "commit-file": ("", False), "dependencies-file": ("dependencies.yaml", True), + "disable-cuda": (False, True), "matrix-entry": ("", True), - "require-cuda": (True, True), "requires": (lambda: [], False), } diff --git a/rapids_build_backend/impls.py b/rapids_build_backend/impls.py index 3dec619..18a100c 100644 --- a/rapids_build_backend/impls.py +++ b/rapids_build_backend/impls.py @@ -10,7 +10,7 @@ from typing import Union import rapids_dependency_file_generator -import tomli_w +import tomlkit from . import utils from .config import Config @@ -39,62 +39,44 @@ def _get_backend(build_backend): @lru_cache -def _get_cuda_version(require_cuda: bool): +def _get_cuda_version(): """Get the CUDA suffix based on nvcc. - Parameters - ---------- - require_cuda : bool - If True, raise an exception if nvcc is not in the PATH. If False, return None. - Returns ------- str or None - The CUDA major version (e.g., "11") or None if CUDA could not be detected. + The CUDA major version (e.g., "11") """ - try: - nvcc_exists = ( - subprocess.run(["which", "nvcc"], capture_output=True).returncode == 0 + nvcc_exists = subprocess.run(["which", "nvcc"], capture_output=True).returncode == 0 + if not nvcc_exists: + raise ValueError( + "Could not determine the CUDA version. Make sure nvcc is in your PATH." ) - if not nvcc_exists: - raise ValueError( - "Could not determine the CUDA version. Make sure nvcc is in your PATH." - ) - try: - process_output = subprocess.run(["nvcc", "--version"], capture_output=True) - except subprocess.CalledProcessError as e: - raise ValueError("Failed to get version from nvcc.") from e + try: + process_output = subprocess.run(["nvcc", "--version"], capture_output=True) + except subprocess.CalledProcessError as e: + raise ValueError("Failed to get version from nvcc.") from e - output_lines = process_output.stdout.decode().splitlines() + output_lines = process_output.stdout.decode().splitlines() - match = re.search(r"release (\d+)\.(\d+)", output_lines[3]) - if match is None: - raise ValueError("Failed to parse CUDA version from nvcc output.") - return match.groups() - except Exception: - if not require_cuda: - return None - raise + match = re.search(r"release (\d+)\.(\d+)", output_lines[3]) + if match is None: + raise ValueError("Failed to parse CUDA version from nvcc output.") + return match.groups() @lru_cache -def _get_cuda_suffix(require_cuda: bool) -> str: +def _get_cuda_suffix() -> str: """Get the CUDA suffix based on nvcc. - Parameters - ---------- - require_cuda : bool - If True, raise an exception if CUDA could not be detected. If False, return an - empty string. - Returns ------- str The CUDA suffix (e.g., "-cu11") or an empty string if CUDA could not be detected. """ - if (version := _get_cuda_version(require_cuda)) is None: + if (version := _get_cuda_version()) is None: return "" return f"-cu{version[0]}" @@ -165,7 +147,8 @@ def _edit_pyproject(config): pyproject_file = "pyproject.toml" bkp_pyproject_file = ".pyproject.toml.rapids-build-backend.bak" - cuda_version = _get_cuda_version(config.require_cuda) + if not config.disable_cuda: + cuda_version_major, cuda_version_minor = _get_cuda_version() try: parsed_config = rapids_dependency_file_generator.load_config_from_file( @@ -193,8 +176,8 @@ def _edit_pyproject(config): ): continue matrix = _parse_matrix(config.matrix_entry) or dict(file_config.matrix) - if cuda_version is not None: - matrix["cuda"] = [f"{cuda_version[0]}.{cuda_version[1]}"] + if not config.disable_cuda: + matrix["cuda"] = [f"{cuda_version_major}.{cuda_version_minor}"] rapids_dependency_file_generator.make_dependency_files( parsed_config=parsed_config, file_keys=[file_key], @@ -203,11 +186,12 @@ def _edit_pyproject(config): prepend_channels=[], to_stdout=False, ) - pyproject = utils._get_pyproject() - project_data = pyproject["project"] - project_data["name"] += _get_cuda_suffix(config.require_cuda) - with open(pyproject_file, "wb") as f: - tomli_w.dump(pyproject, f) + if not config.disable_cuda: + pyproject = utils._get_pyproject() + project_data = pyproject["project"] + project_data["name"] += _get_cuda_suffix() + with open(pyproject_file, "w") as f: + tomlkit.dump(pyproject, f) yield finally: # Restore by moving rather than writing to avoid any formatting changes. diff --git a/rapids_build_backend/utils.py b/rapids_build_backend/utils.py index 41abfee..8208ca8 100644 --- a/rapids_build_backend/utils.py +++ b/rapids_build_backend/utils.py @@ -2,10 +2,10 @@ import os -import tomli +import tomlkit -def _get_pyproject(dirname: str = ".") -> dict: +def _get_pyproject(dirname: str = ".") -> tomlkit.toml_document.TOMLDocument: """Parse and return the pyproject.toml file.""" - with open(os.path.join(dirname, "pyproject.toml"), "rb") as f: - return tomli.load(f) + with open(os.path.join(dirname, "pyproject.toml")) as f: + return tomlkit.load(f) diff --git a/tests/conftest.py b/tests/conftest.py index eed717e..68ff26e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,8 +11,7 @@ from pathlib import Path import pytest -import tomli -import tomli_w +import tomlkit from jinja2 import Environment, FileSystemLoader from packaging.version import parse as parse_version @@ -69,10 +68,10 @@ def patch_nvcc_if_needed(nvcc_version): try: # Only create a patch if one is required. In addition to reducing overhead, this # also ensures that we test the real nvcc and don't mask any relevant errors. - if ( - _get_cuda_version(False) is None - or _get_cuda_version(False)[0] != nvcc_version - ): + try: + if _get_cuda_version()[0] != nvcc_version: + raise ValueError + except ValueError: nvcc = _create_nvcc(nvcc_version) os.environ["PATH"] = os.pathsep.join( [os.path.dirname(nvcc), os.environ["PATH"]] @@ -179,14 +178,14 @@ def wheelhouse(tmp_path_factory, pip_cache): ) pyproject_file = rapids_build_backend_build_dir / "pyproject.toml" - with open(pyproject_file, "rb") as f: - pyproject = tomli.load(f) + with open(pyproject_file) as f: + pyproject = tomlkit.load(f) project_data = pyproject["project"] version = parse_version(project_data["version"]) project_data["version"] = f"{version.major + 1}.{version.minor}.{version.micro}" - with open(pyproject_file, "wb") as f: - tomli_w.dump(pyproject, f) + with open(pyproject_file, "w") as f: + tomlkit.dump(pyproject, f) subprocess.run( [ diff --git a/tests/test_config.py b/tests/test_config.py index ca415f4..3dae383 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,9 +27,9 @@ def setup_config_project(tmp_path, flag, config_value): [ ("commit-file", '"pkg/_version.py"', "pkg/_version.py"), ("commit-file", None, ""), - ("require-cuda", "true", True), - ("require-cuda", "false", False), - ("require-cuda", None, True), + ("disable-cuda", "true", True), + ("disable-cuda", "false", False), + ("disable-cuda", None, False), ], ) def test_config(tmp_path, flag, config_value, expected): @@ -40,8 +40,8 @@ def test_config(tmp_path, flag, config_value, expected): @pytest.mark.parametrize( "flag, config_value, expected", [ - ("require-cuda", "true", True), - ("require-cuda", "false", False), + ("disable-cuda", "true", True), + ("disable-cuda", "false", False), ], ) def test_config_env_var(tmp_path, flag, config_value, expected): @@ -63,8 +63,8 @@ def test_config_env_var(tmp_path, flag, config_value, expected): @pytest.mark.parametrize( "flag, config_value, expected", [ - ("require-cuda", "true", True), - ("require-cuda", "false", False), + ("disable-cuda", "true", True), + ("disable-cuda", "false", False), ], ) def test_config_config_settings(tmp_path, flag, config_value, expected): diff --git a/tests/test_impls.py b/tests/test_impls.py index c537b02..f7e724b 100644 --- a/tests/test_impls.py +++ b/tests/test_impls.py @@ -66,6 +66,7 @@ def check_initial_contents(filename): "pyproject_dir", "dependencies_file", "write_dependencies_file", + "disable_cuda", "cuda_version", "cuda_suffix", "cuda_python_requirement", @@ -77,6 +78,7 @@ def check_initial_contents(filename): ".", "dependencies.yaml", True, + False, ("11", "5"), "-cu11", "cuda-python>=11.5,<11.6.dev0", @@ -87,6 +89,7 @@ def check_initial_contents(filename): ".", "dependencies.yaml", True, + False, ("11", "5"), "-cu11", "cuda-python>=11.5,<11.6.dev0", @@ -97,6 +100,7 @@ def check_initial_contents(filename): "python", "../dependencies.yaml", True, + False, ("12", "1"), "-cu12", "cuda-python>=12.1,<12.2.dev0", @@ -107,165 +111,186 @@ def check_initial_contents(filename): ".", "dependencies.yaml", False, + False, ("11", "5"), "-cu11", None, "", None, ), + ( + ".", + "dependencies.yaml", + True, + True, + None, # Ensure _get_cuda_version() isn't called and unpacked + "", + "cuda-python", + "", + "some-x86-package", + ), ], ) def test_edit_pyproject( + tmp_path, pyproject_dir, dependencies_file, write_dependencies_file, + disable_cuda, cuda_version, cuda_suffix, cuda_python_requirement, matrix, arch_requirement, ): - with tempfile.TemporaryDirectory() as d: - original_contents = dedent( - """\ - [project] - name = "test-project" - dependencies = [] + original_contents = dedent( + """\ + [project] + name = "test-project" + dependencies = [] - [build-system] - requires = [] - """ - ) - full_pyproject_dir = os.path.join(d, pyproject_dir) - if not os.path.exists(full_pyproject_dir): - os.mkdir(full_pyproject_dir) + [build-system] + requires = [] + """ + ) + full_pyproject_dir = os.path.join(tmp_path, pyproject_dir) + if not os.path.exists(full_pyproject_dir): + os.mkdir(full_pyproject_dir) - with set_cwd(full_pyproject_dir): - with open("pyproject.toml", "w") as f: - f.write(original_contents) + with set_cwd(full_pyproject_dir): + with open("pyproject.toml", "w") as f: + f.write(original_contents) - if write_dependencies_file: - with open(dependencies_file, "w") as f: - f.write( - dedent( - f"""\ - files: - project: - output: pyproject - includes: - - project - - arch - pyproject_dir: {pyproject_dir} - matrix: - cuda: ["11.5"] - arch: ["x86_64"] - extras: - table: project - build_system: - output: pyproject - includes: - - build_system - pyproject_dir: {pyproject_dir} - extras: - table: build-system - other_project: - output: pyproject - includes: - - bad - pyproject_dir: python_bad - extras: - table: project - conda: - output: conda - includes: - - bad - dependencies: - project: - common: - - output_types: [pyproject] + if write_dependencies_file: + with open(dependencies_file, "w") as f: + f.write( + dedent( + f"""\ + files: + project: + output: pyproject + includes: + - project + - arch + pyproject_dir: {pyproject_dir} + matrix: + arch: ["x86_64"] + extras: + table: project + build_system: + output: pyproject + includes: + - build_system + pyproject_dir: {pyproject_dir} + extras: + table: build-system + other_project: + output: pyproject + includes: + - bad + pyproject_dir: python_bad + extras: + table: project + conda: + output: conda + includes: + - bad + dependencies: + project: + common: + - output_types: [pyproject] + packages: + - tomli + specific: + - output_types: [pyproject] + matrices: + - matrix: + cuda: "11.5" packages: - - tomli - specific: - - output_types: [pyproject] - matrices: - - matrix: - cuda: "11.5" - packages: - - cuda-python>=11.5,<11.6.dev0 - - matrix: - cuda: "12.1" - packages: - - cuda-python>=12.1,<12.2.dev0 - build_system: - common: - - output_types: [pyproject] + - cuda-python>=11.5,<11.6.dev0 + - matrix: + cuda: "12.1" packages: - - scikit-build-core - arch: - specific: - - output_types: [pyproject] - matrices: - - matrix: - arch: x86_64 - packages: - - some-x86-package - - matrix: - arch: aarch64 - packages: - - some-arm-package - bad: - common: - - output_types: [pyproject, conda] + - cuda-python>=12.1,<12.2.dev0 + - matrix: packages: - - bad-package - """ - ) + - cuda-python + build_system: + common: + - output_types: [pyproject] + packages: + - scikit-build-core + arch: + specific: + - output_types: [pyproject] + matrices: + - matrix: + arch: x86_64 + packages: + - some-x86-package + - matrix: + arch: aarch64 + packages: + - some-arm-package + bad: + common: + - output_types: [pyproject, conda] + packages: + - bad-package + """ ) - config = Mock( - require_cuda=False, - dependencies_file=dependencies_file, - matrix_entry=matrix, - ) + ) + config = Mock( + disable_cuda=disable_cuda, + dependencies_file=dependencies_file, + matrix_entry=matrix, + ) - with patch( - "rapids_build_backend.impls._get_cuda_version", - Mock(return_value=cuda_version), - ), patch( - "rapids_build_backend.impls._get_cuda_suffix", - _get_cuda_suffix.__wrapped__, - ): - with _edit_pyproject(config): - with open("pyproject.toml") as f: - if write_dependencies_file: - assert f.read() == dedent( - f"""\ - [project] - name = "test-project{cuda_suffix}" - dependencies = [ - "{cuda_python_requirement}", - "{arch_requirement}", - "tomli", - ] + with patch( + "rapids_build_backend.impls._get_cuda_version", + Mock(return_value=cuda_version), + ), patch( + "rapids_build_backend.impls._get_cuda_suffix", + _get_cuda_suffix.__wrapped__, + ): + with _edit_pyproject(config): + with open("pyproject.toml") as f: + if write_dependencies_file: + assert f.read() == dedent( + f"""\ + [project] + name = "test-project{cuda_suffix}" + dependencies = [ + "{cuda_python_requirement}", + "{arch_requirement}", + "tomli", + ] # This list was generated by """ + """`rapids-dependency-file-generator`. To make """ + f"""changes, edit {dependencies_file} and run """ + """`rapids-dependency-file-generator`. - [build-system] - requires = [ - "scikit-build-core", - ] - """ - ) - else: - assert f.read() == dedent( - f"""\ - [project] - name = "test-project{cuda_suffix}" - dependencies = [] + [build-system] + requires = [ + "scikit-build-core", + ] # This list was generated by """ + """`rapids-dependency-file-generator`. To make """ + f"""changes, edit {dependencies_file} and run """ + """`rapids-dependency-file-generator`. + """ + ) + else: + assert f.read() == dedent( + f"""\ + [project] + name = "test-project{cuda_suffix}" + dependencies = [] - [build-system] - requires = [] - """ - ) - with open(".pyproject.toml.rapids-build-backend.bak") as f: - assert f.read() == original_contents + [build-system] + requires = [] + """ + ) + with open(".pyproject.toml.rapids-build-backend.bak") as f: + assert f.read() == original_contents - with open("pyproject.toml") as f: - assert f.read() == original_contents + with open("pyproject.toml") as f: + assert f.read() == original_contents