skip get_requires for setuptools projects

.pre-commit-config.yaml

@@ -1,5 +1,10 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
+exclude: |
+  (?x)^(
+      tests/templates/.*py
+  )$
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0

README.md

@@ -29,6 +29,28 @@ Any option without a default is required.
 
 - How should we split up build requirements between `build-system` and `tool.rapids-build-backend`? In theory any dependency that doesn't need suffixing could also go into `build-system.requires`. I think it's easier to teach that all dependencies other than `rapids-build-backend` itself should to into `tool.rapids-build-backend`, but I don't know how others feel.
 
+## `setuptools` support
+
+This project supports builds using `setuptools.build_meta` as their build backend, and which use a `setup.py` for configuration.
+
+However, it does not support passing a list of dependencies through `setup_requires` to `setuptools.setup()`.
+If you're interested in using `setuptools.build_meta` and a `setup.py`, pass a list of dependencies that need to be installed prior to `setup.py` running through `rapids-build-backend`'s requirements, like this:
+
+```toml
+[project]
+build-backend = "rapids_build_backend.build"
+requires = [
+    "rapids-build-backend",
+    "setuptools"
+]
+
+[tool.rapids-build-backend]
+build-backend = "setuptools.build_meta"
+requires = [
+    "Cython"
+]
+```
+
 ## Rejected ideas
 
 - We could also include the rewrite of VERSION that we use for RAPIDS builds, but this is really more specific to our release process than the general process of building our wheels. I don't think someone building a wheel locally should see the same version as what we produce via CI. If we really wanted we could pull dunamai as a dependency and write a different version here, though.

rapids_build_backend/impls.py

@@ -210,6 +210,35 @@ def _edit_pyproject(config):
         shutil.move(bkp_pyproject_file, pyproject_file)
 
 
+def _check_setup_py(setup_py_contents: str) -> None:
+    """
+    ``setuptools.get_requires_for_build_wheel()`` executes setup.py if it exists,
+    to check for dependencies in ``setup_requires`` (passed to ``setuptools.setup()``).
+
+    That's a problem for rapids-build-backend, as at the point where that's invoked,
+    its recalculated list of build dependencies (modified in ``_edit_pyproject()``)
+    haven't yet been installed.
+
+    If any of them are imported in ``setup.py``, those imports will fail.
+
+    This function raises an exception if it detects ``setup_requires`` being used in
+    a ``setup.py``, to clarify that ``rapids-build-backend`` can't support that case.
+
+    ref: https://github.com/rapidsai/rapids-build-backend/issues/39
+    """
+
+    # pattern = "any use of 'setup_requires' on a line that isn't a comment"
+    setup_requires_pat = r"^(?!\s*#+).*setup_requires"
+
+    if re.search(setup_requires_pat, setup_py_contents, re.M) is not None:
+        raise ValueError(
+            "Detected use of 'setup_requires' in a setup.py file. "
+            "rapids-build-backend does not support this pattern. Try moving "
+            "that list of dependencies into the 'requires' list in the "
+            "[tool.rapids-build-backend] table in pyproject.toml."
+        )
+
+
 # The hooks in this file could be defined more programmatically by iterating over the
 # backend's attributes, but it's simpler to just define them explicitly and avoids any
 # potential issues with assuming the right pyproject.toml is readable at import time (we
@@ -234,11 +263,23 @@ def get_requires_for_build_wheel(config_settings):
             backend := _get_backend(config.build_backend),
             "get_requires_for_build_wheel",
         ):
-            requires.extend(
-                backend.get_requires_for_build_wheel(
-                    _remove_rapidsai_from_config(config_settings)
+            if config.build_backend == "setuptools.build_meta":
+                _check_setup_py(setup_py_contents=utils._get_setup_py())
+                # prior to https://github.com/pypa/setuptools/pull/4369 (May 2024),
+                # setuptools.build_meta.get_requires_for_build_wheel() automatically
+                # added 'wheel' to the build requirements. Adding that manually here,
+                # since this code block skips running
+                # setuptools.build_meta.get_requires_for_build_wheel().
+                #
+                # Without this, running 'pip wheel' might result in an error like
+                # "error: invalid command 'bdist_wheel'".
+                requires.extend(["wheel"])
+            else:
+                requires.extend(
+                    backend.get_requires_for_build_wheel(
+                        _remove_rapidsai_from_config(config_settings)
+                    )
                 )
-            )
 
         return requires
 
@@ -256,11 +297,14 @@ def get_requires_for_build_sdist(config_settings):
             backend := _get_backend(config.build_backend),
             "get_requires_for_build_sdist",
         ):
-            requires.extend(
-                backend.get_requires_for_build_sdist(
-                    _remove_rapidsai_from_config(config_settings)
+            if config.build_backend == "setuptools.build_meta":
+                _check_setup_py(setup_py_contents=utils._get_setup_py())
+            else:
+                requires.extend(
+                    backend.get_requires_for_build_sdist(
+                        _remove_rapidsai_from_config(config_settings)
+                    )
                 )
-            )
 
         return requires
 
@@ -276,11 +320,14 @@ def get_requires_for_build_editable(config_settings):
             backend := _get_backend(config.build_backend),
             "get_requires_for_build_editable",
         ):
-            requires.extend(
-                backend.get_requires_for_build_editable(
-                    _remove_rapidsai_from_config(config_settings)
+            if config.build_backend == "setuptools.build_meta":
+                _check_setup_py(setup_py_contents=utils._get_setup_py())
+            else:
+                requires.extend(
+                    backend.get_requires_for_build_editable(
+                        _remove_rapidsai_from_config(config_settings)
+                    )
                 )
-            )
 
         return requires
 

rapids_build_backend/utils.py

@@ -9,3 +9,21 @@ def _get_pyproject(dirname: str = ".") -> tomlkit.toml_document.TOMLDocument:
     """Parse and return the pyproject.toml file."""
     with open(os.path.join(dirname, "pyproject.toml")) as f:
         return tomlkit.load(f)
+
+
+def _get_setup_py() -> str:
+    """
+    Returns a string with the contents of setup.py,
+    or empty string if it doesn't exist.
+    """
+    # setuptools.build_meta.get_requires_for_wheel() assumes that "setup.py" is directly
+    # relative to the current working directly, so rapids-build-backend can too.
+    #
+    # ref: https://github.com/pypa/setuptools/blob/f91fa3d9fc7262e0467e4b2f84fe463f8f8d23cf/setuptools/build_meta.py#L304
+    setup_py_file = "setup.py"
+
+    if not os.path.isfile(setup_py_file):
+        return ""
+
+    with open(setup_py_file) as f:
+        return f.read()

tests/templates/dependencies-rbb-only.yaml

@@ -0,0 +1,32 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+# [description]
+#
+#   dependencies.yaml that intentionally only updates [tool.rapids-build-backend] table
+#   in pyproject.toml.
+#
+#   Create new templates to test other dependencies.yaml contents.
+#
+
+files:
+  py_rapids_build:
+    output: pyproject
+    pyproject_dir: .
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - build_python
+dependencies:
+  build_python:
+    specific:
+      - output_types: [pyproject, requirements]
+        matrices:
+          - matrix: {cuda: "85.*"}
+            packages:
+              - more-itertools
+          # keeping this empty means it'll only be filled in if
+          # rapids-build-backend actually resolves one of the CUDA-specific
+          # matrices
+          - matrix: null
+            packages: null

tests/templates/setup.py

@@ -0,0 +1,5 @@
+from setuptools import setup
+
+{% for line in setup_py_lines %}
+{{ line }}
+{% endfor %}

tests/test_impls.py

@@ -8,6 +8,7 @@
 import pytest
 
 from rapids_build_backend.impls import (
+    _check_setup_py,
     _edit_pyproject,
     _get_cuda_suffix,
     _remove_rapidsai_from_config,
@@ -322,3 +323,38 @@ def test_edit_pyproject(
 
         with open("pyproject.toml") as f:
             assert f.read() == original_contents
+
+
+@pytest.mark.parametrize(
+    ["setup_py_content", "expect_error"],
+    [
+        ("", False),
+        ("from setuptools import setup\n\nsetup()\n", False),
+        # 'setup_requires' in a comment on its own line
+        ("from setuptools import setup\n# setup_requires\n\nsetup()\n", False),
+        # 'setup_requires' actually passed into setup(), on the same line
+        ("from setuptools import setup\nsetup(setup_requires=[])\n", True),
+        # 'setup_requires' actually passed into setup(), on its own line
+        (
+            "from setuptools import setup\nsetup(\n    setup_requires=['rmm']\n)\n# setup_requires\n",  # noqa: E501
+            True,
+        ),
+        # 'setup_requires' actually passed into setup(), via a dictionary
+        (
+            "from setuptools import setup\nopts={'setup_requires': ['rmm']}\nsetup(**opts)\n",  # noqa: E501
+            True,
+        ),
+    ],
+)
+@patch("rapids_build_backend.impls._get_git_commit", Mock(return_value="abc123"))
+def test_check_setup_py(
+    setup_py_content,
+    expect_error,
+):
+    if expect_error:
+        with pytest.raises(
+            ValueError, match=r"Detected use of 'setup_requires' in a setup\.py file"
+        ):
+            _check_setup_py(setup_py_content)
+    else:
+        _check_setup_py(setup_py_content) is None

tests/test_packages.py

@@ -2,6 +2,7 @@
 
 import glob
 import os
+import subprocess
 import zipfile
 from email.parser import BytesParser
 
@@ -81,6 +82,128 @@ def test_simple_setuptools(tmp_path, env, nvcc_version):
     assert extras == {"test": {"dask-cuda==24.4.*,>=0.0.0a0"}}
 
 
+# rapids-build-backend should support projects using setuptools whose setup.py
+# file has 'import' statements depending on some project(s) that need to be extracted
+# from dependencies.yaml at build time
+def test_setuptools_with_imports_in_setup_py_works(
+    tmp_path,
+    isolated_env,
+):
+    package_dir = tmp_path / "pkg"
+    os.makedirs(package_dir)
+
+    template_args = {
+        "name": "setuptools-with-imports-in-setup-py",
+        "build_backend": "setuptools.build_meta",
+        "build_backend_package": "setuptools",
+        "flags": {
+            "commit-files": "[]",
+            "dependencies-file": '"dependencies-rbb-only.yaml"',
+        },
+        "setup_py_lines": [
+            "import more_itertools",
+            "",
+            "print(more_itertools.__version__)",
+            "",
+            "setup()",
+        ],
+    }
+    generate_from_template(package_dir, "dependencies-rbb-only.yaml", template_args)
+    generate_from_template(package_dir, "pyproject.toml", template_args)
+    generate_from_template(package_dir, "setup.py", template_args)
+
+    with patch_nvcc_if_needed(nvcc_version="85"):
+        name, build_requires, requirements, extras = _generate_wheel(
+            env=isolated_env, package_dir=package_dir
+        )
+
+    assert name == "setuptools-with-imports-in-setup-py-cu85"
+    assert {"more-itertools"}.issubset(build_requires)
+    assert requirements == set()
+
+
+def test_setuptools_with_imports_in_setup_py_fails_on_missing_imports(
+    tmp_path, isolated_env, capfd
+):
+    package_dir = tmp_path / "pkg"
+    os.makedirs(package_dir)
+
+    template_args = {
+        "name": "setuptools-with-imports-in-setup-py",
+        "build_backend": "setuptools.build_meta",
+        "build_backend_package": "setuptools",
+        "flags": {
+            "commit-files": "[]",
+            "dependencies-file": '"dependencies-rbb-only.yaml"',
+        },
+        "setup_py_lines": [
+            "import more_itertools",
+            "",
+            "print(more_itertools.__version__)",
+            "",
+            "setup()",
+        ],
+    }
+    generate_from_template(package_dir, "dependencies-rbb-only.yaml", template_args)
+    generate_from_template(package_dir, "pyproject.toml", template_args)
+    generate_from_template(package_dir, "setup.py", template_args)
+
+    # only the CUDA '85.*' in this example provides required build dependency
+    # 'more-itertools', so it won't be found if using some other matrix.
+    #
+    # This test confirms that rapids-build-backend fails loudly in that case, instead of
+    # silently ignoring it.
+    #
+    # It'd also catch the case where other tests accidentally pass because
+    # 'more-itertools' already existed in the environment where tests run.
+    with patch_nvcc_if_needed(nvcc_version="25"):
+        with pytest.raises(subprocess.CalledProcessError, match=".*pip.*"):
+            _generate_wheel(env=isolated_env, package_dir=package_dir)
+
+    captured_output = capfd.readouterr()
+    assert (
+        "ModuleNotFoundError: No module named 'more_itertools'" in captured_output.out
+    )
+
+
+def test_setuptools_with_setup_requires_fails_with_informative_error(
+    tmp_path, isolated_env, capfd
+):
+    package_dir = tmp_path / "pkg"
+    os.makedirs(package_dir)
+    template_args = {
+        "name": "setuptools-with-imports-in-setup-py",
+        "build_backend": "setuptools.build_meta",
+        "build_backend_package": "setuptools",
+        "flags": {
+            "commit-files": "[]",
+            "dependencies-file": '"dependencies-rbb-only.yaml"',
+        },
+        "setup_py_lines": [
+            "import more_itertools",
+            "",
+            "print(more_itertools.__version__)",
+            "",
+            "setup(",
+            "    setup_requires=['more-itertools'],",
+            ")",
+        ],
+    }
+    generate_from_template(package_dir, "dependencies-rbb-only.yaml", template_args)
+    generate_from_template(package_dir, "pyproject.toml", template_args)
+    generate_from_template(package_dir, "setup.py", template_args)
+
+    with patch_nvcc_if_needed(nvcc_version="85"):
+        with pytest.raises(subprocess.CalledProcessError, match=".*pip.*"):
+            _generate_wheel(env=isolated_env, package_dir=package_dir)
+
+    captured_output = capfd.readouterr()
+    assert (
+        "ValueError: Detected use of 'setup_requires' in a setup.py file"
+        in captured_output.out
+    )
+
+
 @pytest.mark.parametrize("nvcc_version", ["11", "12"])
 def test_simple_scikit_build_core(tmp_path, env, nvcc_version):
     template_args = {