From 494b908c6841970cd5567de694c8631decb2a8ac Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Sat, 30 Apr 2022 14:19:02 -0700 Subject: [PATCH 1/2] Fix determinism for generated `setup.py` files. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/backend/python/goals/setup_py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/goals/setup_py.py b/src/python/pants/backend/python/goals/setup_py.py index 6bdee341886..457756b3d4b 100644 --- a/src/python/pants/backend/python/goals/setup_py.py +++ b/src/python/pants/backend/python/goals/setup_py.py @@ -954,7 +954,7 @@ def maybe_add_resource(fp: str) -> None: return ( tuple(sorted(packages)), tuple(sorted(namespace_packages)), - tuple((pkg, tuple(sorted(files))) for pkg, files in package_data.items()), + tuple((pkg, tuple(sorted(files))) for pkg, files in sorted(package_data.items())), ) From a9634ab35149ff3f3a6001868832c952bc7e4b7c Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Sat, 30 Apr 2022 17:34:18 -0700 Subject: [PATCH 2/2] Test that `package_data` entries are deterministic. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/goals/setup_py_integration_test.py | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/goals/setup_py_integration_test.py b/src/python/pants/backend/python/goals/setup_py_integration_test.py index 604bb854222..14eafa62637 100644 --- a/src/python/pants/backend/python/goals/setup_py_integration_test.py +++ b/src/python/pants/backend/python/goals/setup_py_integration_test.py @@ -3,13 +3,17 @@ from __future__ import annotations +import hashlib import os import subprocess import sys import venv +from pathlib import Path from tempfile import TemporaryDirectory +from textwrap import dedent -from pants.testutil.pants_integration_test import run_pants +from pants.testutil.pants_integration_test import run_pants, setup_tmpdir +from pants.util.dirutil import safe_rmtree def test_native_code() -> None: @@ -43,3 +47,64 @@ def test_native_code() -> None: capture_output=True, ) assert proc.stdout == b"Professor Native\n" + + +def package_determinism(expected_artifact_count: int, files: dict[str, str]) -> None: + """Tests that the given sources can be `package`d reproducibly.""" + + def digest(path: str) -> tuple[str, str]: + d = hashlib.sha256(Path(path).read_bytes()).hexdigest() + return path, d + + def run_and_digest(address: str) -> dict[str, str]: + safe_rmtree("dist") + pants_run = run_pants( + [ + "--backend-packages=pants.backend.python", + "--no-pantsd", + "package", + address, + ], + ) + pants_run.assert_success() + return dict(digest(os.path.join("dist", f)) for f in os.listdir("dist")) + + with setup_tmpdir(files) as source_dir: + one = run_and_digest(f"{source_dir}:dist") + two = run_and_digest(f"{source_dir}:dist") + + assert len(one) == expected_artifact_count + assert one == two + + +def test_deterministic_package_data() -> None: + package_determinism( + 2, + { + "BUILD": dedent( + """\ + python_distribution( + name="dist", + dependencies=["{tmpdir}/a", "{tmpdir}/b"], + provides=python_artifact(name="det", version="2.3.4"), + ) + """ + ), + "a/BUILD": dedent( + """\ + python_sources(dependencies=[":resources"]) + resources(name="resources", sources=["*.txt"]) + """ + ), + "a/source.py": "", + "a/a.txt": "", + "b/BUILD": dedent( + """\ + python_sources(dependencies=[":resources"]) + resources(name="resources", sources=["*.txt"]) + """ + ), + "b/source.py": "", + "b/b.txt": "", + }, + )