diff --git a/examples/python_pip-freeze/flake.nix b/examples/python_pip-freeze/flake.nix index 50151c46aa..a479c229cd 100644 --- a/examples/python_pip-freeze/flake.nix +++ b/examples/python_pip-freeze/flake.nix @@ -30,8 +30,18 @@ name = "rosbags"; subsystem = "python"; translator = "pip-freeze"; - subsystemInfo.system = system; - subsystemInfo.pythonVersion = "3.10"; + subsystemInfo = { + system = system; + pythonVersion = "3.10"; + requirementsFiles = [ + "requirements.txt" + "requirements-dev.txt" + ]; + # For now, just installed into same environment before the rest. + buildRequires = { + pytest-runner = "6.0.0"; # flake8-mutable + }; + }; }; }; # checks.package = config.dream2nix.outputs.rosbags.packages.default; diff --git a/flake.lock b/flake.lock index a23219273e..37525d9bf1 100644 --- a/flake.lock +++ b/flake.lock @@ -241,16 +241,15 @@ "poetry2nix": { "flake": false, "locked": { - "lastModified": 1632969109, - "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=", + "lastModified": 1669348052, + "narHash": "sha256-e8ySgeCwNTVEDX3irTEsAtSFnzNKiPbr7GzoRw7Aa/8=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77", + "rev": "6cf6701a4844041528bba6973b801bda753d5b43", "type": "github" }, "original": { "owner": "nix-community", - "ref": "1.21.0", "repo": "poetry2nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 8cc764e3a2..bffe4d9486 100644 --- a/flake.nix +++ b/flake.nix @@ -46,7 +46,7 @@ # required for utils.satisfiesSemver poetry2nix = { - url = "github:nix-community/poetry2nix/1.21.0"; + url = "github:nix-community/poetry2nix"; flake = false; }; diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix index 6312b6caac..8844e8352f 100644 --- a/src/subsystems/python/builders/simple-python/default.nix +++ b/src/subsystems/python/builders/simple-python/default.nix @@ -27,7 +27,7 @@ l.flatten (l.mapAttrsToList (name: versions: - if l.elem name [defaultPackageName "setuptools" "pip" "wheel"] + if l.elem name [defaultPackageName "setuptools" "pip"] then [] else l.map (ver: getSource name ver) versions) packageVersions); @@ -37,34 +37,63 @@ (src: src.original or src) allDependencySources'; + buildReq = subsystemAttrs.buildRequires or {}; + requirementsFiles = subsystemAttrs.requirementsFiles or {}; + buildReqArgs = l.concatStringsSep " " (l.map (name: "${name}==${buildReq.${name}}") (l.attrNames buildReq)); + # Requirements files may contain hashes and markers; we let pip handle + # these. As a fallback we support a [ { name = 'foo'; version = '1.2.3'; }, + # ... ] list which we might want to generate from the deps stored in + # dreamlock already. + reqArgs = + if requirementsFiles != {} + then l.concatStringsSep " " (l.map (x: "-r ${x}") requirementsFiles) + else l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList or []); + package = produceDerivation defaultPackageName (buildFunc { name = defaultPackageName; src = getSource defaultPackageName defaultPackageVersion; - format = "setuptools"; + format = subsystemAttrs.packageFormat or "setuptools"; buildInputs = pkgs.pythonManylinuxPackages.manylinux1; nativeBuildInputs = [pkgs.autoPatchelfHook]; propagatedBuildInputs = [python.pkgs.setuptools]; doCheck = false; dontStrip = true; - preBuild = '' - mkdir dist - for file in ${builtins.toString allDependencySources}; do - # pick right most element of path - fname=''${file##*/} - fname=$(stripHash $fname) - cp $file dist/$fname - done - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" - ${python}/bin/python -m pip install \ - ./dist/*.{whl,tar.gz,zip} \ - --no-build-isolation \ - --no-index \ - --no-warn-script-location \ - --prefix="$out" \ - --no-cache \ - $pipInstallFlags - ''; + preBuild = + '' + mkdir dist + for file in ${builtins.toString allDependencySources}; do + # pick right most element of path + fname=''${file##*/} + fname=$(stripHash $fname) + cp $file dist/$fname + done + mkdir -p "$out/${python.sitePackages}" + export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" + '' + + ( + if buildReq != {} + then '' + ${python}/bin/python -m pip install ${buildReqArgs} \ + --find-links ./dist/ \ + --no-build-isolation \ + --no-index \ + --no-warn-script-location \ + --prefix="$out" \ + --no-cache \ + $pipInstallFlags + '' + else "" + ) + + '' + ${python}/bin/python -m pip install ${reqArgs} \ + --find-links ./dist/ \ + --no-build-isolation \ + --no-index \ + --no-warn-script-location \ + --prefix="$out" \ + --no-cache \ + $pipInstallFlags + ''; }); devShell = pkgs.mkShell { diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix index 6cca0db1c9..b749821709 100644 --- a/src/subsystems/python/translators/pip-freeze/default.nix +++ b/src/subsystems/python/translators/pip-freeze/default.nix @@ -50,6 +50,8 @@ in { tree, pythonVersion, system, + requirementsFiles ? ["requirements.txt"], + buildRequires ? {}, ... }: # if system == null @@ -83,6 +85,7 @@ in { }; nameVersion = builtins.match ''^([[:alnum:]\.\_\-]+)[^=]*==([^[:space:];\]+).*''; + # [{ name = "foo"; version = "1.2.3"; }, ...] readRequirements = path: let lines = l.splitString "\n" (l.readFile path); matched = l.filter (m: m != null) (l.map (line: nameVersion line) lines); @@ -95,7 +98,7 @@ in { in reqs; - requirements = readRequirements "${projectSource}/requirements-dev.txt"; + reqList = l.concatLists (l.map (file: readRequirements "${projectSource}/${file}") requirementsFiles); defaultPackageName = "default"; # pyproject.toml defaultPackageVersion = "unknown-version"; @@ -129,6 +132,22 @@ in { type = "http"; inherit hash url; }; + sources' = + l.foldl + # Multiple versions are not supported, but preserved here through deep update. + (all: req: all // {${req.name} = all.${req.name} or {} // {${req.version} = getSource req;};}) + {} + reqList; + + sources = l.foldl (all: name: + all + // { + ${name}.${buildRequires.${name}} = getSource { + inherit name; + version = buildRequires.${name}; + }; + }) + sources' (l.attrNames buildRequires); in # see example in src/specifications/dream-lock-example.json { @@ -148,19 +167,20 @@ in { }; _subsystem = { + inherit reqList buildRequires requirementsFiles; application = false; - pythonAttr = "python3"; + pythonAttr = "python${l.replaceStrings ["."] [""] pythonVersion}"; sourceFormats = {}; + packageFormat = + if l.pathExists "${projectSource}/setup.py" + then "setuptools" + else "pyproject.toml"; }; cyclicDependencies = {}; dependencies = {}; - sources = - l.foldl - (all: req: all // {"${req.name}"."${req.version}" = getSource req;}) - {} - requirements; + inherit sources; }; extraArgs = {