Support build requirements for pinned python

examples/python_pip-freeze/flake.nix

@@ -30,8 +30,18 @@
             name = "rosbags";
             subsystem = "python";
             translator = "pip-freeze";
-            subsystemInfo.system = system;
-            subsystemInfo.pythonVersion = "3.10";
+            subsystemInfo = {
+              system = system;
+              pythonVersion = "3.10";
+              requirementsFiles = [
+                "requirements.txt"
+                "requirements-dev.txt"
+              ];
+              # For now, just installed into same environment before the rest.
+              buildRequires = {
+                pytest-runner = "6.0.0"; # flake8-mutable
+              };
+            };
           };
         };
         # checks.package = config.dream2nix.outputs.rosbags.packages.default;

flake.nix

@@ -46,7 +46,7 @@
 
     # required for utils.satisfiesSemver
     poetry2nix = {
-      url = "github:nix-community/poetry2nix/1.21.0";
+      url = "github:nix-community/poetry2nix";
       flake = false;
     };
 

src/subsystems/python/builders/simple-python/default.nix

@@ -27,7 +27,7 @@
       l.flatten
       (l.mapAttrsToList
         (name: versions:
-          if l.elem name [defaultPackageName "setuptools" "pip" "wheel"]
+          if l.elem name [defaultPackageName "setuptools" "pip"]
           then []
           else l.map (ver: getSource name ver) versions)
         packageVersions);
@@ -37,34 +37,63 @@
       (src: src.original or src)
       allDependencySources';
 
+    buildReq = subsystemAttrs.buildRequires or {};
+    requirementsFiles = subsystemAttrs.requirementsFiles or {};
+    buildReqArgs = l.concatStringsSep " " (l.map (name: "${name}==${buildReq.${name}}") (l.attrNames buildReq));
+    # Requirements files may contain hashes and markers; we let pip handle
+    # these. As a fallback we support a [ { name = 'foo'; version = '1.2.3'; },
+    # ... ] list which we might want to generate from the deps stored in
+    # dreamlock already.
+    reqArgs =
+      if requirementsFiles != {}
+      then l.concatStringsSep " " (l.map (x: "-r ${x}") requirementsFiles)
+      else l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList or []);
+
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
       src = getSource defaultPackageName defaultPackageVersion;
-      format = "setuptools";
+      format = subsystemAttrs.packageFormat or "setuptools";
       buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
       nativeBuildInputs = [pkgs.autoPatchelfHook];
       propagatedBuildInputs = [python.pkgs.setuptools];
       doCheck = false;
       dontStrip = true;
-      preBuild = ''
-        mkdir dist
-        for file in ${builtins.toString allDependencySources}; do
-          # pick right most element of path
-          fname=''${file##*/}
-          fname=$(stripHash $fname)
-          cp $file dist/$fname
-        done
-        mkdir -p "$out/${python.sitePackages}"
-        export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-        ${python}/bin/python -m pip install \
-          ./dist/*.{whl,tar.gz,zip} \
-          --no-build-isolation \
-          --no-index \
-          --no-warn-script-location \
-          --prefix="$out" \
-          --no-cache \
-          $pipInstallFlags
-      '';
+      preBuild =
+        ''
+          mkdir dist
+          for file in ${builtins.toString allDependencySources}; do
+            # pick right most element of path
+            fname=''${file##*/}
+            fname=$(stripHash $fname)
+            cp $file dist/$fname
+          done
+          mkdir -p "$out/${python.sitePackages}"
+          export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
+        ''
+        + (
+          if buildReq != {}
+          then ''
+            ${python}/bin/python -m pip install ${buildReqArgs} \
+              --find-links ./dist/ \
+              --no-build-isolation \
+              --no-index \
+              --no-warn-script-location \
+              --prefix="$out" \
+              --no-cache \
+              $pipInstallFlags
+          ''
+          else ""
+        )
+        + ''
+          ${python}/bin/python -m pip install ${reqArgs} \
+            --find-links ./dist/ \
+            --no-build-isolation \
+            --no-index \
+            --no-warn-script-location \
+            --prefix="$out" \
+            --no-cache \
+            $pipInstallFlags
+        '';
     });
 
     devShell = pkgs.mkShell {

src/subsystems/python/translators/pip-freeze/default.nix

@@ -50,6 +50,8 @@ in {
     tree,
     pythonVersion,
     system,
+    requirementsFiles ? ["requirements.txt"],
+    buildRequires ? {},
     ...
   }:
   # if system == null
@@ -83,6 +85,7 @@ in {
     };
 
     nameVersion = builtins.match ''^([[:alnum:]\.\_\-]+)[^=]*==([^[:space:];\]+).*'';
+    # [{ name = "foo"; version = "1.2.3"; }, ...]
     readRequirements = path: let
       lines = l.splitString "\n" (l.readFile path);
       matched = l.filter (m: m != null) (l.map (line: nameVersion line) lines);
@@ -95,7 +98,7 @@ in {
     in
       reqs;
 
-    requirements = readRequirements "${projectSource}/requirements-dev.txt";
+    reqList = l.concatLists (l.map (file: readRequirements "${projectSource}/${file}") requirementsFiles);
 
     defaultPackageName = "default"; # pyproject.toml
     defaultPackageVersion = "unknown-version";
@@ -129,6 +132,22 @@ in {
       type = "http";
       inherit hash url;
     };
+    sources' =
+      l.foldl
+      # Multiple versions are not supported, but preserved here through deep update.
+      (all: req: all // {${req.name} = all.${req.name} or {} // {${req.version} = getSource req;};})
+      {}
+      reqList;
+
+    sources = l.foldl (all: name:
+      all
+      // {
+        ${name}.${buildRequires.${name}} = getSource {
+          inherit name;
+          version = buildRequires.${name};
+        };
+      })
+    sources' (l.attrNames buildRequires);
   in
     # see example in src/specifications/dream-lock-example.json
     {
@@ -148,19 +167,20 @@ in {
       };
 
       _subsystem = {
+        inherit reqList buildRequires requirementsFiles;
         application = false;
-        pythonAttr = "python3";
+        pythonAttr = "python${l.replaceStrings ["."] [""] pythonVersion}";
         sourceFormats = {};
+        packageFormat =
+          if l.pathExists "${projectSource}/setup.py"
+          then "setuptools"
+          else "pyproject.toml";
       };
 
       cyclicDependencies = {};
       dependencies = {};
 
-      sources =
-        l.foldl
-        (all: req: all // {"${req.name}"."${req.version}" = getSource req;})
-        {}
-        requirements;
+      inherit sources;
     };
 
   extraArgs = {