From d1d78cc721ef60723e90a8c49de89fbbea66d3db Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 11:30:05 +0000
Subject: [PATCH 1/9] [pinpy] Preserve multiple versions of packages

---
 .../python/translators/pip-freeze/default.nix        | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 6cca0db1c9..19b996f480 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -129,6 +129,12 @@ in {
       type = "http";
       inherit hash url;
     };
+    sources =
+      l.foldl
+      # Multiple versions are not supported, but preserved here through deep update.
+      (all: req: all // {${req.name} = all.${req.name} or {} // {${req.version} = getSource req;};})
+      {}
+      requirements;
   in
     # see example in src/specifications/dream-lock-example.json
     {
@@ -156,11 +162,7 @@ in {
       cyclicDependencies = {};
       dependencies = {};
 
-      sources =
-        l.foldl
-        (all: req: all // {"${req.name}"."${req.version}" = getSource req;})
-        {}
-        requirements;
+      inherit sources;
     };
 
   extraArgs = {

From e977d7d751f04d18795d48cb6feca74fd5068f32 Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 11:31:24 +0000
Subject: [PATCH 2/9] [pinpy] Support reading multiple requirements files

---
 examples/python_pip-freeze/flake.nix                   | 10 ++++++++--
 .../python/translators/pip-freeze/default.nix          |  6 ++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/examples/python_pip-freeze/flake.nix b/examples/python_pip-freeze/flake.nix
index 50151c46aa..66a2d00d21 100644
--- a/examples/python_pip-freeze/flake.nix
+++ b/examples/python_pip-freeze/flake.nix
@@ -30,8 +30,14 @@
             name = "rosbags";
             subsystem = "python";
             translator = "pip-freeze";
-            subsystemInfo.system = system;
-            subsystemInfo.pythonVersion = "3.10";
+            subsystemInfo = {
+              system = system;
+              pythonVersion = "3.10";
+              requirementsFiles = [
+                "requirements.txt"
+                "requirements-dev.txt"
+              ];
+            };
           };
         };
         # checks.package = config.dream2nix.outputs.rosbags.packages.default;
diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 19b996f480..6b6b02f5f2 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -50,6 +50,7 @@ in {
     tree,
     pythonVersion,
     system,
+    requirementsFiles ? ["requirements.txt"],
     ...
   }:
   # if system == null
@@ -83,6 +84,7 @@ in {
     };
 
     nameVersion = builtins.match ''^([[:alnum:]\.\_\-]+)[^=]*==([^[:space:];\]+).*'';
+    # [{ name = "foo"; version = "1.2.3"; }, ...]
     readRequirements = path: let
       lines = l.splitString "\n" (l.readFile path);
       matched = l.filter (m: m != null) (l.map (line: nameVersion line) lines);
@@ -95,7 +97,7 @@ in {
     in
       reqs;
 
-    requirements = readRequirements "${projectSource}/requirements-dev.txt";
+    reqList = l.concatLists (l.map (file: readRequirements "${projectSource}/${file}") requirementsFiles);
 
     defaultPackageName = "default"; # pyproject.toml
     defaultPackageVersion = "unknown-version";
@@ -134,7 +136,7 @@ in {
       # Multiple versions are not supported, but preserved here through deep update.
       (all: req: all // {${req.name} = all.${req.name} or {} // {${req.version} = getSource req;};})
       {}
-      requirements;
+      reqList;
   in
     # see example in src/specifications/dream-lock-example.json
     {

From 49fd195059df50c1e7261b831f8ed0233de24428 Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 12:07:45 +0000
Subject: [PATCH 3/9] [pinpy] Pass requirements list, use find-links

---
 src/subsystems/python/builders/simple-python/default.nix | 5 +++--
 src/subsystems/python/translators/pip-freeze/default.nix | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix
index 6312b6caac..81b4379057 100644
--- a/src/subsystems/python/builders/simple-python/default.nix
+++ b/src/subsystems/python/builders/simple-python/default.nix
@@ -37,6 +37,7 @@
       (src: src.original or src)
       allDependencySources';
 
+    reqArgs = l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList);
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
       src = getSource defaultPackageName defaultPackageVersion;
@@ -56,8 +57,8 @@
         done
         mkdir -p "$out/${python.sitePackages}"
         export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-        ${python}/bin/python -m pip install \
-          ./dist/*.{whl,tar.gz,zip} \
+        ${python}/bin/python -m pip install ${reqArgs} \
+          --find-links ./dist/ \
           --no-build-isolation \
           --no-index \
           --no-warn-script-location \
diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 6b6b02f5f2..5e115168f1 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -156,6 +156,7 @@ in {
       };
 
       _subsystem = {
+        inherit reqList;
         application = false;
         pythonAttr = "python3";
         sourceFormats = {};

From be9d31c7e5344a40561206a1ae23eda9c700038a Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 12:49:08 +0000
Subject: [PATCH 4/9] [pinpy] Install build requirements before rest

---
 examples/python_pip-freeze/flake.nix          |  4 ++
 .../python/builders/simple-python/default.nix | 57 ++++++++++++-------
 .../python/translators/pip-freeze/default.nix | 15 ++++-
 3 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/examples/python_pip-freeze/flake.nix b/examples/python_pip-freeze/flake.nix
index 66a2d00d21..a479c229cd 100644
--- a/examples/python_pip-freeze/flake.nix
+++ b/examples/python_pip-freeze/flake.nix
@@ -37,6 +37,10 @@
                 "requirements.txt"
                 "requirements-dev.txt"
               ];
+              # For now, just installed into same environment before the rest.
+              buildRequires = {
+                pytest-runner = "6.0.0"; # flake8-mutable
+              };
             };
           };
         };
diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix
index 81b4379057..0c74358b2e 100644
--- a/src/subsystems/python/builders/simple-python/default.nix
+++ b/src/subsystems/python/builders/simple-python/default.nix
@@ -37,6 +37,8 @@
       (src: src.original or src)
       allDependencySources';
 
+    buildReq = subsystemAttrs.buildRequires or {};
+    buildReqArgs = l.concatStringsSep " " (l.map (name: "${name}==${buildReq.${name}}") (l.attrNames buildReq));
     reqArgs = l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList);
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
@@ -47,25 +49,42 @@
       propagatedBuildInputs = [python.pkgs.setuptools];
       doCheck = false;
       dontStrip = true;
-      preBuild = ''
-        mkdir dist
-        for file in ${builtins.toString allDependencySources}; do
-          # pick right most element of path
-          fname=''${file##*/}
-          fname=$(stripHash $fname)
-          cp $file dist/$fname
-        done
-        mkdir -p "$out/${python.sitePackages}"
-        export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-        ${python}/bin/python -m pip install ${reqArgs} \
-          --find-links ./dist/ \
-          --no-build-isolation \
-          --no-index \
-          --no-warn-script-location \
-          --prefix="$out" \
-          --no-cache \
-          $pipInstallFlags
-      '';
+      preBuild =
+        ''
+          mkdir dist
+          for file in ${builtins.toString allDependencySources}; do
+            # pick right most element of path
+            fname=''${file##*/}
+            fname=$(stripHash $fname)
+            cp $file dist/$fname
+          done
+          mkdir -p "$out/${python.sitePackages}"
+          export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
+        ''
+        + (
+          if buildReq != {}
+          then ''
+            ${python}/bin/python -m pip install ${buildReqArgs} \
+              --find-links ./dist/ \
+              --no-build-isolation \
+              --no-index \
+              --no-warn-script-location \
+              --prefix="$out" \
+              --no-cache \
+              $pipInstallFlags
+          ''
+          else ""
+        )
+        + ''
+          ${python}/bin/python -m pip install ${reqArgs} \
+            --find-links ./dist/ \
+            --no-build-isolation \
+            --no-index \
+            --no-warn-script-location \
+            --prefix="$out" \
+            --no-cache \
+            $pipInstallFlags
+        '';
     });
 
     devShell = pkgs.mkShell {
diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 5e115168f1..be70aff457 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -51,6 +51,7 @@ in {
     pythonVersion,
     system,
     requirementsFiles ? ["requirements.txt"],
+    buildRequires ? {},
     ...
   }:
   # if system == null
@@ -131,12 +132,22 @@ in {
       type = "http";
       inherit hash url;
     };
-    sources =
+    sources' =
       l.foldl
       # Multiple versions are not supported, but preserved here through deep update.
       (all: req: all // {${req.name} = all.${req.name} or {} // {${req.version} = getSource req;};})
       {}
       reqList;
+
+    sources = l.foldl (all: name:
+      all
+      // {
+        ${name}.${buildRequires.${name}} = getSource {
+          inherit name;
+          version = buildRequires.${name};
+        };
+      })
+    sources' (l.attrNames buildRequires);
   in
     # see example in src/specifications/dream-lock-example.json
     {
@@ -156,7 +167,7 @@ in {
       };
 
       _subsystem = {
-        inherit reqList;
+        inherit reqList buildRequires;
         application = false;
         pythonAttr = "python3";
         sourceFormats = {};

From 5b674a48659b46557bd8ffcb2d66ef432f8ce5ce Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 13:50:46 +0000
Subject: [PATCH 5/9] [pinpy] Detect python package format

---
 src/subsystems/python/builders/simple-python/default.nix | 2 +-
 src/subsystems/python/translators/pip-freeze/default.nix | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix
index 0c74358b2e..385eb46773 100644
--- a/src/subsystems/python/builders/simple-python/default.nix
+++ b/src/subsystems/python/builders/simple-python/default.nix
@@ -43,7 +43,7 @@
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
       src = getSource defaultPackageName defaultPackageVersion;
-      format = "setuptools";
+      format = subsystemAttrs.packageFormat or "setuptools";
       buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
       nativeBuildInputs = [pkgs.autoPatchelfHook];
       propagatedBuildInputs = [python.pkgs.setuptools];
diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index be70aff457..36634b3b4c 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -171,6 +171,10 @@ in {
         application = false;
         pythonAttr = "python3";
         sourceFormats = {};
+        packageFormat =
+          if l.pathExists "${projectSource}/setup.py"
+          then "setuptools"
+          else "pyproject.toml";
       };
 
       cyclicDependencies = {};

From a5ff9901aee421b022beed26e9c078052b5ae8ed Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 14:04:39 +0000
Subject: [PATCH 6/9] [pinpy] Let pip handle hashes and markers

---
 .../python/builders/simple-python/default.nix         | 11 ++++++++++-
 .../python/translators/pip-freeze/default.nix         |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix
index 385eb46773..627cb763e9 100644
--- a/src/subsystems/python/builders/simple-python/default.nix
+++ b/src/subsystems/python/builders/simple-python/default.nix
@@ -38,8 +38,17 @@
       allDependencySources';
 
     buildReq = subsystemAttrs.buildRequires or {};
+    requirementsFiles = subsystemAttrs.requirementsFiles or {};
     buildReqArgs = l.concatStringsSep " " (l.map (name: "${name}==${buildReq.${name}}") (l.attrNames buildReq));
-    reqArgs = l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList);
+    # Requirements files may contain hashes and markers; we let pip handle
+    # these. As a fallback we support a [ { name = 'foo'; version = '1.2.3'; },
+    # ... ] list which we might want to generate from the deps stored in
+    # dreamlock already.
+    reqArgs =
+      if requirementsFiles != {}
+      then l.concatStringsSep " " (l.map (x: "-r ${x}") requirementsFiles)
+      else l.concatStringsSep " " (l.map (x: "${x.name}==${x.version}") subsystemAttrs.reqList or []);
+
     package = produceDerivation defaultPackageName (buildFunc {
       name = defaultPackageName;
       src = getSource defaultPackageName defaultPackageVersion;
diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 36634b3b4c..2390866f4e 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -167,7 +167,7 @@ in {
       };
 
       _subsystem = {
-        inherit reqList buildRequires;
+        inherit reqList buildRequires requirementsFiles;
         application = false;
         pythonAttr = "python3";
         sourceFormats = {};

From 421cd8a55493e4fca31f3c7276cff5a11a95696c Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 18:08:44 +0000
Subject: [PATCH 7/9] [pinpy] Pick python from selected version

---
 src/subsystems/python/translators/pip-freeze/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/subsystems/python/translators/pip-freeze/default.nix b/src/subsystems/python/translators/pip-freeze/default.nix
index 2390866f4e..b749821709 100644
--- a/src/subsystems/python/translators/pip-freeze/default.nix
+++ b/src/subsystems/python/translators/pip-freeze/default.nix
@@ -169,7 +169,7 @@ in {
       _subsystem = {
         inherit reqList buildRequires requirementsFiles;
         application = false;
-        pythonAttr = "python3";
+        pythonAttr = "python${l.replaceStrings ["."] [""] pythonVersion}";
         sourceFormats = {};
         packageFormat =
           if l.pathExists "${projectSource}/setup.py"

From 74da597055b33f7c63349eb4e878b28256ab75e6 Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 18:24:01 +0000
Subject: [PATCH 8/9] [pinpy] Enable installing wheel as buildRequires

---
 src/subsystems/python/builders/simple-python/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix
index 627cb763e9..8844e8352f 100644
--- a/src/subsystems/python/builders/simple-python/default.nix
+++ b/src/subsystems/python/builders/simple-python/default.nix
@@ -27,7 +27,7 @@
       l.flatten
       (l.mapAttrsToList
         (name: versions:
-          if l.elem name [defaultPackageName "setuptools" "pip" "wheel"]
+          if l.elem name [defaultPackageName "setuptools" "pip"]
           then []
           else l.map (ver: getSource name ver) versions)
         packageVersions);

From fac4bcd682788b7a38a816c83fd641d63e0a059f Mon Sep 17 00:00:00 2001
From: Florian Friesdorf <florian@ternaris.com>
Date: Fri, 25 Nov 2022 19:58:26 +0000
Subject: [PATCH 9/9] Update poetry2nix for improved selection of wheels

---
 flake.lock | 7 +++----
 flake.nix  | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index a23219273e..37525d9bf1 100644
--- a/flake.lock
+++ b/flake.lock
@@ -241,16 +241,15 @@
     "poetry2nix": {
       "flake": false,
       "locked": {
-        "lastModified": 1632969109,
-        "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=",
+        "lastModified": 1669348052,
+        "narHash": "sha256-e8ySgeCwNTVEDX3irTEsAtSFnzNKiPbr7GzoRw7Aa/8=",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77",
+        "rev": "6cf6701a4844041528bba6973b801bda753d5b43",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "1.21.0",
         "repo": "poetry2nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 8cc764e3a2..bffe4d9486 100644
--- a/flake.nix
+++ b/flake.nix
@@ -46,7 +46,7 @@
 
     # required for utils.satisfiesSemver
     poetry2nix = {
-      url = "github:nix-community/poetry2nix/1.21.0";
+      url = "github:nix-community/poetry2nix";
       flake = false;
     };