Better lockfile generation withpip-compile (#1075)

* Specify manifest path in pip-compile

* Recognize more python requirements filenames

* Note Python changes in CHANGELOG

* Allow `phylum parse -t pip pyproject.toml`

Keep `poetry` as the default generator for `pyproject.toml` files
because it is very common. But allow the user to use `-t pip` for other
backends (requires `pip-compile`)

CHANGELOG.md

@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Added
+- Support more Python manifest files: `requirements.in`, `setup.py`, `setup.cfg`
+- Recognize all `requirements*.txt` file names as Python lockfiles
+
 ## [5.1.0] - 2023-05-04
 
 ### Added

cli/src/commands/parse.rs

@@ -96,10 +96,7 @@ pub fn parse_lockfile(
     eprintln!("Generating lockfile for manifest {path:?} using {format:?}…");
 
     // Generate a new lockfile.
-    let canonicalized = fs::canonicalize(&path)?;
-    let project_path =
-        canonicalized.parent().ok_or_else(|| anyhow!("invalid manifest path: {path:?}"))?;
-    let generated_lockfile = generator.generate_lockfile(project_path)?;
+    let generated_lockfile = generator.generate_lockfile(&path)?;
 
     // Parse the generated lockfile.
     let packages = parse_lockfile_content(&generated_lockfile, parser)?;

docs/command_line_tool/phylum_analyze.md

@@ -31,7 +31,7 @@ Usage: phylum analyze [OPTIONS] [LOCKFILE]...
 
 -t, --lockfile-type <type>
 &emsp; Lock file type used for all lock files (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `gem`, `pip`, `pipenv`, `poetry`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `gem`, `poetry`, `pip`, `pipenv`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
 
 -v, --verbose...
 &emsp; Increase the level of verbosity (the maximum is -vvv)

docs/command_line_tool/phylum_init.md

@@ -25,7 +25,7 @@ Usage: phylum init [OPTIONS] [PROJECT_NAME]
 
 -t, --lockfile-type <type>
 &emsp; Lock file type used for all lock files (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `gem`, `pip`, `pipenv`, `poetry`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `gem`, `poetry`, `pip`, `pipenv`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
 
 -f, --force
 &emsp; Overwrite existing configurations without confirmation

docs/command_line_tool/phylum_parse.md

@@ -19,7 +19,7 @@ Usage: phylum parse [OPTIONS] [LOCKFILE]...
 
 -t, --lockfile-type <type>
 &emsp; Lock file type used for all lock files (default: auto)
-&emsp; Accepted values: `npm`, `yarn`, `gem`, `pip`, `pipenv`, `poetry`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
+&emsp; Accepted values: `npm`, `yarn`, `gem`, `poetry`, `pip`, `pipenv`, `mvn`, `gradle`, `nuget`, `go`, `cargo`, `spdx`, `auto`
 
 -v, --verbose...
 &emsp; Increase the level of verbosity (the maximum is -vvv)

lockfile/src/lib.rs

@@ -136,9 +136,9 @@ impl Iterator for LockfileFormatIter {
             0 => LockfileFormat::Npm,
             1 => LockfileFormat::Yarn,
             2 => LockfileFormat::Gem,
-            3 => LockfileFormat::Pip,
-            4 => LockfileFormat::Pipenv,
-            5 => LockfileFormat::Poetry,
+            3 => LockfileFormat::Poetry,
+            4 => LockfileFormat::Pip,
+            5 => LockfileFormat::Pipenv,
             6 => LockfileFormat::Maven,
             7 => LockfileFormat::Gradle,
             8 => LockfileFormat::Msbuild,

lockfile/src/python.rs

@@ -23,6 +23,13 @@ pub struct PyRequirements;
 pub struct PipFile;
 pub struct Poetry;
 
+/// Check if filename is `requirements*.txt`
+fn is_requirements_file(path: &Path) -> bool {
+    path.file_name().and_then(|f| f.to_str()).map_or(false, |file_name| {
+        file_name.starts_with("requirements") && file_name.ends_with(".txt")
+    })
+}
+
 impl Parse for PyRequirements {
     /// Parses `requirements.txt` files into a vec of packages
     fn parse(&self, data: &str) -> anyhow::Result<Vec<Package>> {
@@ -34,11 +41,15 @@ impl Parse for PyRequirements {
     }
 
     fn is_path_lockfile(&self, path: &Path) -> bool {
-        path.file_name() == Some(OsStr::new("requirements.txt"))
+        is_requirements_file(path)
     }
 
     fn is_path_manifest(&self, path: &Path) -> bool {
-        path.file_name() == Some(OsStr::new("requirements.txt"))
+        path.file_name() == Some(OsStr::new("requirements.in"))
+            || path.file_name() == Some(OsStr::new("pyproject.toml"))
+            || path.file_name() == Some(OsStr::new("setup.cfg"))
+            || path.file_name() == Some(OsStr::new("setup.py"))
+            || is_requirements_file(path)
     }
 
     #[cfg(feature = "generator")]

lockfile_generator/src/bundler.rs

@@ -1,5 +1,6 @@
 //! Ruby bundler ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Bundler {
         "Gemfile.lock"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("bundle");
         command.args(["lock"]);
         command

lockfile_generator/src/cargo.rs

@@ -1,5 +1,6 @@
 //! Rust cargo ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Cargo {
         "Cargo.lock"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("cargo");
         command.args(["generate-lockfile"]);
         command

lockfile_generator/src/go.rs

@@ -1,5 +1,6 @@
 //! Go ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Go {
         "go.sum"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("go");
         command.args(["get", "-d"]);
         command

lockfile_generator/src/gradle.rs

@@ -1,5 +1,6 @@
 //! Java gradle ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Gradle {
         "gradle.lockfile"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("gradle");
         command.args(["dependencies", "--write-locks"]);
         command

lockfile_generator/src/lib.rs

@@ -20,7 +20,7 @@ pub trait Generator {
     fn lockfile_name(&self) -> &'static str;
 
     /// Command for generating the lockfile.
-    fn command(&self) -> Command;
+    fn command(&self, manifest_path: &Path) -> Command;
 
     /// List of files conflicting with lockfile generation.
     ///
@@ -34,7 +34,12 @@ pub trait Generator {
     ///
     /// This will ignore all existing lockfiles and create a new lockfile based
     /// on the current project configuration.
-    fn generate_lockfile(&self, project_path: &Path) -> Result<String> {
+    fn generate_lockfile(&self, manifest_path: &Path) -> Result<String> {
+        let canonicalized = fs::canonicalize(manifest_path)?;
+        let project_path = canonicalized
+            .parent()
+            .ok_or_else(|| Error::InvalidManifest(manifest_path.to_path_buf()))?;
+
         // Move files which interfere with lockfile generation.
         let _relocators = self
             .conflicting_files()
@@ -43,7 +48,7 @@ pub trait Generator {
             .collect::<Result<Vec<_>>>()?;
 
         // Generate lockfile at the target location.
-        let mut command = self.command();
+        let mut command = self.command(&canonicalized);
         command.current_dir(project_path);
         command.stdin(Stdio::null());
         command.stdout(Stdio::null());
@@ -117,6 +122,8 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub enum Error {
     #[error("I/O error: {0}")]
     Io(#[from] io::Error),
+    #[error("invalid manifest path: {0:?}")]
+    InvalidManifest(PathBuf),
     #[error("failed to spawn command {0}: {1}")]
     ProcessCreation(String, io::Error),
     #[error("package manager exited with non-zero status")]

lockfile_generator/src/maven.rs

@@ -1,5 +1,6 @@
 //! Java maven ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Maven {
         "effective-pom.xml"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("mvn");
         command.args(["help:effective-pom", &format!("-Doutput={}", self.lockfile_name())]);
         command

lockfile_generator/src/npm.rs

@@ -1,5 +1,6 @@
 //! JavaScript npm ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -15,7 +16,7 @@ impl Generator for Npm {
         vec![self.lockfile_name(), "npm-shrinkwrap.json", "yarn.lock"]
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("npm");
         command.args(["install", "--package-lock-only", "--ignore-scripts"]);
         command

lockfile_generator/src/pip.rs

@@ -1,5 +1,6 @@
 //! Python pipenv ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Pip {
         "Pipfile.lock"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("pipenv");
         command.args(["lock"]);
         command

lockfile_generator/src/poetry.rs

@@ -1,5 +1,6 @@
 //! Python poetry ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Poetry {
         "poetry.lock"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("poetry");
         command.args(["lock"]);
         command

lockfile_generator/src/python_requirements.rs

@@ -1,5 +1,6 @@
 //! Python pip ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,9 +12,9 @@ impl Generator for PythonRequirements {
         "requirements-locked.txt"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, manifest_path: &Path) -> Command {
         let mut command = Command::new("pip-compile");
-        command.args(["-o", self.lockfile_name()]);
+        command.arg("-o").arg(self.lockfile_name()).arg(manifest_path);
         command
     }
 }

lockfile_generator/src/yarn.rs

@@ -1,5 +1,6 @@
 //! JavaScript yarn ecosystem.
 
+use std::path::Path;
 use std::process::Command;
 
 use crate::Generator;
@@ -11,7 +12,7 @@ impl Generator for Yarn {
         "yarn.lock"
     }
 
-    fn command(&self) -> Command {
+    fn command(&self, _manifest_path: &Path) -> Command {
         let mut command = Command::new("yarn");
         command.args(["install", "--mode=skip-build", "--mode=update-lockfile"]);
         command