From 28b2e65ff65a825b76dd55f6b51eaabccf9dff1f Mon Sep 17 00:00:00 2001 From: Skyler Hawthorne Date: Sat, 23 Nov 2024 15:44:56 -0500 Subject: [PATCH 1/2] feat: export --prune This adds a `--prune` flag to the `export` command to correspond with the `--prune` flag of the `tree` command. The purpose is for generating a `requirements.txt` that omits a package and all of that package's unique dependencies. This is useful for cases where the project has a dependency on a common core package, but where that package does not need to be installed in the target environment. For example, a pyspark job needs spark for development, but when installing into a cluster that already has pyspark installed, it is desirable to omit pyspark's whole dependency tree so that only the unique dependencies that your job needs get installed, and do not risk breaking the pyspark dependencies with something incompatible. Dev groups cannot always cover this case because there are other projects where this common dependency occurs as a transitive. One example is Airflow providers, which include Airflow itself as a dependency, but it is unnecessary and undesirable to include Airflow's dependency tree in the `requirements.txt` for your DAGs. --- crates/uv-cli/src/lib.rs | 4 ++ .../uv-resolver/src/lock/requirements_txt.rs | 15 ++++- crates/uv/src/commands/project/export.rs | 2 + crates/uv/src/lib.rs | 1 + crates/uv/src/settings.rs | 3 + crates/uv/tests/it/export.rs | 57 +++++++++++++++++++ docs/reference/cli.md | 2 + 7 files changed, 83 insertions(+), 1 deletion(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 50c5ba28a206..cddae54f2e1c 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -3401,6 +3401,10 @@ pub struct ExportArgs { #[arg(long, conflicts_with = "all_packages")] pub package: Option, + /// Prune the given package from the dependency tree. + #[arg(long, conflicts_with = "all_packages")] + pub prune: Vec, + /// Include optional dependencies from the specified extra name. /// /// May be provided more than once. diff --git a/crates/uv-resolver/src/lock/requirements_txt.rs b/crates/uv-resolver/src/lock/requirements_txt.rs index 5c6e83fb7dd1..c0e010eee60a 100644 --- a/crates/uv-resolver/src/lock/requirements_txt.rs +++ b/crates/uv-resolver/src/lock/requirements_txt.rs @@ -14,7 +14,7 @@ use uv_configuration::{DevGroupsManifest, EditableMode, ExtrasSpecification, Ins use uv_distribution_filename::{DistExtension, SourceDistExtension}; use uv_fs::Simplified; use uv_git::GitReference; -use uv_normalize::ExtraName; +use uv_normalize::{ExtraName, PackageName}; use uv_pep508::MarkerTree; use uv_pypi_types::{ParsedArchiveUrl, ParsedGitUrl}; @@ -33,6 +33,7 @@ pub struct RequirementsTxtExport<'lock> { impl<'lock> RequirementsTxtExport<'lock> { pub fn from_lock( target: InstallTarget<'lock>, + prune: &[PackageName], extras: &ExtrasSpecification, dev: &DevGroupsManifest, editable: EditableMode, @@ -50,6 +51,10 @@ impl<'lock> RequirementsTxtExport<'lock> { // Add the workspace package to the queue. for root_name in target.packages() { + if prune.contains(root_name) { + continue; + } + let dist = target .lock() .find_by_name(root_name) @@ -96,6 +101,10 @@ impl<'lock> RequirementsTxtExport<'lock> { }) .flatten() { + if prune.contains(&dep.package_id.name) { + continue; + } + let dep_dist = target.lock().find_by_id(&dep.package_id); // Add the dependency to the graph. @@ -142,6 +151,10 @@ impl<'lock> RequirementsTxtExport<'lock> { }; for dep in deps { + if prune.contains(&dep.package_id.name) { + continue; + } + let dep_dist = target.lock().find_by_id(&dep.package_id); // Add the dependency to the graph. diff --git a/crates/uv/src/commands/project/export.rs b/crates/uv/src/commands/project/export.rs index 206a83b03332..32751bd45da9 100644 --- a/crates/uv/src/commands/project/export.rs +++ b/crates/uv/src/commands/project/export.rs @@ -33,6 +33,7 @@ pub(crate) async fn export( format: ExportFormat, all_packages: bool, package: Option, + prune: Vec, hashes: bool, install_options: InstallOptions, output_file: Option, @@ -178,6 +179,7 @@ pub(crate) async fn export( ExportFormat::RequirementsTxt => { let export = RequirementsTxtExport::from_lock( target, + &prune, &extras, &dev.with_defaults(defaults), editable, diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 8ddecaa4a30d..0950304c652d 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1572,6 +1572,7 @@ async fn run_project( args.format, args.all_packages, args.package, + args.prune, args.hashes, args.install_options, args.output_file, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index 3d29f532158c..b9d82bae209e 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -1279,6 +1279,7 @@ pub(crate) struct ExportSettings { pub(crate) format: ExportFormat, pub(crate) all_packages: bool, pub(crate) package: Option, + pub(crate) prune: Vec, pub(crate) extras: ExtrasSpecification, pub(crate) dev: DevGroupsSpecification, pub(crate) editable: EditableMode, @@ -1302,6 +1303,7 @@ impl ExportSettings { format, all_packages, package, + prune, extra, all_extras, no_all_extras, @@ -1337,6 +1339,7 @@ impl ExportSettings { format, all_packages, package, + prune, extras: ExtrasSpecification::from_args( flag(all_extras, no_all_extras).unwrap_or_default(), extra.unwrap_or_default(), diff --git a/crates/uv/tests/it/export.rs b/crates/uv/tests/it/export.rs index 200d29f6177d..0ae9a41561a1 100644 --- a/crates/uv/tests/it/export.rs +++ b/crates/uv/tests/it/export.rs @@ -254,6 +254,63 @@ fn project_extra() -> Result<()> { Ok(()) } +#[test] +fn prune() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "jupyter-client" + ] + "#, + )?; + + // project v0.1.0 + // └── jupyter-client v8.6.1 + // ├── jupyter-core v5.7.2 + // │ ├── platformdirs v4.2.0 + // │ └── traitlets v5.14.2 + // ├── python-dateutil v2.9.0.post0 + // │ └── six v1.16.0 + // ├── pyzmq v25.1.2 + // ├── tornado v6.4 + // └── traitlets v5.14.2 + + uv_snapshot!( + context.filters(), + context.export() + .arg("--no-hashes") + .arg("--prune") + .arg("jupyter-core"), + @r" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv export --cache-dir [CACHE_DIR] --no-hashes --prune jupyter-core + cffi==1.16.0 ; implementation_name == 'pypy' + jupyter-client==8.6.1 + pycparser==2.21 ; implementation_name == 'pypy' + python-dateutil==2.9.0.post0 + pyzmq==25.1.2 + six==1.16.0 + tornado==6.4 + traitlets==5.14.2 + + ----- stderr ----- + Resolved 12 packages in [TIME] + " + ); + + Ok(()) +} + #[test] fn dependency_marker() -> Result<()> { let context = TestContext::new("3.12"); diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 87c779ae0a87..02b581992102 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -2355,6 +2355,8 @@ uv export [OPTIONS]

This setting has no effect when used in the uv pip interface.

+
--prune prune

Prune the given package from the dependency tree

+
--python, -p python

The Python interpreter to use during resolution.

A Python interpreter is required for building source distributions to determine package metadata when there are not wheels.

From acf70978cb1edcee8ae6c8323d1c177a5ba1d64e Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 23 Nov 2024 20:59:19 -0500 Subject: [PATCH 2/2] Expand docs --- crates/uv-cli/src/lib.rs | 3 +++ docs/reference/cli.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index cddae54f2e1c..48255c611fac 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -3402,6 +3402,9 @@ pub struct ExportArgs { pub package: Option, /// Prune the given package from the dependency tree. + /// + /// Pruned packages will be excluded from the exported requirements file, as will any + /// dependencies that are no longer required after the pruned package is removed. #[arg(long, conflicts_with = "all_packages")] pub prune: Vec, diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 02b581992102..57e1835f11f2 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -2355,7 +2355,9 @@ uv export [OPTIONS]

This setting has no effect when used in the uv pip interface.

-
--prune prune

Prune the given package from the dependency tree

+
--prune prune

Prune the given package from the dependency tree.

+ +

Pruned packages will be excluded from the exported requirements file, as will any dependencies that are no longer required after the pruned package is removed.

--python, -p python

The Python interpreter to use during resolution.