From 2ec7c69861ca5c5a45c0a170d857e1ce86cec5f0 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 25 Aug 2024 20:31:42 -0400 Subject: [PATCH] Respect extras and markers on virtual dev dependencies (#6620) ## Summary Closes https://github.com/astral-sh/uv/issues/6617. --- crates/uv-resolver/src/lock.rs | 52 +++++++- crates/uv-workspace/src/workspace.rs | 9 +- crates/uv/tests/lock.rs | 181 ++++++++++++++++++++++++--- crates/uv/tests/sync.rs | 20 +-- 4 files changed, 232 insertions(+), 30 deletions(-) diff --git a/crates/uv-resolver/src/lock.rs b/crates/uv-resolver/src/lock.rs index 3c78c664a0bd..5f99c4ee98a8 100644 --- a/crates/uv-resolver/src/lock.rs +++ b/crates/uv-resolver/src/lock.rs @@ -457,11 +457,20 @@ impl Lock { // dependencies in virtual workspaces). for group in dev { for dependency in project.group(group) { - let root = self - .find_by_name(dependency) - .expect("found too many packages matching root") - .expect("could not find root"); - queue.push_back((root, None)); + if dependency.marker.evaluate(marker_env, &[]) { + let root = self + .find_by_markers(&dependency.name, marker_env) + .expect("found too many packages matching root") + .expect("could not find root"); + + // Add the base package. + queue.push_back((root, None)); + + // Add any extras. + for extra in &dependency.extras { + queue.push_back((root, Some(extra))); + } + } } } @@ -674,6 +683,39 @@ impl Lock { Ok(found_dist) } + /// Returns the package with the given name. + /// + /// If there are multiple matching packages, returns the package that + /// corresponds to the given marker tree. + /// + /// If there are multiple packages that are relevant to the current + /// markers, then an error is returned. + /// + /// If there are no matching packages, then `Ok(None)` is returned. + fn find_by_markers( + &self, + name: &PackageName, + marker_env: &MarkerEnvironment, + ) -> Result, String> { + let mut found_dist = None; + for dist in &self.packages { + if &dist.id.name == name { + if dist.fork_markers.is_empty() + || dist + .fork_markers + .iter() + .any(|marker| marker.evaluate(marker_env, &[])) + { + if found_dist.is_some() { + return Err(format!("found multiple packages matching `{name}`")); + } + found_dist = Some(dist); + } + } + } + Ok(found_dist) + } + fn find_by_id(&self, id: &PackageId) -> &Package { let index = *self.by_id.get(id).expect("locked package for ID"); let dist = self.packages.get(index).expect("valid index for package"); diff --git a/crates/uv-workspace/src/workspace.rs b/crates/uv-workspace/src/workspace.rs index 63ba222e0edf..9e5c5604401b 100644 --- a/crates/uv-workspace/src/workspace.rs +++ b/crates/uv-workspace/src/workspace.rs @@ -9,7 +9,7 @@ use rustc_hash::FxHashSet; use tracing::{debug, trace, warn}; use pep508_rs::{MarkerTree, RequirementOrigin, VerbatimUrl}; -use pypi_types::{Requirement, RequirementSource}; +use pypi_types::{Requirement, RequirementSource, VerbatimParsedUrl}; use uv_fs::Simplified; use uv_normalize::{GroupName, PackageName, DEV_DEPENDENCIES}; use uv_warnings::warn_user; @@ -1309,7 +1309,10 @@ impl VirtualProject { /// Returns dependencies that apply to the workspace root, but not any of its members. As such, /// only returns a non-empty iterator for virtual workspaces, which can include dev dependencies /// on the virtual root. - pub fn group(&self, name: &GroupName) -> impl Iterator { + pub fn group( + &self, + name: &GroupName, + ) -> impl Iterator> { match self { VirtualProject::Project(_) => { // For non-virtual projects, dev dependencies are attached to the members. @@ -1326,7 +1329,7 @@ impl VirtualProject { .as_ref() .and_then(|tool| tool.uv.as_ref()) .and_then(|uv| uv.dev_dependencies.as_ref()) - .map(|dev| dev.iter().map(|req| &req.name)) + .map(|dev| dev.iter()) .into_iter() .flatten(), ) diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index 53fe9f960c49..38080386a855 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -10228,10 +10228,10 @@ fn lock_overlapping_environment() -> Result<()> { Ok(()) } -/// Lock a requirement from PyPI. +/// Lock a virtual project with forked dev dependencies. #[test] -fn lock_virtual() -> Result<()> { - let context = TestContext::new("3.12"); +fn lock_virtual_fork() -> Result<()> { + let context = TestContext::new("3.10"); let pyproject_toml = context.temp_dir.child("pyproject.toml"); pyproject_toml.write_str( @@ -10241,7 +10241,8 @@ fn lock_virtual() -> Result<()> { [tool.uv] dev-dependencies = [ - "anyio" + "anyio < 3 ; python_version >= '3.11'", + "anyio > 3 ; python_version < '3.11'", ] "#, )?; @@ -10252,7 +10253,7 @@ fn lock_virtual() -> Result<()> { ----- stdout ----- ----- stderr ----- - Resolved 3 packages in [TIME] + Resolved 6 packages in [TIME] "###); let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); @@ -10263,27 +10264,64 @@ fn lock_virtual() -> Result<()> { assert_snapshot!( lock, @r###" version = 1 - requires-python = ">=3.12" + requires-python = ">=3.10" + resolution-markers = [ + "python_full_version < '3.11'", + "python_full_version >= '3.11'", + ] [options] exclude-newer = "2024-03-25T00:00:00Z" [manifest] - requirements = [{ name = "anyio" }] + requirements = [ + { name = "anyio", marker = "python_full_version < '3.11'", specifier = ">3" }, + { name = "anyio", marker = "python_full_version >= '3.11'", specifier = "<3" }, + ] + + [[package]] + name = "anyio" + version = "2.2.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version >= '3.11'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version >= '3.11'" }, + { name = "sniffio", marker = "python_full_version >= '3.11'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/d3/e6/901a94731af20e7109415525666cb3753a2bd1edd19616c2730448dffd0d/anyio-2.2.0.tar.gz", hash = "sha256:4a41c5b3a65ed92e469d51b6fba3779301850ea2e352afcf9e36c46f21ee14a9", size = 97217 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/49/c3/b83a3c02c7d6f66932e9a72621d7f207cbfd2bd72b4c8931567ee386fb55/anyio-2.2.0-py3-none-any.whl", hash = "sha256:aa3da546ed17f097ca876c78024dea380a3b7fa80759abfdda59f12176a3dac8", size = 65320 }, + ] [[package]] name = "anyio" version = "4.3.0" source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version < '3.11'", + ] dependencies = [ - { name = "idna" }, - { name = "sniffio" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna", marker = "python_full_version < '3.11'" }, + { name = "sniffio", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6", size = 159642 } wheels = [ { url = "https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", size = 85584 }, ] + [[package]] + name = "exceptiongroup" + version = "1.2.0" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/8e/1c/beef724eaf5b01bb44b6338c8c3494eff7cab376fab4904cfbbc3585dc79/exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68", size = 26264 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/9a/5028fd52db10e600f1c4674441b968cf2ea4959085bfb5b99fb1250e5f68/exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", size = 16210 }, + ] + [[package]] name = "idna" version = "3.6" @@ -10301,6 +10339,15 @@ fn lock_virtual() -> Result<()> { wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, ] + + [[package]] + name = "typing-extensions" + version = "4.10.0" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/16/3a/0d26ce356c7465a19c9ea8814b960f8a36c3b0d07c323176620b7b483e44/typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb", size = 77558 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/de/dc04a3ea60b22624b51c703a84bbe0184abcd1d0b9bc8074b5d6b7ab90bb/typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", size = 33926 }, + ] "### ); }); @@ -10312,7 +10359,7 @@ fn lock_virtual() -> Result<()> { ----- stdout ----- ----- stderr ----- - Resolved 3 packages in [TIME] + Resolved 6 packages in [TIME] "###); // Re-run with `--offline`. We shouldn't need a network connection to validate an @@ -10323,7 +10370,7 @@ fn lock_virtual() -> Result<()> { ----- stdout ----- ----- stderr ----- - Resolved 3 packages in [TIME] + Resolved 6 packages in [TIME] "###); // Add `iniconfig`. @@ -10334,7 +10381,8 @@ fn lock_virtual() -> Result<()> { [tool.uv] dev-dependencies = [ - "anyio", + "anyio < 3 ; python_version >= '3.11'", + "anyio > 3 ; python_version < '3.11'", "iniconfig" ] "#, @@ -10346,7 +10394,7 @@ fn lock_virtual() -> Result<()> { ----- stdout ----- ----- stderr ----- - Resolved 4 packages in [TIME] + Resolved 7 packages in [TIME] Added iniconfig v2.0.0 "###); @@ -10356,12 +10404,115 @@ fn lock_virtual() -> Result<()> { ----- stdout ----- ----- stderr ----- - Prepared 4 packages in [TIME] - Installed 4 packages in [TIME] + Prepared 6 packages in [TIME] + Installed 6 packages in [TIME] + anyio==4.3.0 + + exceptiongroup==1.2.0 + idna==3.6 + iniconfig==2.0.0 + sniffio==1.3.1 + + typing-extensions==4.10.0 + "###); + + Ok(()) +} + +/// Lock a virtual project with a conditional dependency. +#[test] +fn lock_virtual_conditional() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [tool.uv.workspace] + members = [] + + [tool.uv] + dev-dependencies = [ + "anyio > 3 ; sys_platform == 'linux'", + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [manifest] + requirements = [{ name = "anyio", marker = "sys_platform == 'linux'", specifier = ">3" }] + + [[package]] + name = "anyio" + version = "4.3.0" + source = { registry = "https://pypi.org/simple" } + dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6", size = 159642 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", size = 85584 }, + ] + + [[package]] + name = "idna" + version = "3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }, + ] + + [[package]] + name = "sniffio" + version = "1.3.1" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + // Re-run with `--offline`. We shouldn't need a network connection to validate an + // already-correct lockfile with immutable metadata. + uv_snapshot!(context.filters(), context.lock().arg("--locked").arg("--offline").arg("--no-cache"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] "###); Ok(()) diff --git a/crates/uv/tests/sync.rs b/crates/uv/tests/sync.rs index 7e7ac7692505..b2c459f16acd 100644 --- a/crates/uv/tests/sync.rs +++ b/crates/uv/tests/sync.rs @@ -357,7 +357,7 @@ fn virtual_workspace_dev_dependencies() -> Result<()> { pyproject_toml.write_str( r#" [tool.uv] - dev-dependencies = ["anyio>3"] + dev-dependencies = ["anyio>3", "requests[socks]", "typing-extensions ; sys_platform == ''"] [tool.uv.workspace] members = ["child"] @@ -390,33 +390,39 @@ fn virtual_workspace_dev_dependencies() -> Result<()> { let init = src.child("__init__.py"); init.touch()?; - // Syncing with `--no-dev` should omit `anyio`. + // Syncing with `--no-dev` should omit all dependencies except `iniconfig`. uv_snapshot!(context.filters(), context.sync().arg("--no-dev"), @r###" success: true exit_code: 0 ----- stdout ----- ----- stderr ----- - Resolved 5 packages in [TIME] + Resolved 11 packages in [TIME] Prepared 2 packages in [TIME] Installed 2 packages in [TIME] + child==0.1.0 (from file://[TEMP_DIR]/child) + iniconfig==2.0.0 "###); - // Syncing without `--no-dev` should include `anyio`. + // Syncing without `--no-dev` should include `anyio`, `requests`, `pysocks`, and their + // dependencies, but not `typing-extensions`. uv_snapshot!(context.filters(), context.sync(), @r###" success: true exit_code: 0 ----- stdout ----- ----- stderr ----- - Resolved 5 packages in [TIME] - Prepared 3 packages in [TIME] - Installed 3 packages in [TIME] + Resolved 11 packages in [TIME] + Prepared 8 packages in [TIME] + Installed 8 packages in [TIME] + anyio==4.3.0 + + certifi==2024.2.2 + + charset-normalizer==3.3.2 + idna==3.6 + + pysocks==1.7.1 + + requests==2.31.0 + sniffio==1.3.1 + + urllib3==2.2.1 "###); Ok(())