From 6fa4e32ad318afbec4c6997c69731bb4c256ef82 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 9 Jul 2024 10:21:52 +0100 Subject: [PATCH] [red-knot] Use vendored typeshed stubs for stdlib module resolution (#12224) --- crates/red_knot_module_resolver/src/db.rs | 119 +++-- crates/red_knot_module_resolver/src/path.rs | 466 +++++++++++++----- .../red_knot_module_resolver/src/resolver.rs | 47 +- crates/red_knot_module_resolver/src/state.rs | 5 + .../red_knot_module_resolver/src/typeshed.rs | 103 +--- .../src/typeshed/vendored.rs | 99 ++++ .../src/typeshed/versions.rs | 24 +- crates/ruff_db/src/files/path.rs | 8 + crates/ruff_db/src/vendored/path.rs | 54 ++ 9 files changed, 622 insertions(+), 303 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/typeshed/vendored.rs diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index d7a97e150664e..05771856f543f 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -25,8 +25,9 @@ pub(crate) mod tests { use salsa::DebugWithDb; use ruff_db::files::Files; - use ruff_db::system::TestSystem; - use ruff_db::system::{DbWithTestSystem, SystemPathBuf}; + use ruff_db::system::{ + DbWithTestSystem, MemoryFileSystem, SystemPath, SystemPathBuf, TestSystem, + }; use ruff_db::vendored::VendoredFileSystem; use crate::resolver::{set_module_resolution_settings, RawModuleResolutionSettings}; @@ -130,9 +131,9 @@ pub(crate) mod tests { pub(crate) struct TestCaseBuilder { db: TestDb, src: SystemPathBuf, - custom_typeshed: SystemPathBuf, site_packages: SystemPathBuf, target_version: Option, + with_vendored_stubs: bool, } impl TestCaseBuilder { @@ -142,31 +143,88 @@ pub(crate) mod tests { self } - pub(crate) fn build(self) -> TestCase { + #[must_use] + pub(crate) fn with_vendored_stubs_used(mut self) -> Self { + self.with_vendored_stubs = true; + self + } + + fn create_mocked_typeshed( + typeshed_dir: &SystemPath, + fs: &MemoryFileSystem, + ) -> std::io::Result<()> { + static VERSIONS_DATA: &str = "\ + asyncio: 3.8- # 'Regular' package on py38+ + asyncio.tasks: 3.9-3.11 + collections: 3.9- # 'Regular' package on py39+ + functools: 3.8- + importlib: 3.9- # Namespace package on py39+ + xml: 3.8-3.8 # Namespace package on py38 only + "; + + fs.create_directory_all(typeshed_dir)?; + fs.write_file(typeshed_dir.join("stdlib/VERSIONS"), VERSIONS_DATA)?; + + // Regular package on py38+ + fs.create_directory_all(typeshed_dir.join("stdlib/asyncio"))?; + fs.touch(typeshed_dir.join("stdlib/asyncio/__init__.pyi"))?; + fs.write_file( + typeshed_dir.join("stdlib/asyncio/tasks.pyi"), + "class Task: ...", + )?; + + // Regular package on py39+ + fs.create_directory_all(typeshed_dir.join("stdlib/collections"))?; + fs.touch(typeshed_dir.join("stdlib/collections/__init__.pyi"))?; + + // Namespace package on py38 only + fs.create_directory_all(typeshed_dir.join("stdlib/xml"))?; + fs.touch(typeshed_dir.join("stdlib/xml/etree.pyi"))?; + + // Namespace package on py39+ + fs.create_directory_all(typeshed_dir.join("stdlib/importlib"))?; + fs.touch(typeshed_dir.join("stdlib/importlib/abc.pyi"))?; + + fs.write_file( + typeshed_dir.join("stdlib/functools.pyi"), + "def update_wrapper(): ...", + ) + } + + pub(crate) fn build(self) -> std::io::Result { let TestCaseBuilder { mut db, src, - custom_typeshed, + with_vendored_stubs, site_packages, target_version, } = self; + let typeshed_dir = SystemPathBuf::from("/typeshed"); + + let custom_typeshed = if with_vendored_stubs { + None + } else { + Self::create_mocked_typeshed(&typeshed_dir, db.memory_file_system())?; + Some(typeshed_dir.clone()) + }; + let settings = RawModuleResolutionSettings { target_version: target_version.unwrap_or_default(), extra_paths: vec![], workspace_root: src.clone(), - custom_typeshed: Some(custom_typeshed.clone()), + custom_typeshed: custom_typeshed.clone(), site_packages: Some(site_packages.clone()), }; set_module_resolution_settings(&mut db, settings); - TestCase { + Ok(TestCase { db, - src, - custom_typeshed, - site_packages, - } + src: src.clone(), + custom_typeshed: typeshed_dir, + site_packages: site_packages.clone(), + }) } } @@ -178,57 +236,20 @@ pub(crate) mod tests { } pub(crate) fn create_resolver_builder() -> std::io::Result { - static VERSIONS_DATA: &str = "\ - asyncio: 3.8- # 'Regular' package on py38+ - asyncio.tasks: 3.9-3.11 - collections: 3.9- # 'Regular' package on py39+ - functools: 3.8- - importlib: 3.9- # Namespace package on py39+ - xml: 3.8-3.8 # Namespace package on py38 only - "; - let db = TestDb::new(); let src = SystemPathBuf::from("/src"); let site_packages = SystemPathBuf::from("/site_packages"); - let custom_typeshed = SystemPathBuf::from("/typeshed"); let fs = db.memory_file_system(); fs.create_directory_all(&src)?; fs.create_directory_all(&site_packages)?; - fs.create_directory_all(&custom_typeshed)?; - fs.write_file(custom_typeshed.join("stdlib/VERSIONS"), VERSIONS_DATA)?; - - // Regular package on py38+ - fs.create_directory_all(custom_typeshed.join("stdlib/asyncio"))?; - fs.touch(custom_typeshed.join("stdlib/asyncio/__init__.pyi"))?; - fs.write_file( - custom_typeshed.join("stdlib/asyncio/tasks.pyi"), - "class Task: ...", - )?; - - // Regular package on py39+ - fs.create_directory_all(custom_typeshed.join("stdlib/collections"))?; - fs.touch(custom_typeshed.join("stdlib/collections/__init__.pyi"))?; - - // Namespace package on py38 only - fs.create_directory_all(custom_typeshed.join("stdlib/xml"))?; - fs.touch(custom_typeshed.join("stdlib/xml/etree.pyi"))?; - - // Namespace package on py39+ - fs.create_directory_all(custom_typeshed.join("stdlib/importlib"))?; - fs.touch(custom_typeshed.join("stdlib/importlib/abc.pyi"))?; - - fs.write_file( - custom_typeshed.join("stdlib/functools.pyi"), - "def update_wrapper(): ...", - )?; Ok(TestCaseBuilder { db, src, - custom_typeshed, + with_vendored_stubs: false, site_packages, target_version: None, }) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 173697577812c..e0958dad3f041 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -5,13 +5,61 @@ use std::fmt; -use ruff_db::files::{system_path_to_file, File}; +use ruff_db::files::{system_path_to_file, vendored_path_to_file, File, FilePath}; use ruff_db::system::{SystemPath, SystemPathBuf}; +use ruff_db::vendored::{VendoredPath, VendoredPathBuf}; +use crate::db::Db; use crate::module_name::ModuleName; use crate::state::ResolverState; use crate::typeshed::TypeshedVersionsQueryResult; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum FilePathRef<'a> { + System(&'a SystemPath), + Vendored(&'a VendoredPath), +} + +impl<'a> FilePathRef<'a> { + fn parent(&self) -> Option { + match self { + Self::System(path) => path.parent().map(Self::System), + Self::Vendored(path) => path.parent().map(Self::Vendored), + } + } + + fn components(&self) -> camino::Utf8Components { + match self { + Self::System(path) => path.components(), + Self::Vendored(path) => path.components(), + } + } + + fn file_stem(&self) -> Option<&str> { + match self { + Self::System(path) => path.file_stem(), + Self::Vendored(path) => path.file_stem(), + } + } + + #[inline] + fn to_file(self, db: &dyn Db) -> Option { + match self { + Self::System(path) => system_path_to_file(db.upcast(), path), + Self::Vendored(path) => vendored_path_to_file(db.upcast(), path), + } + } +} + +impl<'a> From<&'a FilePath> for FilePathRef<'a> { + fn from(value: &'a FilePath) -> Self { + match value { + FilePath::System(path) => FilePathRef::System(path), + FilePath::Vendored(path) => FilePathRef::Vendored(path), + } + } +} + /// Enumeration of the different kinds of search paths type checkers are expected to support. /// /// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the @@ -23,14 +71,14 @@ use crate::typeshed::TypeshedVersionsQueryResult; enum ModuleResolutionPathBufInner { Extra(SystemPathBuf), FirstParty(SystemPathBuf), - StandardLibrary(SystemPathBuf), + StandardLibrary(FilePath), SitePackages(SystemPathBuf), } impl ModuleResolutionPathBufInner { fn push(&mut self, component: &str) { let extension = camino::Utf8Path::new(component).extension(); - let inner = match self { + match self { Self::Extra(ref mut path) => { if let Some(extension) = extension { assert!( @@ -38,7 +86,11 @@ impl ModuleResolutionPathBufInner { "Extension must be `py` or `pyi`; got `{extension}`" ); } - path + assert!( + path.extension().is_none(), + "Cannot push part {component} to {path}, which already has an extension" + ); + path.push(component); } Self::FirstParty(ref mut path) => { if let Some(extension) = extension { @@ -47,7 +99,11 @@ impl ModuleResolutionPathBufInner { "Extension must be `py` or `pyi`; got `{extension}`" ); } - path + assert!( + path.extension().is_none(), + "Cannot push part {component} to {path}, which already has an extension" + ); + path.push(component); } Self::StandardLibrary(ref mut path) => { if let Some(extension) = extension { @@ -56,7 +112,14 @@ impl ModuleResolutionPathBufInner { "Extension must be `pyi`; got `{extension}`" ); } - path + assert!( + path.extension().is_none(), + "Cannot push part {component} to {path:?}, which already has an extension" + ); + match path { + FilePath::System(path) => path.push(component), + FilePath::Vendored(path) => path.push(component), + } } Self::SitePackages(ref mut path) => { if let Some(extension) = extension { @@ -65,14 +128,13 @@ impl ModuleResolutionPathBufInner { "Extension must be `py` or `pyi`; got `{extension}`" ); } - path + assert!( + path.extension().is_none(), + "Cannot push part {component} to {path}, which already has an extension" + ); + path.push(component); } - }; - assert!( - inner.extension().is_none(), - "Cannot push part {component} to {inner}, which already has an extension" - ); - inner.push(component); + } } } @@ -107,16 +169,24 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn standard_library(path: impl Into) -> Option { - let path = path.into(); + pub(crate) fn standard_library(path: FilePath) -> Option { path.extension() .map_or(true, |ext| ext == "pyi") .then_some(Self(ModuleResolutionPathBufInner::StandardLibrary(path))) } #[must_use] - pub(crate) fn stdlib_from_typeshed_root(typeshed_root: &SystemPath) -> Option { - Self::standard_library(typeshed_root.join(SystemPath::new("stdlib"))) + pub(crate) fn stdlib_from_custom_typeshed_root(typeshed_root: &SystemPath) -> Option { + Self::standard_library(FilePath::System( + typeshed_root.join(SystemPath::new("stdlib")), + )) + } + + #[must_use] + pub(crate) fn vendored_stdlib() -> Self { + Self(ModuleResolutionPathBufInner::StandardLibrary( + FilePath::Vendored(VendoredPathBuf::from("stdlib")), + )) } #[must_use] @@ -150,9 +220,9 @@ impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn relativize_path<'a>( &'a self, - absolute_path: &'a (impl AsRef + ?Sized), + absolute_path: &'a FilePath, ) -> Option> { - ModuleResolutionPathRef::from(self).relativize_path(absolute_path.as_ref()) + ModuleResolutionPathRef::from(self).relativize_path(&FilePathRef::from(absolute_path)) } /// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory. @@ -163,15 +233,24 @@ impl ModuleResolutionPathBuf { impl fmt::Debug for ModuleResolutionPathBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (name, path) = match &self.0 { - ModuleResolutionPathBufInner::Extra(path) => ("Extra", path), - ModuleResolutionPathBufInner::FirstParty(path) => ("FirstParty", path), - ModuleResolutionPathBufInner::SitePackages(path) => ("SitePackages", path), - ModuleResolutionPathBufInner::StandardLibrary(path) => ("StandardLibrary", path), - }; - f.debug_tuple(&format!("ModuleResolutionPathBuf::{name}")) - .field(path) - .finish() + match &self.0 { + ModuleResolutionPathBufInner::Extra(path) => f + .debug_tuple("ModuleResolutionPathBuf::Extra") + .field(path) + .finish(), + ModuleResolutionPathBufInner::FirstParty(path) => f + .debug_tuple("ModuleResolutionPathBuf::FirstParty") + .field(path) + .finish(), + ModuleResolutionPathBufInner::SitePackages(path) => f + .debug_tuple("ModuleResolutionPathBuf::SitePackages") + .field(path) + .finish(), + ModuleResolutionPathBufInner::StandardLibrary(path) => f + .debug_tuple("ModuleResolutionPathBuf::StandardLibrary") + .field(path) + .finish(), + } } } @@ -179,16 +258,16 @@ impl fmt::Debug for ModuleResolutionPathBuf { enum ModuleResolutionPathRefInner<'a> { Extra(&'a SystemPath), FirstParty(&'a SystemPath), - StandardLibrary(&'a SystemPath), + StandardLibrary(FilePathRef<'a>), SitePackages(&'a SystemPath), } impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn query_stdlib_version<'db>( - module_path: &'a SystemPath, + module_path: &FilePathRef<'a>, stdlib_search_path: Self, - stdlib_root: &SystemPath, + stdlib_root: &FilePathRef<'a>, resolver_state: &ResolverState<'db>, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_search_path @@ -202,7 +281,11 @@ impl<'a> ModuleResolutionPathRefInner<'a> { typeshed_versions, target_version, } = resolver_state; - typeshed_versions.query_module(&module_name, *db, stdlib_root, *target_version) + let root_to_pass = match stdlib_root { + FilePathRef::System(root) => Some(*root), + FilePathRef::Vendored(_) => None, + }; + typeshed_versions.query_module(*db, &module_name, root_to_pass, *target_version) } #[must_use] @@ -212,10 +295,12 @@ impl<'a> ModuleResolutionPathRefInner<'a> { (Self::FirstParty(path), Self::FirstParty(_)) => resolver.system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => resolver.system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version( path, search_path, stdlib_root, resolver) { + match Self::query_stdlib_version(path, search_path, &stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, - TypeshedVersionsQueryResult::Exists => resolver.system().is_directory(path), - TypeshedVersionsQueryResult::MaybeExists => resolver.system().is_directory(path), + TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => match path { + FilePathRef::System(path) => resolver.system().is_directory(path), + FilePathRef::Vendored(path) => resolver.vendored().is_directory(path) + } } } (path, root) => unreachable!( @@ -240,10 +325,12 @@ impl<'a> ModuleResolutionPathRefInner<'a> { // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version( path, search_path, stdlib_root, resolver) { + match Self::query_stdlib_version( path, search_path, &stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, - TypeshedVersionsQueryResult::Exists => resolver.db.system().path_exists(&path.join("__init__.pyi")), - TypeshedVersionsQueryResult::MaybeExists => resolver.db.system().path_exists(&path.join("__init__.pyi")), + TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => match path { + FilePathRef::System(path) => resolver.db.system().path_exists(&path.join("__init__.pyi")), + FilePathRef::Vendored(path) => resolver.db.vendored().exists(path.join("__init__.pyi")), + }, } } (path, root) => unreachable!( @@ -260,10 +347,10 @@ impl<'a> ModuleResolutionPathRefInner<'a> { system_path_to_file(resolver.db.upcast(), path) } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, search_path, stdlib_root, resolver) { + match Self::query_stdlib_version(&path, search_path, &stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => None, - TypeshedVersionsQueryResult::Exists => system_path_to_file(resolver.db.upcast(), path), - TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(resolver.db.upcast(), path) + TypeshedVersionsQueryResult::Exists => path.to_file(resolver.db), + TypeshedVersionsQueryResult::MaybeExists => path.to_file(resolver.db), } } (path, root) => unreachable!( @@ -274,23 +361,31 @@ impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn to_module_name(self) -> Option { - let (fs_path, skip_final_part) = match self { - Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => ( - path, - path.ends_with("__init__.py") || path.ends_with("__init__.pyi"), - ), - Self::StandardLibrary(path) => (path, path.ends_with("__init__.pyi")), - }; - - let parent_components = fs_path - .parent()? - .components() - .map(|component| component.as_str()); - - if skip_final_part { - ModuleName::from_components(parent_components) - } else { - ModuleName::from_components(parent_components.chain(fs_path.file_stem())) + match self { + Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => { + let parent = path.parent()?; + let parent_components = parent.components().map(|component| component.as_str()); + let skip_final_part = + path.ends_with("__init__.py") || path.ends_with("__init__.pyi"); + if skip_final_part { + ModuleName::from_components(parent_components) + } else { + ModuleName::from_components(parent_components.chain(path.file_stem())) + } + } + Self::StandardLibrary(path) => { + let parent = path.parent()?; + let parent_components = parent.components().map(|component| component.as_str()); + let skip_final_part = match path { + FilePathRef::System(path) => path.ends_with("__init__.pyi"), + FilePathRef::Vendored(path) => path.ends_with("__init__.pyi"), + }; + if skip_final_part { + ModuleName::from_components(parent_components) + } else { + ModuleName::from_components(parent_components.chain(path.file_stem())) + } + } } } @@ -301,8 +396,15 @@ impl<'a> ModuleResolutionPathRefInner<'a> { Self::FirstParty(path) => { ModuleResolutionPathBufInner::FirstParty(path.with_extension("pyi")) } - Self::StandardLibrary(path) => { - ModuleResolutionPathBufInner::StandardLibrary(path.with_extension("pyi")) + Self::StandardLibrary(FilePathRef::System(path)) => { + ModuleResolutionPathBufInner::StandardLibrary(FilePath::System( + path.with_extension("pyi"), + )) + } + Self::StandardLibrary(FilePathRef::Vendored(path)) => { + ModuleResolutionPathBufInner::StandardLibrary(FilePath::Vendored( + path.with_pyi_extension(), + )) } Self::SitePackages(path) => { ModuleResolutionPathBufInner::SitePackages(path.with_extension("pyi")) @@ -327,28 +429,52 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn relativize_path(&self, absolute_path: &'a SystemPath) -> Option { - match self { - Self::Extra(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { - path.extension() - .map_or(true, |ext| matches!(ext, "py" | "pyi")) - .then_some(Self::Extra(path)) - }), - Self::FirstParty(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { - path.extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - .then_some(Self::FirstParty(path)) - }), - Self::StandardLibrary(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { - path.extension() - .map_or(true, |ext| ext == "pyi") - .then_some(Self::StandardLibrary(path)) - }), - Self::SitePackages(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { - path.extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - .then_some(Self::SitePackages(path)) - }), + fn relativize_path(&self, absolute_path: &FilePathRef<'a>) -> Option { + match (self, absolute_path) { + (Self::Extra(root), FilePathRef::System(absolute_path)) => { + absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "py" | "pyi")) + .then_some(Self::Extra(path)) + }) + } + (Self::FirstParty(root), FilePathRef::System(absolute_path)) => { + absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self::FirstParty(path)) + }) + } + (Self::StandardLibrary(root), FilePathRef::System(absolute_path)) => match root { + FilePathRef::System(root) => { + absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| ext == "pyi") + .then_some(Self::StandardLibrary(FilePathRef::System(path))) + }) + } + FilePathRef::Vendored(_) => None, + }, + (Self::SitePackages(root), FilePathRef::System(absolute_path)) => { + absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self::SitePackages(path)) + }) + } + (Self::Extra(_), FilePathRef::Vendored(_)) => None, + (Self::FirstParty(_), FilePathRef::Vendored(_)) => None, + (Self::StandardLibrary(root), FilePathRef::Vendored(absolute_path)) => match root { + FilePathRef::System(_) => None, + FilePathRef::Vendored(root) => { + absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| ext == "pyi") + .then_some(Self::StandardLibrary(FilePathRef::Vendored(path))) + }) + } + }, + (Self::SitePackages(_), FilePathRef::Vendored(_)) => None, } } } @@ -400,22 +526,31 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn relativize_path(&self, absolute_path: &'a SystemPath) -> Option { + fn relativize_path(&self, absolute_path: &FilePathRef<'a>) -> Option { self.0.relativize_path(absolute_path).map(Self) } } impl fmt::Debug for ModuleResolutionPathRef<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (name, path) = match &self.0 { - ModuleResolutionPathRefInner::Extra(path) => ("Extra", path), - ModuleResolutionPathRefInner::FirstParty(path) => ("FirstParty", path), - ModuleResolutionPathRefInner::SitePackages(path) => ("SitePackages", path), - ModuleResolutionPathRefInner::StandardLibrary(path) => ("StandardLibrary", path), - }; - f.debug_tuple(&format!("ModuleResolutionPathRef::{name}")) - .field(path) - .finish() + match &self.0 { + ModuleResolutionPathRefInner::Extra(path) => f + .debug_tuple("ModuleResolutionPathRef::Extra") + .field(path) + .finish(), + ModuleResolutionPathRefInner::FirstParty(path) => f + .debug_tuple("ModuleResolutionPathRef::FirstParty") + .field(path) + .finish(), + ModuleResolutionPathRefInner::SitePackages(path) => f + .debug_tuple("ModuleResolutionPathRef::SitePackages") + .field(path) + .finish(), + ModuleResolutionPathRefInner::StandardLibrary(path) => f + .debug_tuple("ModuleResolutionPathRef::StandardLibrary") + .field(path) + .finish(), + } } } @@ -426,8 +561,11 @@ impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { ModuleResolutionPathBufInner::FirstParty(path) => { ModuleResolutionPathRefInner::FirstParty(path) } - ModuleResolutionPathBufInner::StandardLibrary(path) => { - ModuleResolutionPathRefInner::StandardLibrary(path) + ModuleResolutionPathBufInner::StandardLibrary(FilePath::System(path)) => { + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::System(path)) + } + ModuleResolutionPathBufInner::StandardLibrary(FilePath::Vendored(path)) => { + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::Vendored(path)) } ModuleResolutionPathBufInner::SitePackages(path) => { ModuleResolutionPathRefInner::SitePackages(path) @@ -439,13 +577,15 @@ impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { impl PartialEq for ModuleResolutionPathRef<'_> { fn eq(&self, other: &SystemPath) -> bool { - let fs_path = match self.0 { - ModuleResolutionPathRefInner::Extra(path) => path, - ModuleResolutionPathRefInner::FirstParty(path) => path, - ModuleResolutionPathRefInner::SitePackages(path) => path, - ModuleResolutionPathRefInner::StandardLibrary(path) => path, - }; - fs_path == other + match self.0 { + ModuleResolutionPathRefInner::Extra(path) => path == other, + ModuleResolutionPathRefInner::FirstParty(path) => path == other, + ModuleResolutionPathRefInner::SitePackages(path) => path == other, + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::System(path)) => { + path == other + } + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::Vendored(_)) => false, + } } } @@ -467,6 +607,38 @@ impl PartialEq> for SystemPathBuf { } } +impl PartialEq for ModuleResolutionPathRef<'_> { + fn eq(&self, other: &VendoredPath) -> bool { + match self.0 { + ModuleResolutionPathRefInner::Extra(_) => false, + ModuleResolutionPathRefInner::FirstParty(_) => false, + ModuleResolutionPathRefInner::SitePackages(_) => false, + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::System(_)) => false, + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::Vendored(path)) => { + path == other + } + } + } +} + +impl PartialEq> for VendoredPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { + other == self + } +} + +impl PartialEq for ModuleResolutionPathRef<'_> { + fn eq(&self, other: &VendoredPathBuf) -> bool { + self == &**other + } +} + +impl PartialEq> for VendoredPathBuf { + fn eq(&self, other: &ModuleResolutionPathRef<'_>) -> bool { + &**self == other + } +} + #[cfg(test)] mod tests { use insta::assert_debug_snapshot; @@ -477,6 +649,12 @@ mod tests { use super::*; + impl<'a> FilePathRef<'a> { + fn system(path: &'a (impl AsRef + ?Sized)) -> Self { + Self::System(path.as_ref()) + } + } + impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn join(&self, component: &str) -> Self { @@ -504,8 +682,15 @@ mod tests { ModuleResolutionPathRefInner::FirstParty(path) => { ModuleResolutionPathBufInner::FirstParty(path.to_path_buf()) } - ModuleResolutionPathRefInner::StandardLibrary(path) => { - ModuleResolutionPathBufInner::StandardLibrary(path.to_path_buf()) + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::System(path)) => { + ModuleResolutionPathBufInner::StandardLibrary(FilePath::System( + path.to_path_buf(), + )) + } + ModuleResolutionPathRefInner::StandardLibrary(FilePathRef::Vendored(path)) => { + ModuleResolutionPathBufInner::StandardLibrary(FilePath::Vendored( + path.to_path_buf(), + )) } ModuleResolutionPathRefInner::SitePackages(path) => { ModuleResolutionPathBufInner::SitePackages(path.to_path_buf()) @@ -522,9 +707,12 @@ mod tests { #[test] fn constructor_rejects_non_pyi_stdlib_paths() { - assert_eq!(ModuleResolutionPathBuf::standard_library("foo.py"), None); assert_eq!( - ModuleResolutionPathBuf::standard_library("foo/__init__.py"), + ModuleResolutionPathBuf::standard_library(FilePath::system("foo.py")), + None + ); + assert_eq!( + ModuleResolutionPathBuf::standard_library(FilePath::system("foo/__init__.py")), None ); } @@ -532,10 +720,12 @@ mod tests { #[test] fn path_buf_debug_impl() { assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo/bar.pyi").unwrap(), + ModuleResolutionPathBuf::standard_library(FilePath::system("foo/bar.pyi")).unwrap(), @r###" ModuleResolutionPathBuf::StandardLibrary( - "foo/bar.pyi", + System( + "foo/bar.pyi", + ), ) "### ); @@ -556,18 +746,18 @@ mod tests { #[test] fn with_extension_methods() { assert_eq!( - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .with_py_extension(), None ); assert_eq!( - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .with_pyi_extension(), ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( - SystemPathBuf::from("foo.pyi") + FilePath::System(SystemPathBuf::from("foo.pyi")) )) ); @@ -592,7 +782,7 @@ mod tests { assert_eq!( ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( - SystemPath::new("foo.pyi") + FilePathRef::system("foo.pyi") )) .to_module_name(), ModuleName::new_static("foo") @@ -611,7 +801,7 @@ mod tests { fn module_name_2_parts() { assert_eq!( ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( - SystemPath::new("foo/bar") + FilePathRef::system("foo/bar") )) .to_module_name(), ModuleName::new_static("foo.bar") @@ -656,19 +846,19 @@ mod tests { #[test] fn join() { assert_eq!( - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .join("bar"), ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( - SystemPathBuf::from("foo/bar") + FilePath::system("foo/bar") )) ); assert_eq!( - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .join("bar.pyi"), ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( - SystemPathBuf::from("foo/bar.pyi") + FilePath::system("foo/bar.pyi") )) ); assert_eq!( @@ -684,7 +874,7 @@ mod tests { #[test] #[should_panic(expected = "Extension must be `pyi`; got `py`")] fn stdlib_path_invalid_join_py() { - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .push("bar.py"); } @@ -692,7 +882,7 @@ mod tests { #[test] #[should_panic(expected = "Extension must be `pyi`; got `rs`")] fn stdlib_path_invalid_join_rs() { - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo")) .unwrap() .push("bar.rs"); } @@ -708,46 +898,47 @@ mod tests { #[test] #[should_panic(expected = "already has an extension")] fn invalid_stdlib_join_too_many_extensions() { - ModuleResolutionPathBuf::standard_library("foo.pyi") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo.pyi")) .unwrap() .push("bar.pyi"); } #[test] fn relativize_stdlib_path_errors() { - let root = ModuleResolutionPathBuf::standard_library("foo/stdlib").unwrap(); + let root = + ModuleResolutionPathBuf::standard_library(FilePath::system("foo/stdlib")).unwrap(); // Must have a `.pyi` extension or no extension: - let bad_absolute_path = SystemPath::new("foo/stdlib/x.py"); - assert_eq!(root.relativize_path(bad_absolute_path), None); - let second_bad_absolute_path = SystemPath::new("foo/stdlib/x.rs"); - assert_eq!(root.relativize_path(second_bad_absolute_path), None); + let bad_absolute_path = FilePath::system("foo/stdlib/x.py"); + assert_eq!(root.relativize_path(&bad_absolute_path), None); + let second_bad_absolute_path = FilePath::system("foo/stdlib/x.rs"); + assert_eq!(root.relativize_path(&second_bad_absolute_path), None); // Must be a path that is a child of `root`: - let third_bad_absolute_path = SystemPath::new("bar/stdlib/x.pyi"); - assert_eq!(root.relativize_path(third_bad_absolute_path), None); + let third_bad_absolute_path = FilePath::system("bar/stdlib/x.pyi"); + assert_eq!(root.relativize_path(&third_bad_absolute_path), None); } #[test] fn relativize_non_stdlib_path_errors() { let root = ModuleResolutionPathBuf::extra("foo/stdlib").unwrap(); // Must have a `.py` extension, a `.pyi` extension, or no extension: - let bad_absolute_path = SystemPath::new("foo/stdlib/x.rs"); - assert_eq!(root.relativize_path(bad_absolute_path), None); + let bad_absolute_path = FilePath::system("foo/stdlib/x.rs"); + assert_eq!(root.relativize_path(&bad_absolute_path), None); // Must be a path that is a child of `root`: - let second_bad_absolute_path = SystemPath::new("bar/stdlib/x.pyi"); - assert_eq!(root.relativize_path(second_bad_absolute_path), None); + let second_bad_absolute_path = FilePath::system("bar/stdlib/x.pyi"); + assert_eq!(root.relativize_path(&second_bad_absolute_path), None); } #[test] fn relativize_path() { assert_eq!( - ModuleResolutionPathBuf::standard_library("foo/baz") + ModuleResolutionPathBuf::standard_library(FilePath::system("foo/baz")) .unwrap() - .relativize_path("foo/baz/eggs/__init__.pyi") + .relativize_path(&FilePath::system("foo/baz/eggs/__init__.pyi")) .unwrap(), ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( - SystemPath::new("eggs/__init__.pyi") + FilePathRef::system("eggs/__init__.pyi") )) ); } @@ -757,9 +948,9 @@ mod tests { db, custom_typeshed, .. - } = create_resolver_builder().unwrap().build(); + } = create_resolver_builder().unwrap().build().unwrap(); let stdlib_module_path = - ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + ModuleResolutionPathBuf::stdlib_from_custom_typeshed_root(&custom_typeshed).unwrap(); (db, stdlib_module_path) } @@ -893,9 +1084,10 @@ mod tests { } = create_resolver_builder() .unwrap() .with_target_version(TargetVersion::Py39) - .build(); + .build() + .unwrap(); let stdlib_module_path = - ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + ModuleResolutionPathBuf::stdlib_from_custom_typeshed_root(&custom_typeshed).unwrap(); (db, stdlib_module_path) } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index deff02e1d4163..047e51c3cf061 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -78,9 +78,7 @@ pub(crate) fn path_to_module(db: &dyn Db, path: &FilePath) -> Option { pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option { let _span = tracing::trace_span!("file_to_module", ?file).entered(); - let FilePath::System(path) = file.path(db.upcast()) else { - todo!("VendoredPaths are not yet supported") - }; + let path = file.path(db.upcast()); let resolver_settings = module_resolver_settings(db); @@ -161,11 +159,11 @@ impl RawModuleResolutionSettings { paths.push(ModuleResolutionPathBuf::first_party(workspace_root).unwrap()); - if let Some(custom_typeshed) = custom_typeshed { - paths.push( - ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(), - ); - } + paths.push( + custom_typeshed.map_or_else(ModuleResolutionPathBuf::vendored_stdlib, |custom| { + ModuleResolutionPathBuf::stdlib_from_custom_typeshed_root(&custom).unwrap() + }), + ); // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step if let Some(site_packages) = site_packages { @@ -388,6 +386,8 @@ impl PackageKind { mod tests { use ruff_db::files::{system_path_to_file, File, FilePath}; use ruff_db::system::DbWithTestSystem; + use ruff_db::vendored::{VendoredPath, VendoredPathBuf}; + use ruff_db::Upcast; use crate::db::tests::{create_resolver_builder, TestCase}; use crate::module::ModuleKind; @@ -396,7 +396,7 @@ mod tests { use super::*; fn setup_resolver_test() -> TestCase { - create_resolver_builder().unwrap().build() + create_resolver_builder().unwrap().build().unwrap() } #[test] @@ -436,7 +436,7 @@ mod tests { } = setup_resolver_test(); let stdlib_dir = - ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + ModuleResolutionPathBuf::stdlib_from_custom_typeshed_root(&custom_typeshed).unwrap(); let functools_module_name = ModuleName::new_static("functools").unwrap(); let functools_module = resolve_module(&db, functools_module_name.clone()).unwrap(); @@ -518,7 +518,8 @@ mod tests { } = create_resolver_builder() .unwrap() .with_target_version(TargetVersion::Py39) - .build(); + .build() + .unwrap(); let existing_modules = create_module_names(&[ "asyncio", @@ -548,7 +549,8 @@ mod tests { let TestCase { db, .. } = create_resolver_builder() .unwrap() .with_target_version(TargetVersion::Py39) - .build(); + .build() + .unwrap(); let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]); for module_name in nonexisting_modules { @@ -588,6 +590,27 @@ mod tests { Ok(()) } + #[test] + fn stdlib_uses_vendored_typeshed_when_no_custom_typeshed_supplied() { + let TestCase { db, .. } = create_resolver_builder() + .unwrap() + .with_vendored_stubs_used() + .build() + .unwrap(); + + let pydoc_data_topics_name = ModuleName::new_static("pydoc_data.topics").unwrap(); + let pydoc_data_topics = resolve_module(&db, pydoc_data_topics_name).unwrap(); + assert_eq!("pydoc_data.topics", pydoc_data_topics.name()); + assert_eq!( + pydoc_data_topics.search_path(), + VendoredPathBuf::from("stdlib") + ); + assert_eq!( + &pydoc_data_topics.file().path(db.upcast()), + &VendoredPath::new("stdlib/pydoc_data/topics.pyi") + ); + } + #[test] fn resolve_package() -> anyhow::Result<()> { let TestCase { src, mut db, .. } = setup_resolver_test(); diff --git a/crates/red_knot_module_resolver/src/state.rs b/crates/red_knot_module_resolver/src/state.rs index 0a0763840dcf4..42fb1f46111f2 100644 --- a/crates/red_knot_module_resolver/src/state.rs +++ b/crates/red_knot_module_resolver/src/state.rs @@ -1,4 +1,5 @@ use ruff_db::system::System; +use ruff_db::vendored::VendoredFileSystem; use crate::db::Db; use crate::supported_py_version::TargetVersion; @@ -22,4 +23,8 @@ impl<'db> ResolverState<'db> { pub(crate) fn system(&self) -> &dyn System { self.db.system() } + + pub(crate) fn vendored(&self) -> &VendoredFileSystem { + self.db.vendored() + } } diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index f73e870268b0f..08d269a111e78 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,107 +1,8 @@ -use once_cell::sync::Lazy; - -use ruff_db::vendored::VendoredFileSystem; - +pub use self::vendored::vendored_typeshed_stubs; pub(crate) use self::versions::{ parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsQueryResult, }; pub use self::versions::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; +mod vendored; mod versions; - -// The file path here is hardcoded in this crate's `build.rs` script. -// Luckily this crate will fail to build if this file isn't available at build time. -static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); - -pub fn vendored_typeshed_stubs() -> &'static VendoredFileSystem { - static VENDORED_TYPESHED_STUBS: Lazy = - Lazy::new(|| VendoredFileSystem::new_static(TYPESHED_ZIP_BYTES).unwrap()); - &VENDORED_TYPESHED_STUBS -} - -#[cfg(test)] -mod tests { - use std::io::{self, Read}; - use std::path::Path; - - use ruff_db::vendored::VendoredPath; - - use crate::typeshed::TYPESHED_ZIP_BYTES; - use crate::vendored_typeshed_stubs; - - #[test] - fn typeshed_zip_created_at_build_time() { - let mut typeshed_zip_archive = - zip::ZipArchive::new(io::Cursor::new(TYPESHED_ZIP_BYTES)).unwrap(); - - let mut functools_module_stub = typeshed_zip_archive - .by_name("stdlib/functools.pyi") - .unwrap(); - assert!(functools_module_stub.is_file()); - - let mut functools_module_stub_source = String::new(); - functools_module_stub - .read_to_string(&mut functools_module_stub_source) - .unwrap(); - - assert!(functools_module_stub_source.contains("def update_wrapper(")); - } - - #[test] - fn typeshed_vfs_consistent_with_vendored_stubs() { - let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap(); - let vendored_typeshed_stubs = vendored_typeshed_stubs(); - - let mut empty_iterator = true; - for entry in walkdir::WalkDir::new(&vendored_typeshed_dir).min_depth(1) { - empty_iterator = false; - let entry = entry.unwrap(); - let absolute_path = entry.path(); - let file_type = entry.file_type(); - - let relative_path = absolute_path - .strip_prefix(&vendored_typeshed_dir) - .unwrap_or_else(|_| { - panic!("Expected {absolute_path:?} to be a child of {vendored_typeshed_dir:?}") - }); - - let vendored_path = <&VendoredPath>::try_from(relative_path) - .unwrap_or_else(|_| panic!("Expected {relative_path:?} to be valid UTF-8")); - - assert!( - vendored_typeshed_stubs.exists(vendored_path), - "Expected {vendored_path:?} to exist in the `VendoredFileSystem`! - - Vendored file system: - - {vendored_typeshed_stubs:#?} - " - ); - - let vendored_path_kind = vendored_typeshed_stubs - .metadata(vendored_path) - .unwrap_or_else(|_| { - panic!( - "Expected metadata for {vendored_path:?} to be retrievable from the `VendoredFileSystem! - - Vendored file system: - - {vendored_typeshed_stubs:#?} - " - ) - }) - .kind(); - - assert_eq!( - vendored_path_kind.is_directory(), - file_type.is_dir(), - "{vendored_path:?} had type {vendored_path_kind:?}, inconsistent with fs path {relative_path:?}: {file_type:?}" - ); - } - - assert!( - !empty_iterator, - "Expected there to be at least one file or directory in the vendored typeshed stubs!" - ); - } -} diff --git a/crates/red_knot_module_resolver/src/typeshed/vendored.rs b/crates/red_knot_module_resolver/src/typeshed/vendored.rs new file mode 100644 index 0000000000000..e28eadbc3f9c5 --- /dev/null +++ b/crates/red_knot_module_resolver/src/typeshed/vendored.rs @@ -0,0 +1,99 @@ +use once_cell::sync::Lazy; + +use ruff_db::vendored::VendoredFileSystem; + +// The file path here is hardcoded in this crate's `build.rs` script. +// Luckily this crate will fail to build if this file isn't available at build time. +static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); + +pub fn vendored_typeshed_stubs() -> &'static VendoredFileSystem { + static VENDORED_TYPESHED_STUBS: Lazy = + Lazy::new(|| VendoredFileSystem::new_static(TYPESHED_ZIP_BYTES).unwrap()); + &VENDORED_TYPESHED_STUBS +} + +#[cfg(test)] +mod tests { + use std::io::{self, Read}; + use std::path::Path; + + use ruff_db::vendored::VendoredPath; + + use super::*; + + #[test] + fn typeshed_zip_created_at_build_time() { + let mut typeshed_zip_archive = + zip::ZipArchive::new(io::Cursor::new(TYPESHED_ZIP_BYTES)).unwrap(); + + let mut functools_module_stub = typeshed_zip_archive + .by_name("stdlib/functools.pyi") + .unwrap(); + assert!(functools_module_stub.is_file()); + + let mut functools_module_stub_source = String::new(); + functools_module_stub + .read_to_string(&mut functools_module_stub_source) + .unwrap(); + + assert!(functools_module_stub_source.contains("def update_wrapper(")); + } + + #[test] + fn typeshed_vfs_consistent_with_vendored_stubs() { + let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap(); + let vendored_typeshed_stubs = vendored_typeshed_stubs(); + + let mut empty_iterator = true; + for entry in walkdir::WalkDir::new(&vendored_typeshed_dir).min_depth(1) { + empty_iterator = false; + let entry = entry.unwrap(); + let absolute_path = entry.path(); + let file_type = entry.file_type(); + + let relative_path = absolute_path + .strip_prefix(&vendored_typeshed_dir) + .unwrap_or_else(|_| { + panic!("Expected {absolute_path:?} to be a child of {vendored_typeshed_dir:?}") + }); + + let vendored_path = <&VendoredPath>::try_from(relative_path) + .unwrap_or_else(|_| panic!("Expected {relative_path:?} to be valid UTF-8")); + + assert!( + vendored_typeshed_stubs.exists(vendored_path), + "Expected {vendored_path:?} to exist in the `VendoredFileSystem`! + + Vendored file system: + + {vendored_typeshed_stubs:#?} + " + ); + + let vendored_path_kind = vendored_typeshed_stubs + .metadata(vendored_path) + .unwrap_or_else(|_| { + panic!( + "Expected metadata for {vendored_path:?} to be retrievable from the `VendoredFileSystem! + + Vendored file system: + + {vendored_typeshed_stubs:#?} + " + ) + }) + .kind(); + + assert_eq!( + vendored_path_kind.is_directory(), + file_type.is_dir(), + "{vendored_path:?} had type {vendored_path_kind:?}, inconsistent with fs path {relative_path:?}: {file_type:?}" + ); + } + + assert!( + !empty_iterator, + "Expected there to be at least one file or directory in the vendored typeshed stubs!" + ); + } +} diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 4600ddf0bd069..c4d2a9189f216 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -5,16 +5,19 @@ use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; +use once_cell::sync::Lazy; +use ruff_db::system::SystemPath; use rustc_hash::FxHashMap; use ruff_db::files::{system_path_to_file, File}; use ruff_db::source::source_text; -use ruff_db::system::SystemPath; use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::TargetVersion; +use super::vendored::vendored_typeshed_stubs; + #[derive(Debug)] pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>); @@ -39,13 +42,17 @@ impl<'db> LazyTypeshedVersions<'db> { #[must_use] pub(crate) fn query_module( &self, - module: &ModuleName, db: &'db dyn Db, - stdlib_root: &SystemPath, + module: &ModuleName, + stdlib_root: Option<&SystemPath>, target_version: TargetVersion, ) -> TypeshedVersionsQueryResult { let versions = self.0.get_or_init(|| { - let versions_path = stdlib_root.join("VERSIONS"); + let versions_path = if let Some(system_path) = stdlib_root { + system_path.join("VERSIONS") + } else { + return &VENDORED_VERSIONS; + }; let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { todo!( "Still need to figure out how to handle VERSIONS files being deleted \ @@ -71,6 +78,15 @@ pub(crate) fn parse_typeshed_versions( file_content.parse() } +static VENDORED_VERSIONS: Lazy = Lazy::new(|| { + TypeshedVersions::from_str( + &vendored_typeshed_stubs() + .read_to_string("stdlib/VERSIONS") + .unwrap(), + ) + .unwrap() +}); + #[derive(Debug, PartialEq, Eq, Clone)] pub struct TypeshedVersionsParseError { line_number: Option, diff --git a/crates/ruff_db/src/files/path.rs b/crates/ruff_db/src/files/path.rs index 8def5dec869d1..b474e3fb6a833 100644 --- a/crates/ruff_db/src/files/path.rs +++ b/crates/ruff_db/src/files/path.rs @@ -86,6 +86,14 @@ impl FilePath { FilePath::Vendored(path) => vendored_path_to_file(db, path), } } + + #[must_use] + pub fn extension(&self) -> Option<&str> { + match self { + FilePath::System(path) => path.extension(), + FilePath::Vendored(path) => path.extension(), + } + } } impl AsRef for FilePath { diff --git a/crates/ruff_db/src/vendored/path.rs b/crates/ruff_db/src/vendored/path.rs index 194d3e8ff88a2..a4f37c1d5f025 100644 --- a/crates/ruff_db/src/vendored/path.rs +++ b/crates/ruff_db/src/vendored/path.rs @@ -30,6 +30,43 @@ impl VendoredPath { pub fn components(&self) -> Utf8Components { self.0.components() } + + #[must_use] + pub fn extension(&self) -> Option<&str> { + self.0.extension() + } + + #[must_use] + pub fn with_pyi_extension(&self) -> VendoredPathBuf { + VendoredPathBuf(self.0.with_extension("pyi")) + } + + #[must_use] + pub fn join(&self, other: impl AsRef) -> VendoredPathBuf { + VendoredPathBuf(self.0.join(other.as_ref())) + } + + #[must_use] + pub fn ends_with(&self, suffix: impl AsRef) -> bool { + self.0.ends_with(suffix.as_ref()) + } + + #[must_use] + pub fn parent(&self) -> Option<&Self> { + self.0.parent().map(Self::new) + } + + #[must_use] + pub fn file_stem(&self) -> Option<&str> { + self.0.file_stem() + } + + pub fn strip_prefix( + &self, + prefix: impl AsRef, + ) -> Result<&Self, path::StripPrefixError> { + self.0.strip_prefix(prefix.as_ref()).map(Self::new) + } } #[repr(transparent)] @@ -50,6 +87,10 @@ impl VendoredPathBuf { pub fn as_path(&self) -> &VendoredPath { VendoredPath::new(&self.0) } + + pub fn push(&mut self, component: impl AsRef) { + self.0.push(component.as_ref()) + } } impl AsRef for VendoredPathBuf { @@ -86,6 +127,13 @@ impl AsRef for VendoredPath { } } +impl AsRef for VendoredPath { + #[inline] + fn as_ref(&self) -> &Utf8Path { + &self.0 + } +} + impl Deref for VendoredPathBuf { type Target = VendoredPath; @@ -94,6 +142,12 @@ impl Deref for VendoredPathBuf { } } +impl From<&str> for VendoredPathBuf { + fn from(value: &str) -> Self { + Self(Utf8PathBuf::from(value)) + } +} + impl<'a> TryFrom<&'a path::Path> for &'a VendoredPath { type Error = camino::FromPathError;