From 2a9116fa23510147f9a72849dafe9576ad4d2776 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 13 Jan 2023 22:16:44 -0500 Subject: [PATCH] Add support for namespace packages --- README.md | 19 +++++++++++++++++ flake8_to_ruff/src/converter.rs | 7 +++++++ ruff.schema.json | 10 +++++++++ ruff_cli/src/commands.rs | 4 +++- src/lib_native.rs | 2 +- src/lib_wasm.rs | 5 +++-- src/packaging.rs | 37 ++++++++++++++++++++++++++------- src/settings/configuration.rs | 6 ++++++ src/settings/mod.rs | 4 ++++ src/settings/options.rs | 11 ++++++++++ src/settings/pyproject.rs | 6 ++++++ 11 files changed, 99 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8820c8adf3e9fe..08917a3e06ce9b 100644 --- a/README.md +++ b/README.md @@ -2154,6 +2154,25 @@ line-length = 120 --- +#### [`namespace-packages`](#namespace-packages) + +Mark the specified directories as namespace packages. For the purpose of +module resolution, Ruff will treat those directories as if they +contained an `__init__.py` file. + +**Default value**: `[]` + +**Type**: `Vec` + +**Example usage**: + +```toml +[tool.ruff] +namespace-packages = ["airflow/providers"] +``` + +--- + #### [`per-file-ignores`](#per-file-ignores) A list of mappings from file pattern to rule codes or prefixes to diff --git a/flake8_to_ruff/src/converter.rs b/flake8_to_ruff/src/converter.rs index 1803321d8c6b21..695cf6933f36d0 100644 --- a/flake8_to_ruff/src/converter.rs +++ b/flake8_to_ruff/src/converter.rs @@ -423,6 +423,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -488,6 +489,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: Some(100), + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -553,6 +555,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: Some(100), + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -618,6 +621,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -683,6 +687,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -756,6 +761,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -824,6 +830,7 @@ mod tests { ignore: Some(vec![]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, diff --git a/ruff.schema.json b/ruff.schema.json index 04598a3405458e..fd1eddc1ef1d13 100644 --- a/ruff.schema.json +++ b/ruff.schema.json @@ -285,6 +285,16 @@ } ] }, + "namespace-packages": { + "description": "Mark the specified directories as namespace packages. For the purpose of module resolution, Ruff will treat those directories as if they contained an `__init__.py` file.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, "pep8-naming": { "description": "Options for the `pep8-naming` plugin.", "anyOf": [ diff --git a/ruff_cli/src/commands.rs b/ruff_cli/src/commands.rs index dffd83ac8d3f3a..a1895f65e351fc 100644 --- a/ruff_cli/src/commands.rs +++ b/ruff_cli/src/commands.rs @@ -83,6 +83,8 @@ pub fn run( .flatten() .map(ignore::DirEntry::path) .collect::>(), + &resolver, + pyproject_strategy, ); let start = Instant::now(); @@ -169,7 +171,7 @@ pub fn run_stdin( }; let package_root = filename .and_then(Path::parent) - .and_then(packaging::detect_package_root); + .and_then(|path| packaging::detect_package_root(path, &settings.namespace_packages)); let stdin = read_from_stdin()?; let mut diagnostics = lint_stdin(filename, package_root, &stdin, settings, autofix)?; diagnostics.messages.sort_unstable(); diff --git a/src/lib_native.rs b/src/lib_native.rs index 5b0fd6cb3613e5..121aa6b76502c5 100644 --- a/src/lib_native.rs +++ b/src/lib_native.rs @@ -51,7 +51,7 @@ pub fn check(path: &Path, contents: &str, autofix: bool) -> Result Result { force_exclude: None, format: None, ignore_init_module_imports: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, show_source: None, src: None, - unfixable: None, - typing_modules: None, task_tags: None, + typing_modules: None, + unfixable: None, update_check: None, // Use default options for all plugins. flake8_annotations: Some(flake8_annotations::settings::Settings::default().into()), diff --git a/src/packaging.rs b/src/packaging.rs index 0cef592c4cf162..9601ad8d30977e 100644 --- a/src/packaging.rs +++ b/src/packaging.rs @@ -1,9 +1,11 @@ //! Detect Python package roots and file associations. -use std::path::Path; +use std::path::{Path, PathBuf}; use rustc_hash::FxHashMap; +use crate::resolver::{PyprojectDiscovery, Resolver}; + // If we have a Python package layout like: // - root/ // - foo/ @@ -27,15 +29,21 @@ use rustc_hash::FxHashMap; /// Return `true` if the directory at the given `Path` appears to be a Python /// package. -pub fn is_package(path: &Path) -> bool { +pub fn is_package(path: &Path, namespace_packages: &[PathBuf]) -> bool { path.join("__init__.py").is_file() + || namespace_packages + .iter() + .any(|namespace_package| namespace_package == path) } /// Return the package root for the given Python file. -pub fn detect_package_root(path: &Path) -> Option<&Path> { +pub fn detect_package_root<'a>( + path: &'a Path, + namespace_packages: &'a [PathBuf], +) -> Option<&'a Path> { let mut current = None; for parent in path.ancestors() { - if !is_package(parent) { + if !is_package(parent, namespace_packages) { return current; } current = Some(parent); @@ -46,21 +54,23 @@ pub fn detect_package_root(path: &Path) -> Option<&Path> { /// A wrapper around `is_package` to cache filesystem lookups. fn is_package_with_cache<'a>( path: &'a Path, + namespace_packages: &'a [PathBuf], package_cache: &mut FxHashMap<&'a Path, bool>, ) -> bool { *package_cache .entry(path) - .or_insert_with(|| is_package(path)) + .or_insert_with(|| is_package(path, namespace_packages)) } /// A wrapper around `detect_package_root` to cache filesystem lookups. fn detect_package_root_with_cache<'a>( path: &'a Path, + namespace_packages: &'a [PathBuf], package_cache: &mut FxHashMap<&'a Path, bool>, ) -> Option<&'a Path> { let mut current = None; for parent in path.ancestors() { - if !is_package_with_cache(parent, package_cache) { + if !is_package_with_cache(parent, namespace_packages, package_cache) { return current; } current = Some(parent); @@ -69,7 +79,11 @@ fn detect_package_root_with_cache<'a>( } /// Return a mapping from Python file to its package root. -pub fn detect_package_roots<'a>(files: &[&'a Path]) -> FxHashMap<&'a Path, Option<&'a Path>> { +pub fn detect_package_roots<'a>( + files: &[&'a Path], + resolver: &'a Resolver, + pyproject_strategy: &'a PyprojectDiscovery, +) -> FxHashMap<&'a Path, Option<&'a Path>> { // Pre-populate the module cache, since the list of files could (but isn't // required to) contain some `__init__.py` files. let mut package_cache: FxHashMap<&Path, bool> = FxHashMap::default(); @@ -84,13 +98,16 @@ pub fn detect_package_roots<'a>(files: &[&'a Path]) -> FxHashMap<&'a Path, Optio // Search for the package root for each file. let mut package_roots: FxHashMap<&Path, Option<&Path>> = FxHashMap::default(); for file in files { + let namespace_packages = &resolver + .resolve(file, pyproject_strategy) + .namespace_packages; if let Some(package) = file.parent() { if package_roots.contains_key(package) { continue; } package_roots.insert( package, - detect_package_root_with_cache(package, &mut package_cache), + detect_package_root_with_cache(package, namespace_packages, &mut package_cache), ); } } @@ -111,6 +128,7 @@ mod tests { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources/test/package/src/package") .as_path(), + &[], ), Some( PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -124,6 +142,7 @@ mod tests { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources/test/project/python_modules/core/core") .as_path(), + &[], ), Some( PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -137,6 +156,7 @@ mod tests { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources/test/project/examples/docs/docs/concepts") .as_path(), + &[], ), Some( PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -150,6 +170,7 @@ mod tests { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("setup.py") .as_path(), + &[], ), None, ); diff --git a/src/settings/configuration.rs b/src/settings/configuration.rs index 092cb47e30427b..2e24d9af10f487 100644 --- a/src/settings/configuration.rs +++ b/src/settings/configuration.rs @@ -44,6 +44,7 @@ pub struct Configuration { pub ignore: Option>, pub ignore_init_module_imports: Option, pub line_length: Option, + pub namespace_packages: Option>, pub per_file_ignores: Option>, pub required_version: Option, pub respect_gitignore: Option, @@ -135,6 +136,10 @@ impl Configuration { ignore: options.ignore, ignore_init_module_imports: options.ignore_init_module_imports, line_length: options.line_length, + namespace_packages: options + .namespace_packages + .map(|namespace_package| resolve_src(&namespace_package, project_root)) + .transpose()?, per_file_ignores: options.per_file_ignores.map(|per_file_ignores| { per_file_ignores .into_iter() @@ -211,6 +216,7 @@ impl Configuration { .ignore_init_module_imports .or(config.ignore_init_module_imports), line_length: self.line_length.or(config.line_length), + namespace_packages: self.namespace_packages.or(config.namespace_packages), per_file_ignores: self.per_file_ignores.or(config.per_file_ignores), required_version: self.required_version.or(config.required_version), respect_gitignore: self.respect_gitignore.or(config.respect_gitignore), diff --git a/src/settings/mod.rs b/src/settings/mod.rs index b645afd3bcf41d..cdadbba0c27b98 100644 --- a/src/settings/mod.rs +++ b/src/settings/mod.rs @@ -55,6 +55,7 @@ pub struct Settings { pub format: SerializationFormat, pub ignore_init_module_imports: bool, pub line_length: usize, + pub namespace_packages: Vec, pub per_file_ignores: Vec<(GlobMatcher, GlobMatcher, FxHashSet)>, pub required_version: Option, pub respect_gitignore: bool, @@ -169,6 +170,7 @@ impl Settings { force_exclude: config.force_exclude.unwrap_or(false), ignore_init_module_imports: config.ignore_init_module_imports.unwrap_or_default(), line_length: config.line_length.unwrap_or(88), + namespace_packages: config.namespace_packages.unwrap_or_default(), per_file_ignores: resolve_per_file_ignores( config.per_file_ignores.unwrap_or_default(), )?, @@ -235,6 +237,7 @@ impl Settings { format: SerializationFormat::Text, ignore_init_module_imports: false, line_length: 88, + namespace_packages: vec![], per_file_ignores: vec![], required_version: None, respect_gitignore: true, @@ -279,6 +282,7 @@ impl Settings { format: SerializationFormat::Text, ignore_init_module_imports: false, line_length: 88, + namespace_packages: vec![], per_file_ignores: vec![], required_version: None, respect_gitignore: true, diff --git a/src/settings/options.rs b/src/settings/options.rs index 6fed0ee7eb9a4b..c6bb5c30c4b606 100644 --- a/src/settings/options.rs +++ b/src/settings/options.rs @@ -352,6 +352,17 @@ pub struct Options { /// packages in that directory. User home directory and environment /// variables will also be expanded. pub src: Option>, + #[option( + default = r#"[]"#, + value_type = "Vec", + example = r#" + namespace-packages = ["airflow/providers"] + "# + )] + /// Mark the specified directories as namespace packages. For the purpose of + /// module resolution, Ruff will treat those directories as if they + /// contained an `__init__.py` file. + pub namespace_packages: Option>, #[option( default = r#""py310""#, value_type = "PythonVersion", diff --git a/src/settings/pyproject.rs b/src/settings/pyproject.rs index 200d01c13fe545..786a3e917260a8 100644 --- a/src/settings/pyproject.rs +++ b/src/settings/pyproject.rs @@ -181,6 +181,7 @@ mod tests { ignore: None, ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -239,6 +240,7 @@ line-length = 79 ignore: None, ignore_init_module_imports: None, line_length: Some(79), + namespace_packages: None, per_file_ignores: None, respect_gitignore: None, required_version: None, @@ -299,6 +301,7 @@ exclude = ["foo.py"] ignore: None, ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -358,6 +361,7 @@ select = ["E501"] ignore: None, ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -418,6 +422,7 @@ ignore = ["E501"] ignore: Some(vec![RuleCodePrefix::E501]), ignore_init_module_imports: None, line_length: None, + namespace_packages: None, per_file_ignores: None, required_version: None, respect_gitignore: None, @@ -515,6 +520,7 @@ other-attribute = 1 fixable: None, format: None, force_exclude: None, + namespace_packages: None, unfixable: None, typing_modules: None, task_tags: None,