Merge #1255

1255: Add `[tool.maturin.include]` and `[tool.maturin.exclude]` r=messense a=mbrobbel Based on the discussion in #1253, this is another attempt to allow users to override ignore files in Python modules by adding `[tool.maturin.include]` and `[tool.maturin.exclude]` similar to [Poetry](https://python-poetry.org/docs/pyproject/#include-and-exclude). This deprecates `[tool.maturin.sdist-include]`. After some feedback on this approach, I'll add: - [x] Documentation updates - [x] Integration tests Closes #1253 Co-authored-by: Matthijs Brobbel <[email protected]> Co-authored-by: messense <[email protected]>
PyO3 · Nov 9, 2022 · ee86fd8 · ee86fd8
2 parents 21ea2a4 + 66e3421
commit ee86fd8
Show file tree

Hide file tree

Showing 25 changed files with 920 additions and 36 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   Fixes the compatibility tags for Pyston.
 * Set default macOS deployment target version if `MACOSX_DEPLOYMENT_TARGET` isn't specified in [#1251](https://github.com/PyO3/maturin/pull/1251)
 * Add support for 32-bit x86 FreeBSD target in [#1254](https://github.com/PyO3/maturin/pull/1254)
+* Add `[tool.maturin.include]` and `[tool.maturin.exclude]` and deprecate `[tool.maturin.sdist-include]` [#1255](https://github.com/PyO3/maturin/pull/1255)
 
 ## [0.13.7] - 2022-10-29
 

diff --git a/README.md b/README.md
@@ -223,11 +223,11 @@ compatibility = "linux"
 
 `manylinux` option is also accepted as an alias of `compatibility` for backward compatibility with old version of maturin.
 
-To include arbitrary files in the sdist for use during compilation specify `sdist-include` as an array of globs:
+To include arbitrary files in the sdist for use during compilation specify `include` as an array of `path` globs with `format` set to `sdist`:
 
 ```toml
 [tool.maturin]
-sdist-include = ["path/**/*"]
+include = [{ path = "path/**/*", format = "sdist" }]
 ```
 
 There's a `maturin sdist` command for only building a source distribution as workaround for [pypa/pip#6041](https://github.com/pypa/pip/issues/6041).

diff --git a/guide/src/distribution.md b/guide/src/distribution.md
@@ -32,11 +32,11 @@ compatibility = "linux"
 
 `manylinux` option is also accepted as an alias of `compatibility` for backward compatibility with old version of maturin.
 
-To include arbitrary files in the sdist for use during compilation specify `sdist-include` as an array of globs:
+To include arbitrary files in the sdist for use during compilation specify `include` as an array of `path` globs with `format` set to `sdist`:
 
 ```toml
 [tool.maturin]
-sdist-include = ["path/**/*"]
+include = [{ path = "path/**/*", format = "sdist" }]
 ```
 
 ## Build Wheels

diff --git a/guide/src/metadata.md b/guide/src/metadata.md
@@ -108,7 +108,12 @@ in the `tool.maturin` section of `pyproject.toml`.
 ```toml
 [tool.maturin]
 # Include arbitrary files in the sdist
+# NOTE: deprecated, please use `include` with `format="sdist"`
 sdist-include = []
+# Include additional files
+include = []
+# Exclude files
+exclude = []
 # Bindings type
 bindings = "pyo3"
 # Control the platform tag on linux
@@ -138,3 +143,26 @@ unstable-flags = []
 # Extra arguments that will be passed to rustc as `cargo rustc [...] -- [...] [arg1] [arg2]`
 rustc-args = []
 ```
+
+The `[tool.maturin.include]` and `[tool.maturin.exclude]` configuration are
+inspired by
+[Poetry](https://python-poetry.org/docs/pyproject/#include-and-exclude).
+
+To specify files or globs directly:
+
+```toml
+include = ["path/**/*", "some/other/file"]
+```
+
+To specify a specific target format (`sdist` or `wheel`):
+
+```toml
+include = [
+  { path = "path/**/*", format = "sdist" },
+  { path = "all", format = ["sdist", "wheel"] },
+  { path = "for/wheel/**/*", format = "wheel" }
+]
+```
+
+The default behavior is apply these configurations to both `sdist` and `wheel`
+targets.
diff --git a/src/build_context.rs b/src/build_context.rs
@@ -10,12 +10,15 @@ use crate::project_layout::ProjectLayout;
 use crate::python_interpreter::InterpreterKind;
 use crate::source_distribution::source_distribution;
 use crate::{
-    compile, BuildArtifact, Metadata21, ModuleWriter, PyProjectToml, PythonInterpreter, Target,
+    compile, pyproject_toml::Format, BuildArtifact, Metadata21, ModuleWriter, PyProjectToml,
+    PythonInterpreter, Target,
 };
 use anyhow::{anyhow, bail, Context, Result};
 use cargo_metadata::Metadata;
 use fs_err as fs;
+use ignore::overrides::{Override, OverrideBuilder};
 use lddtree::Library;
+use normpath::PathExt;
 use regex::Regex;
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
@@ -249,8 +252,9 @@ impl BuildContext {
 
         match self.pyproject_toml.as_ref() {
             Some(pyproject) => {
-                let sdist_path = source_distribution(self, pyproject)
-                    .context("Failed to build source distribution")?;
+                let sdist_path =
+                    source_distribution(self, pyproject, self.excludes(Format::Sdist)?)
+                        .context("Failed to build source distribution")?;
                 Ok(Some((sdist_path, "source".to_string())))
             }
             None => Ok(None),
@@ -450,6 +454,23 @@ impl BuildContext {
         Ok(())
     }
 
+    fn excludes(&self, format: Format) -> Result<Option<Override>> {
+        if let Some(pyproject) = self.pyproject_toml.as_ref() {
+            let pyproject_dir = self.pyproject_toml_path.normalize()?.into_path_buf();
+            if let Some(glob_patterns) = &pyproject.exclude() {
+                let mut excludes = OverrideBuilder::new(pyproject_dir.parent().unwrap());
+                for glob in glob_patterns
+                    .iter()
+                    .filter_map(|glob_pattern| glob_pattern.targets(format))
+                {
+                    excludes.add(glob)?;
+                }
+                return Ok(Some(excludes.build()?));
+            }
+        }
+        Ok(None)
+    }
+
     fn write_binding_wheel_abi3(
         &self,
         artifact: BuildArtifact,
@@ -463,7 +484,13 @@ impl BuildContext {
             .get_platform_tag(platform_tags, self.universal2)?;
         let tag = format!("cp{}{}-abi3-{}", major, min_minor, platform);
 
-        let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?;
+        let mut writer = WheelWriter::new(
+            &tag,
+            &self.out,
+            &self.metadata21,
+            &[tag.clone()],
+            self.excludes(Format::Wheel)?,
+        )?;
         self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?;
 
         write_bindings_module(
@@ -474,6 +501,7 @@ impl BuildContext {
             None,
             &self.target,
             self.editable,
+            self.pyproject_toml.as_ref(),
         )
         .context("Failed to add the files to the wheel")?;
 
@@ -534,7 +562,13 @@ impl BuildContext {
     ) -> Result<BuiltWheelMetadata> {
         let tag = python_interpreter.get_tag(&self.target, platform_tags, self.universal2)?;
 
-        let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &[tag.clone()])?;
+        let mut writer = WheelWriter::new(
+            &tag,
+            &self.out,
+            &self.metadata21,
+            &[tag.clone()],
+            self.excludes(Format::Wheel)?,
+        )?;
         self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?;
 
         write_bindings_module(
@@ -545,6 +579,7 @@ impl BuildContext {
             Some(python_interpreter),
             &self.target,
             self.editable,
+            self.pyproject_toml.as_ref(),
         )
         .context("Failed to add the files to the wheel")?;
 
@@ -651,7 +686,13 @@ impl BuildContext {
             .target
             .get_universal_tags(platform_tags, self.universal2)?;
 
-        let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?;
+        let mut writer = WheelWriter::new(
+            &tag,
+            &self.out,
+            &self.metadata21,
+            &tags,
+            self.excludes(Format::Wheel)?,
+        )?;
         self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?;
 
         write_cffi_module(
@@ -663,6 +704,7 @@ impl BuildContext {
             &artifact.path,
             &self.interpreter[0].executable,
             self.editable,
+            self.pyproject_toml.as_ref(),
         )?;
 
         self.add_pth(&mut writer)?;
@@ -754,7 +796,13 @@ impl BuildContext {
             self.metadata21.clone()
         };
 
-        let mut writer = WheelWriter::new(&tag, &self.out, &metadata21, &tags)?;
+        let mut writer = WheelWriter::new(
+            &tag,
+            &self.out,
+            &metadata21,
+            &tags,
+            self.excludes(Format::Wheel)?,
+        )?;
 
         if let Some(python_module) = &self.project_layout.python_module {
             if self.target.is_wasi() {
@@ -763,7 +811,7 @@ impl BuildContext {
                 bail!("Sorry, adding python code to a wasm binary is currently not supported")
             }
             if !self.editable {
-                write_python_part(&mut writer, python_module)
+                write_python_part(&mut writer, python_module, self.pyproject_toml.as_ref())
                     .context("Failed to add the python module to the package")?;
             }
         }

diff --git a/src/lib.rs b/src/lib.rs
@@ -54,7 +54,7 @@ mod module_writer;
 mod new_project;
 mod polyfill;
 mod project_layout;
-mod pyproject_toml;
+pub mod pyproject_toml;
 mod python_interpreter;
 mod source_distribution;
 mod target;