Merge #1587

1587: Introduce `git` source distribution generator r=konstin a=messense TODO: - [x] How to handle projects that live inside a subdirectory of a git repository? For example our `test-crates/*` - [x] Add a test case Closes #1586 Co-authored-by: messense <[email protected]>
PyO3 · May 4, 2023 · f626a46 · f626a46
2 parents b1fd845 + 3db4431
commit f626a46
Show file tree

Hide file tree

Showing 9 changed files with 239 additions and 52 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -28,7 +28,7 @@ freebsd_task:
       - echo $CIRRUS_OS
       - cat Cargo.lock
   install_script:
-    - pkg install -y bash python
+    - pkg install -y bash git python
     - python3 -m ensurepip
   <<: *BUILD_AND_TEST
 

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -287,12 +287,12 @@ jobs:
       CARGO_TERM_COLOR: always
     container: alpine:latest
     steps:
-      - uses: actions/checkout@v3
       - name: Install build requirements
         run: |
           set -ex
-          apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd
+          apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd git
           pip3 install cffi virtualenv
+      - uses: actions/checkout@v3
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
         with:
@@ -485,6 +485,8 @@ jobs:
     container: pyston/pyston:2.3.5
     steps:
       - uses: actions/checkout@v3
+      - name: Fix git permissions
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
       - uses: dtolnay/rust-toolchain@stable
         id: rustup
         with:

diff --git a/Changelog.md b/Changelog.md
@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add support for configuring macOS deployment target version in `pyproject.toml` in [#1536](https://github.com/PyO3/maturin/pull/1536)
 * Rewrite platform specific dependencies in `Cargo.toml` by viccie30 in [#1572](https://github.com/PyO3/maturin/pull/1572)
 * Add trusted publisher support in [#1578](https://github.com/PyO3/maturin/pull/1578)
+* Add new `git` source distribution generator in [#1587](https://github.com/PyO3/maturin/pull/1587)
 
 ## [0.14.17] - 2023-04-06
 

diff --git a/guide/src/config.md b/guide/src/config.md
@@ -54,6 +54,9 @@ python-source = "src"
 python-packages = ["foo", "bar"]
 # Strip the library for minimum file size
 strip = true
+# Source distribution generator,
+# supports cargo (default) and git.
+sdist-generator = "cargo"
 ```
 
 The `[tool.maturin.include]` and `[tool.maturin.exclude]` configuration are

diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs
@@ -9,10 +9,11 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 /// The `[tool]` section of a pyproject.toml
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
 #[serde(rename_all = "kebab-case")]
 pub struct Tool {
-    maturin: Option<ToolMaturin>,
+    /// maturin options
+    pub maturin: Option<ToolMaturin>,
 }
 
 #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
@@ -104,28 +105,48 @@ pub struct TargetConfig {
     pub macos_deployment_target: Option<String>,
 }
 
+/// Source distribution generator
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
+#[serde(rename_all = "kebab-case")]
+pub enum SdistGenerator {
+    /// Use `cargo package --list`
+    #[default]
+    Cargo,
+    /// Use `git ls-files`
+    Git,
+}
+
 /// The `[tool.maturin]` section of a pyproject.toml
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
 #[serde(rename_all = "kebab-case")]
 pub struct ToolMaturin {
     // maturin specific options
-    // extension module name, accepts setuptools style import name like `foo.bar`
-    module_name: Option<String>,
-    include: Option<Vec<GlobPattern>>,
-    exclude: Option<Vec<GlobPattern>>,
-    bindings: Option<String>,
+    /// Module name, accepts setuptools style import name like `foo.bar`
+    pub module_name: Option<String>,
+    /// Include files matching the given glob pattern(s)
+    pub include: Option<Vec<GlobPattern>>,
+    /// Exclude files matching the given glob pattern(s)
+    pub exclude: Option<Vec<GlobPattern>>,
+    /// Bindings type
+    pub bindings: Option<String>,
+    /// Platform compatibility
     #[serde(alias = "manylinux")]
-    compatibility: Option<PlatformTag>,
+    pub compatibility: Option<PlatformTag>,
+    /// Skip audit wheel
+    #[serde(default)]
+    pub skip_auditwheel: bool,
+    /// Strip the final binary
     #[serde(default)]
-    skip_auditwheel: bool,
+    pub strip: bool,
+    /// Source distribution generator
     #[serde(default)]
-    strip: bool,
+    pub sdist_generator: SdistGenerator,
     /// The directory with python module, contains `<module_name>/__init__.py`
-    python_source: Option<PathBuf>,
+    pub python_source: Option<PathBuf>,
     /// Python packages to include
-    python_packages: Option<Vec<String>>,
+    pub python_packages: Option<Vec<String>>,
     /// Path to the wheel directory, defaults to `<module_name>.data`
-    data: Option<PathBuf>,
+    pub data: Option<PathBuf>,
     /// Cargo compile targets
     pub targets: Option<Vec<CargoTarget>>,
     /// Target configuration
@@ -239,6 +260,13 @@ impl PyProjectToml {
             .unwrap_or_default()
     }
 
+    /// Returns the value of `[tool.maturin.sdist-generator]` in pyproject.toml
+    pub fn sdist_generator(&self) -> SdistGenerator {
+        self.maturin()
+            .map(|maturin| maturin.sdist_generator)
+            .unwrap_or_default()
+    }
+
     /// Returns the value of `[tool.maturin.python-source]` in pyproject.toml
     pub fn python_source(&self) -> Option<&Path> {
         self.maturin()

diff --git a/src/source_distribution.rs b/src/source_distribution.rs
@@ -1,4 +1,5 @@
 use crate::module_writer::{add_data, ModuleWriter};
+use crate::pyproject_toml::SdistGenerator;
 use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter};
 use anyhow::{bail, Context, Result};
 use cargo_metadata::{Metadata, MetadataCommand};
@@ -564,29 +565,53 @@ fn find_path_deps(cargo_metadata: &Metadata) -> Result<HashMap<String, PathBuf>>
     Ok(path_deps)
 }
 
-/// Creates a source distribution, packing the root crate and all local dependencies
+/// Copies the files of git to a source distribution
 ///
-/// The source distribution format is specified in
-/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist)
-/// and in
-/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format
-pub fn source_distribution(
+/// Runs `git ls-files -z` to obtain a list of files to package.
+fn add_git_tracked_files_to_sdist(
+    pyproject_toml_path: &Path,
+    writer: &mut SDistWriter,
+    prefix: impl AsRef<Path>,
+) -> Result<()> {
+    let pyproject_dir = pyproject_toml_path.parent().unwrap();
+    let output = Command::new("git")
+        .args(["ls-files", "-z"])
+        .current_dir(pyproject_dir)
+        .output()
+        .context("Failed to run `git ls-files -z`")?;
+    if !output.status.success() {
+        bail!(
+            "Failed to query file list from git: {}\n--- Project Path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
+            output.status,
+            pyproject_dir.display(),
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        );
+    }
+
+    let prefix = prefix.as_ref();
+    writer.add_directory(prefix)?;
+
+    let file_paths = str::from_utf8(&output.stdout)
+        .context("git printed invalid utf-8 ಠ_ಠ")?
+        .split('\0')
+        .filter(|s| !s.is_empty())
+        .map(Path::new);
+    for source in file_paths {
+        writer.add_file(prefix.join(source), pyproject_dir.join(source))?;
+    }
+    Ok(())
+}
+
+/// Copies the files of a crate to a source distribution, recursively adding path dependencies
+/// and rewriting path entries in Cargo.toml
+fn add_cargo_package_files_to_sdist(
     build_context: &BuildContext,
-    pyproject: &PyProjectToml,
-    excludes: Option<Override>,
-) -> Result<PathBuf> {
-    let metadata21 = &build_context.metadata21;
+    pyproject_toml_path: &Path,
+    writer: &mut SDistWriter,
+    root_dir: &Path,
+) -> Result<()> {
     let manifest_path = &build_context.manifest_path;
-    let pyproject_toml_path = build_context
-        .pyproject_toml_path
-        .normalize()
-        .with_context(|| {
-            format!(
-                "failed to normalize path `{}`",
-                build_context.pyproject_toml_path.display()
-            )
-        })?
-        .into_path_buf();
     let workspace_manifest_path = build_context
         .cargo_metadata
         .workspace_root
@@ -596,13 +621,6 @@ pub fn source_distribution(
 
     let known_path_deps = find_path_deps(&build_context.cargo_metadata)?;
 
-    let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?;
-    let root_dir = PathBuf::from(format!(
-        "{}-{}",
-        &metadata21.get_distribution_escaped(),
-        &metadata21.get_version_escaped()
-    ));
-
     // Add local path dependencies
     let mut path_dep_workspace_manifests = HashMap::new();
     for (name, path_dep) in known_path_deps.iter() {
@@ -635,8 +653,8 @@ pub fn source_distribution(
                 &path_dep_workspace_manifests[&path_dep_metadata.workspace_root]
             };
         add_crate_to_source_distribution(
-            &mut writer,
-            &pyproject_toml_path,
+            writer,
+            pyproject_toml_path,
             path_dep,
             path_dep_workspace_manifest,
             &root_dir.join(LOCAL_DEPENDENCIES_FOLDER).join(name),
@@ -652,11 +670,11 @@ pub fn source_distribution(
 
     // Add the main crate
     add_crate_to_source_distribution(
-        &mut writer,
-        &pyproject_toml_path,
+        writer,
+        pyproject_toml_path,
         manifest_path,
         &workspace_manifest,
-        &root_dir,
+        root_dir,
         &known_path_deps,
         true,
     )?;
@@ -733,6 +751,61 @@ pub fn source_distribution(
         }
     }
 
+    Ok(())
+}
+
+/// Creates a source distribution, packing the root crate and all local dependencies
+///
+/// The source distribution format is specified in
+/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist)
+/// and in
+/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format
+pub fn source_distribution(
+    build_context: &BuildContext,
+    pyproject: &PyProjectToml,
+    excludes: Option<Override>,
+) -> Result<PathBuf> {
+    let pyproject_toml_path = build_context
+        .pyproject_toml_path
+        .normalize()
+        .with_context(|| {
+            format!(
+                "failed to normalize path `{}`",
+                build_context.pyproject_toml_path.display()
+            )
+        })?
+        .into_path_buf();
+    let metadata21 = &build_context.metadata21;
+    let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?;
+    let root_dir = PathBuf::from(format!(
+        "{}-{}",
+        &metadata21.get_distribution_escaped(),
+        &metadata21.get_version_escaped()
+    ));
+
+    match pyproject.sdist_generator() {
+        SdistGenerator::Cargo => add_cargo_package_files_to_sdist(
+            build_context,
+            &pyproject_toml_path,
+            &mut writer,
+            &root_dir,
+        )?,
+        SdistGenerator::Git => {
+            add_git_tracked_files_to_sdist(&pyproject_toml_path, &mut writer, &root_dir)?
+        }
+    }
+
+    let pyproject_toml_path = build_context
+        .pyproject_toml_path
+        .normalize()
+        .with_context(|| {
+            format!(
+                "failed to normalize path `{}`",
+                build_context.pyproject_toml_path.display()
+            )
+        })?
+        .into_path_buf();
+    let pyproject_dir = pyproject_toml_path.parent().unwrap();
     // Add readme, license
     if let Some(project) = pyproject.project.as_ref() {
         if let Some(pyproject_toml::ReadMe::RelativePath(readme)) = project.readme.as_ref() {

diff --git a/...space_with_path_dep/python/pyproject.toml → ...es/workspace_with_path_dep/pyproject.toml b/...space_with_path_dep/python/pyproject.toml → ...es/workspace_with_path_dep/pyproject.toml
@@ -1,3 +1,6 @@
 [build-system]
 requires = ["maturin>=0.14,<0.15"]
 build-backend = "maturin"
+
+[tool.maturin]
+manifest-path = "python/Cargo.toml"
diff --git a/tests/common/other.rs b/tests/common/other.rs
@@ -1,6 +1,7 @@
 use anyhow::{Context, Result};
 use clap::Parser;
 use flate2::read::GzDecoder;
+use maturin::pyproject_toml::{SdistGenerator, ToolMaturin};
 use maturin::{BuildOptions, CargoOptions, PlatformTag};
 use pretty_assertions::assert_eq;
 use std::collections::BTreeSet;
@@ -115,6 +116,7 @@ pub fn test_workspace_cargo_lock() -> Result<()> {
 
 pub fn test_source_distribution(
     package: impl AsRef<Path>,
+    sdist_generator: SdistGenerator,
     expected_files: Vec<&str>,
     expected_cargo_toml: Option<(&Path, &str)>,
     unique_name: &str,
@@ -135,7 +137,22 @@ pub fn test_source_distribution(
         ..Default::default()
     };
 
-    let build_context = build_options.into_build_context(false, false, false)?;
+    let mut build_context = build_options.into_build_context(false, false, false)?;
+
+    // Override the sdist generator for testing
+    let mut pyproject_toml = build_context.pyproject_toml.take().unwrap();
+    let mut tool = pyproject_toml.tool.clone().unwrap_or_default();
+    if let Some(ref mut tool_maturin) = tool.maturin {
+        tool_maturin.sdist_generator = sdist_generator;
+    } else {
+        tool.maturin = Some(ToolMaturin {
+            sdist_generator,
+            ..Default::default()
+        });
+    }
+    pyproject_toml.tool = Some(tool);
+    build_context.pyproject_toml = Some(pyproject_toml);
+
     let (path, _) = build_context
         .build_source_distribution()?
         .context("Failed to build source distribution")?;