Introduce git source distribution generator

Changelog.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add support for configuring macOS deployment target version in `pyproject.toml` in [#1536](https://github.com/PyO3/maturin/pull/1536)
 * Rewrite platform specific dependencies in `Cargo.toml` by viccie30 in [#1572](https://github.com/PyO3/maturin/pull/1572)
 * Add trusted publisher support in [#1578](https://github.com/PyO3/maturin/pull/1578)
+* Add new `git` source distribution generator in [#1587](https://github.com/PyO3/maturin/pull/1587)
 
 ## [0.14.17] - 2023-04-06
 

guide/src/config.md

@@ -54,6 +54,9 @@ python-source = "src"
 python-packages = ["foo", "bar"]
 # Strip the library for minimum file size
 strip = true
+# Source distribution generator,
+# supports cargo (default) and git.
+sdist-generator = "cargo"
 ```
 
 The `[tool.maturin.include]` and `[tool.maturin.exclude]` configuration are

src/pyproject_toml.rs

@@ -104,6 +104,17 @@ pub struct TargetConfig {
     pub macos_deployment_target: Option<String>,
 }
 
+/// Source distribution generator
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
+#[serde(rename_all = "kebab-case")]
+pub enum SdistGenerator {
+    /// Use `cargo package --list`
+    #[default]
+    Cargo,
+    /// Use `git ls-files`
+    Git,
+}
+
 /// The `[tool.maturin]` section of a pyproject.toml
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "kebab-case")]
@@ -120,6 +131,8 @@ pub struct ToolMaturin {
     skip_auditwheel: bool,
     #[serde(default)]
     strip: bool,
+    #[serde(default)]
+    sdist_generator: SdistGenerator,
     /// The directory with python module, contains `<module_name>/__init__.py`
     python_source: Option<PathBuf>,
     /// Python packages to include
@@ -239,6 +252,13 @@ impl PyProjectToml {
             .unwrap_or_default()
     }
 
+    /// Returns the value of `[tool.maturin.sdist-generator]` in pyproject.toml
+    pub fn sdist_generator(&self) -> SdistGenerator {
+        self.maturin()
+            .map(|maturin| maturin.sdist_generator)
+            .unwrap_or_default()
+    }
+
     /// Returns the value of `[tool.maturin.python-source]` in pyproject.toml
     pub fn python_source(&self) -> Option<&Path> {
         self.maturin()

src/source_distribution.rs

@@ -1,4 +1,5 @@
 use crate::module_writer::{add_data, ModuleWriter};
+use crate::pyproject_toml::SdistGenerator;
 use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter};
 use anyhow::{bail, Context, Result};
 use cargo_metadata::{Metadata, MetadataCommand};
@@ -564,18 +565,47 @@ fn find_path_deps(cargo_metadata: &Metadata) -> Result<HashMap<String, PathBuf>>
     Ok(path_deps)
 }
 
-/// Creates a source distribution, packing the root crate and all local dependencies
+/// Copies the files of git to a source distribution
 ///
-/// The source distribution format is specified in
-/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist)
-/// and in
-/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format
-pub fn source_distribution(
+/// Runs `git ls-files -z` to obtain a list of files to package.
+fn add_git_tracked_files_to_sdist(
+    writer: &mut SDistWriter,
+    prefix: impl AsRef<Path>,
+) -> Result<()> {
+    let output = Command::new("git")
+        .args(["ls-files", "-z"])
+        .output()
+        .context("Failed to run `git ls-files -z`")?;
+    if !output.status.success() {
+        bail!(
+            "Failed to query file list from git: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
+            output.status,
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        );
+    }
+
+    let prefix = prefix.as_ref();
+    writer.add_directory(prefix)?;
+
+    let file_paths = str::from_utf8(&output.stdout)
+        .context("git printed invalid utf-8 ಠ_ಠ")?
+        .split('\0')
+        .filter(|s| !s.is_empty())
+        .map(Path::new);
+    for source in file_paths {
+        writer.add_file(prefix.join(source), source)?;
+    }
+    Ok(())
+}
+
+/// Copies the files of a crate to a source distribution, recursively adding path dependencies
+/// and rewriting path entries in Cargo.toml
+fn add_cargo_package_files_to_sdist(
     build_context: &BuildContext,
-    pyproject: &PyProjectToml,
-    excludes: Option<Override>,
-) -> Result<PathBuf> {
-    let metadata21 = &build_context.metadata21;
+    writer: &mut SDistWriter,
+    root_dir: &Path,
+) -> Result<()> {
     let manifest_path = &build_context.manifest_path;
     let pyproject_toml_path = build_context
         .pyproject_toml_path
@@ -596,13 +626,6 @@ pub fn source_distribution(
 
     let known_path_deps = find_path_deps(&build_context.cargo_metadata)?;
 
-    let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?;
-    let root_dir = PathBuf::from(format!(
-        "{}-{}",
-        &metadata21.get_distribution_escaped(),
-        &metadata21.get_version_escaped()
-    ));
-
     // Add local path dependencies
     let mut path_dep_workspace_manifests = HashMap::new();
     for (name, path_dep) in known_path_deps.iter() {
@@ -635,7 +658,7 @@ pub fn source_distribution(
                 &path_dep_workspace_manifests[&path_dep_metadata.workspace_root]
             };
         add_crate_to_source_distribution(
-            &mut writer,
+            writer,
             &pyproject_toml_path,
             path_dep,
             path_dep_workspace_manifest,
@@ -652,11 +675,11 @@ pub fn source_distribution(
 
     // Add the main crate
     add_crate_to_source_distribution(
-        &mut writer,
+        writer,
         &pyproject_toml_path,
         manifest_path,
         &workspace_manifest,
-        &root_dir,
+        root_dir,
         &known_path_deps,
         true,
     )?;
@@ -733,6 +756,46 @@ pub fn source_distribution(
         }
     }
 
+    Ok(())
+}
+
+/// Creates a source distribution, packing the root crate and all local dependencies
+///
+/// The source distribution format is specified in
+/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist)
+/// and in
+/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format
+pub fn source_distribution(
+    build_context: &BuildContext,
+    pyproject: &PyProjectToml,
+    excludes: Option<Override>,
+) -> Result<PathBuf> {
+    let metadata21 = &build_context.metadata21;
+    let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?;
+    let root_dir = PathBuf::from(format!(
+        "{}-{}",
+        &metadata21.get_distribution_escaped(),
+        &metadata21.get_version_escaped()
+    ));
+
+    match pyproject.sdist_generator() {
+        SdistGenerator::Cargo => {
+            add_cargo_package_files_to_sdist(build_context, &mut writer, &root_dir)?
+        }
+        SdistGenerator::Git => add_git_tracked_files_to_sdist(&mut writer, &root_dir)?,
+    }
+
+    let pyproject_toml_path = build_context
+        .pyproject_toml_path
+        .normalize()
+        .with_context(|| {
+            format!(
+                "failed to normalize path `{}`",
+                build_context.pyproject_toml_path.display()
+            )
+        })?
+        .into_path_buf();
+    let pyproject_dir = pyproject_toml_path.parent().unwrap();
     // Add readme, license
     if let Some(project) = pyproject.project.as_ref() {
         if let Some(pyproject_toml::ReadMe::RelativePath(readme)) = project.readme.as_ref() {