diff --git a/.cirrus.yml b/.cirrus.yml index aa828d251..d1e9a8c7b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -28,7 +28,7 @@ freebsd_task: - echo $CIRRUS_OS - cat Cargo.lock install_script: - - pkg install -y bash python + - pkg install -y bash git python - python3 -m ensurepip <<: *BUILD_AND_TEST diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f296048ff..150d33f5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -287,12 +287,14 @@ jobs: CARGO_TERM_COLOR: always container: alpine:latest steps: - - uses: actions/checkout@v3 - name: Install build requirements run: | set -ex - apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd + apk add cargo python3-dev libffi-dev py3-pip curl bash tar zstd git pip3 install cffi virtualenv + - uses: actions/checkout@v3 + - name: Fix git permissions + run: git config --global --add safe.directory $GITHUB_WORKSPACE - name: Cache cargo build uses: Swatinem/rust-cache@v2 with: @@ -485,6 +487,8 @@ jobs: container: pyston/pyston:2.3.5 steps: - uses: actions/checkout@v3 + - name: Fix git permissions + run: git config --global --add safe.directory $GITHUB_WORKSPACE - uses: dtolnay/rust-toolchain@stable id: rustup with: diff --git a/Changelog.md b/Changelog.md index beecfdfb8..194331d25 100644 --- a/Changelog.md +++ b/Changelog.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add support for configuring macOS deployment target version in `pyproject.toml` in [#1536](https://github.com/PyO3/maturin/pull/1536) * Rewrite platform specific dependencies in `Cargo.toml` by viccie30 in [#1572](https://github.com/PyO3/maturin/pull/1572) * Add trusted publisher support in [#1578](https://github.com/PyO3/maturin/pull/1578) +* Add new `git` source distribution generator in [#1587](https://github.com/PyO3/maturin/pull/1587) ## [0.14.17] - 2023-04-06 diff --git a/guide/src/config.md b/guide/src/config.md index d1e0a93e7..644061db2 100644 --- a/guide/src/config.md +++ b/guide/src/config.md @@ -54,6 +54,9 @@ python-source = "src" python-packages = ["foo", "bar"] # Strip the library for minimum file size strip = true +# Source distribution generator, +# supports cargo (default) and git. +sdist-generator = "cargo" ``` The `[tool.maturin.include]` and `[tool.maturin.exclude]` configuration are diff --git a/src/pyproject_toml.rs b/src/pyproject_toml.rs index 8fe645247..a2afb1c63 100644 --- a/src/pyproject_toml.rs +++ b/src/pyproject_toml.rs @@ -9,10 +9,11 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; /// The `[tool]` section of a pyproject.toml -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(rename_all = "kebab-case")] pub struct Tool { - maturin: Option, + /// maturin options + pub maturin: Option, } #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -104,28 +105,48 @@ pub struct TargetConfig { pub macos_deployment_target: Option, } +/// Source distribution generator +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)] +#[serde(rename_all = "kebab-case")] +pub enum SdistGenerator { + /// Use `cargo package --list` + #[default] + Cargo, + /// Use `git ls-files` + Git, +} + /// The `[tool.maturin]` section of a pyproject.toml -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(rename_all = "kebab-case")] pub struct ToolMaturin { // maturin specific options - // extension module name, accepts setuptools style import name like `foo.bar` - module_name: Option, - include: Option>, - exclude: Option>, - bindings: Option, + /// Module name, accepts setuptools style import name like `foo.bar` + pub module_name: Option, + /// Include files matching the given glob pattern(s) + pub include: Option>, + /// Exclude files matching the given glob pattern(s) + pub exclude: Option>, + /// Bindings type + pub bindings: Option, + /// Platform compatibility #[serde(alias = "manylinux")] - compatibility: Option, + pub compatibility: Option, + /// Skip audit wheel + #[serde(default)] + pub skip_auditwheel: bool, + /// Strip the final binary #[serde(default)] - skip_auditwheel: bool, + pub strip: bool, + /// Source distribution generator #[serde(default)] - strip: bool, + pub sdist_generator: SdistGenerator, /// The directory with python module, contains `/__init__.py` - python_source: Option, + pub python_source: Option, /// Python packages to include - python_packages: Option>, + pub python_packages: Option>, /// Path to the wheel directory, defaults to `.data` - data: Option, + pub data: Option, /// Cargo compile targets pub targets: Option>, /// Target configuration @@ -239,6 +260,13 @@ impl PyProjectToml { .unwrap_or_default() } + /// Returns the value of `[tool.maturin.sdist-generator]` in pyproject.toml + pub fn sdist_generator(&self) -> SdistGenerator { + self.maturin() + .map(|maturin| maturin.sdist_generator) + .unwrap_or_default() + } + /// Returns the value of `[tool.maturin.python-source]` in pyproject.toml pub fn python_source(&self) -> Option<&Path> { self.maturin() diff --git a/src/source_distribution.rs b/src/source_distribution.rs index 084f4adaa..dd93cff79 100644 --- a/src/source_distribution.rs +++ b/src/source_distribution.rs @@ -1,4 +1,5 @@ use crate::module_writer::{add_data, ModuleWriter}; +use crate::pyproject_toml::SdistGenerator; use crate::{pyproject_toml::Format, BuildContext, PyProjectToml, SDistWriter}; use anyhow::{bail, Context, Result}; use cargo_metadata::{Metadata, MetadataCommand}; @@ -564,29 +565,53 @@ fn find_path_deps(cargo_metadata: &Metadata) -> Result> Ok(path_deps) } -/// Creates a source distribution, packing the root crate and all local dependencies +/// Copies the files of git to a source distribution /// -/// The source distribution format is specified in -/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist) -/// and in -/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format -pub fn source_distribution( +/// Runs `git ls-files -z` to obtain a list of files to package. +fn add_git_tracked_files_to_sdist( + pyproject_toml_path: &Path, + writer: &mut SDistWriter, + prefix: impl AsRef, +) -> Result<()> { + let pyproject_dir = pyproject_toml_path.parent().unwrap(); + let output = Command::new("git") + .args(["ls-files", "-z"]) + .current_dir(pyproject_dir) + .output() + .context("Failed to run `git ls-files -z`")?; + if !output.status.success() { + bail!( + "Failed to query file list from git: {}\n--- Project Path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}", + output.status, + pyproject_dir.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + + let prefix = prefix.as_ref(); + writer.add_directory(prefix)?; + + let file_paths = str::from_utf8(&output.stdout) + .context("git printed invalid utf-8 ಠ_ಠ")? + .split('\0') + .filter(|s| !s.is_empty()) + .map(Path::new); + for source in file_paths { + writer.add_file(prefix.join(source), pyproject_dir.join(source))?; + } + Ok(()) +} + +/// Copies the files of a crate to a source distribution, recursively adding path dependencies +/// and rewriting path entries in Cargo.toml +fn add_cargo_package_files_to_sdist( build_context: &BuildContext, - pyproject: &PyProjectToml, - excludes: Option, -) -> Result { - let metadata21 = &build_context.metadata21; + pyproject_toml_path: &Path, + writer: &mut SDistWriter, + root_dir: &Path, +) -> Result<()> { let manifest_path = &build_context.manifest_path; - let pyproject_toml_path = build_context - .pyproject_toml_path - .normalize() - .with_context(|| { - format!( - "failed to normalize path `{}`", - build_context.pyproject_toml_path.display() - ) - })? - .into_path_buf(); let workspace_manifest_path = build_context .cargo_metadata .workspace_root @@ -596,13 +621,6 @@ pub fn source_distribution( let known_path_deps = find_path_deps(&build_context.cargo_metadata)?; - let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?; - let root_dir = PathBuf::from(format!( - "{}-{}", - &metadata21.get_distribution_escaped(), - &metadata21.get_version_escaped() - )); - // Add local path dependencies let mut path_dep_workspace_manifests = HashMap::new(); for (name, path_dep) in known_path_deps.iter() { @@ -635,8 +653,8 @@ pub fn source_distribution( &path_dep_workspace_manifests[&path_dep_metadata.workspace_root] }; add_crate_to_source_distribution( - &mut writer, - &pyproject_toml_path, + writer, + pyproject_toml_path, path_dep, path_dep_workspace_manifest, &root_dir.join(LOCAL_DEPENDENCIES_FOLDER).join(name), @@ -652,11 +670,11 @@ pub fn source_distribution( // Add the main crate add_crate_to_source_distribution( - &mut writer, - &pyproject_toml_path, + writer, + pyproject_toml_path, manifest_path, &workspace_manifest, - &root_dir, + root_dir, &known_path_deps, true, )?; @@ -733,6 +751,61 @@ pub fn source_distribution( } } + Ok(()) +} + +/// Creates a source distribution, packing the root crate and all local dependencies +/// +/// The source distribution format is specified in +/// [PEP 517 under "build_sdist"](https://www.python.org/dev/peps/pep-0517/#build-sdist) +/// and in +/// https://packaging.python.org/specifications/source-distribution-format/#source-distribution-file-format +pub fn source_distribution( + build_context: &BuildContext, + pyproject: &PyProjectToml, + excludes: Option, +) -> Result { + let pyproject_toml_path = build_context + .pyproject_toml_path + .normalize() + .with_context(|| { + format!( + "failed to normalize path `{}`", + build_context.pyproject_toml_path.display() + ) + })? + .into_path_buf(); + let metadata21 = &build_context.metadata21; + let mut writer = SDistWriter::new(&build_context.out, metadata21, excludes)?; + let root_dir = PathBuf::from(format!( + "{}-{}", + &metadata21.get_distribution_escaped(), + &metadata21.get_version_escaped() + )); + + match pyproject.sdist_generator() { + SdistGenerator::Cargo => add_cargo_package_files_to_sdist( + build_context, + &pyproject_toml_path, + &mut writer, + &root_dir, + )?, + SdistGenerator::Git => { + add_git_tracked_files_to_sdist(&pyproject_toml_path, &mut writer, &root_dir)? + } + } + + let pyproject_toml_path = build_context + .pyproject_toml_path + .normalize() + .with_context(|| { + format!( + "failed to normalize path `{}`", + build_context.pyproject_toml_path.display() + ) + })? + .into_path_buf(); + let pyproject_dir = pyproject_toml_path.parent().unwrap(); // Add readme, license if let Some(project) = pyproject.project.as_ref() { if let Some(pyproject_toml::ReadMe::RelativePath(readme)) = project.readme.as_ref() { diff --git a/test-crates/workspace_with_path_dep/python/pyproject.toml b/test-crates/workspace_with_path_dep/pyproject.toml similarity index 59% rename from test-crates/workspace_with_path_dep/python/pyproject.toml rename to test-crates/workspace_with_path_dep/pyproject.toml index 50dd4f373..b79c30ced 100644 --- a/test-crates/workspace_with_path_dep/python/pyproject.toml +++ b/test-crates/workspace_with_path_dep/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["maturin>=0.14,<0.15"] build-backend = "maturin" + +[tool.maturin] +manifest-path = "python/Cargo.toml" diff --git a/tests/common/other.rs b/tests/common/other.rs index 095ea4ec2..e0fca6060 100644 --- a/tests/common/other.rs +++ b/tests/common/other.rs @@ -1,6 +1,7 @@ use anyhow::{Context, Result}; use clap::Parser; use flate2::read::GzDecoder; +use maturin::pyproject_toml::{SdistGenerator, ToolMaturin}; use maturin::{BuildOptions, CargoOptions, PlatformTag}; use pretty_assertions::assert_eq; use std::collections::BTreeSet; @@ -115,6 +116,7 @@ pub fn test_workspace_cargo_lock() -> Result<()> { pub fn test_source_distribution( package: impl AsRef, + sdist_generator: SdistGenerator, expected_files: Vec<&str>, expected_cargo_toml: Option<(&Path, &str)>, unique_name: &str, @@ -135,7 +137,22 @@ pub fn test_source_distribution( ..Default::default() }; - let build_context = build_options.into_build_context(false, false, false)?; + let mut build_context = build_options.into_build_context(false, false, false)?; + + // Override the sdist generator for testing + let mut pyproject_toml = build_context.pyproject_toml.take().unwrap(); + let mut tool = pyproject_toml.tool.clone().unwrap_or_default(); + if let Some(ref mut tool_maturin) = tool.maturin { + tool_maturin.sdist_generator = sdist_generator; + } else { + tool.maturin = Some(ToolMaturin { + sdist_generator, + ..Default::default() + }); + } + pyproject_toml.tool = Some(tool); + build_context.pyproject_toml = Some(pyproject_toml); + let (path, _) = build_context .build_source_distribution()? .context("Failed to build source distribution")?; diff --git a/tests/run.rs b/tests/run.rs index e72948eb7..dea8dedad 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -4,6 +4,7 @@ use common::{ develop, errors, get_python_implementation, handle_result, integration, other, test_python_path, }; use indoc::indoc; +use maturin::pyproject_toml::SdistGenerator; use maturin::Target; use std::env; use std::path::{Path, PathBuf}; @@ -441,6 +442,7 @@ fn workspace_members_non_local_dep_sdist() { ); handle_result(other::test_source_distribution( "test-crates/pyo3-pure", + SdistGenerator::Cargo, vec![ "pyo3_pure-0.1.0+abc123de/Cargo.lock", "pyo3_pure-0.1.0+abc123de/Cargo.toml", @@ -463,6 +465,7 @@ fn workspace_members_non_local_dep_sdist() { fn lib_with_path_dep_sdist() { handle_result(other::test_source_distribution( "test-crates/sdist_with_path_dep", + SdistGenerator::Cargo, vec![ "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml", "sdist_with_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs", @@ -502,6 +505,7 @@ fn lib_with_target_path_dep() { ); handle_result(other::test_source_distribution( "test-crates/sdist_with_target_path_dep", + SdistGenerator::Cargo, vec![ "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/Cargo.toml", "sdist_with_target_path_dep-0.1.0/local_dependencies/some_path_dep/src/lib.rs", @@ -525,6 +529,7 @@ fn lib_with_target_path_dep() { fn pyo3_mixed_src_layout_sdist() { handle_result(other::test_source_distribution( "test-crates/pyo3-mixed-src/rust", + SdistGenerator::Cargo, vec![ "pyo3_mixed_src-2.1.3/pyproject.toml", "pyo3_mixed_src-2.1.3/src/pyo3_mixed_src/__init__.py", @@ -545,6 +550,7 @@ fn pyo3_mixed_src_layout_sdist() { fn pyo3_mixed_include_exclude_sdist() { handle_result(other::test_source_distribution( "test-crates/pyo3-mixed-include-exclude", + SdistGenerator::Cargo, vec![ // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded "pyo3_mixed_include_exclude-2.1.3/Cargo.lock", @@ -567,6 +573,33 @@ fn pyo3_mixed_include_exclude_sdist() { )) } +#[test] +fn pyo3_mixed_include_exclude_git_sdist_generator() { + handle_result(other::test_source_distribution( + "test-crates/pyo3-mixed-include-exclude", + SdistGenerator::Git, + vec![ + // "pyo3_mixed_include_exclude-2.1.3/.gitignore", // excluded + "pyo3_mixed_include_exclude-2.1.3/Cargo.lock", + "pyo3_mixed_include_exclude-2.1.3/Cargo.toml", + "pyo3_mixed_include_exclude-2.1.3/PKG-INFO", + "pyo3_mixed_include_exclude-2.1.3/README.md", + "pyo3_mixed_include_exclude-2.1.3/check_installed/check_installed.py", + // "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/exclude_this_file, excluded + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/__init__.py", + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/include_this_file", // included + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/__init__.py", + "pyo3_mixed_include_exclude-2.1.3/pyo3_mixed_include_exclude/python_module/double.py", + "pyo3_mixed_include_exclude-2.1.3/pyproject.toml", + "pyo3_mixed_include_exclude-2.1.3/src/lib.rs", + // "pyo3_mixed_include_exclude-2.1.3/tests/test_pyo3_mixed_include_exclude.py", excluded + "pyo3_mixed_include_exclude-2.1.3/tox.ini", + ], + None, + "sdist-pyo3-mixed-include-exclude-git", + )) +} + #[test] fn pyo3_mixed_include_exclude_wheel_files() { handle_result(other::check_wheel_files( @@ -590,6 +623,7 @@ fn pyo3_mixed_include_exclude_wheel_files() { fn workspace_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace/py", + SdistGenerator::Cargo, vec![ "py-0.1.0/Cargo.lock", "py-0.1.0/Cargo.toml", @@ -606,19 +640,44 @@ fn workspace_sdist() { fn workspace_with_path_dep_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace_with_path_dep/python", + SdistGenerator::Cargo, vec![ "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/Cargo.toml", "workspace_with_path_dep-0.1.0/local_dependencies/generic_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/Cargo.toml", "workspace_with_path_dep-0.1.0/local_dependencies/transitive_lib/src/lib.rs", + "workspace_with_path_dep-0.1.0/python/Cargo.lock", + "workspace_with_path_dep-0.1.0/python/Cargo.toml", + "workspace_with_path_dep-0.1.0/python/src/lib.rs", + "workspace_with_path_dep-0.1.0/pyproject.toml", + "workspace_with_path_dep-0.1.0/PKG-INFO", + ], + None, + "sdist-workspace-with-path-dep", + )) +} + +#[test] +fn workspace_with_path_dep_git_sdist_generator() { + handle_result(other::test_source_distribution( + "test-crates/workspace_with_path_dep/python", + SdistGenerator::Git, + vec![ "workspace_with_path_dep-0.1.0/Cargo.lock", "workspace_with_path_dep-0.1.0/Cargo.toml", + "workspace_with_path_dep-0.1.0/dont_include_in_sdist/Cargo.toml", + "workspace_with_path_dep-0.1.0/dont_include_in_sdist/src/main.rs", + "workspace_with_path_dep-0.1.0/generic_lib/Cargo.toml", + "workspace_with_path_dep-0.1.0/generic_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/pyproject.toml", - "workspace_with_path_dep-0.1.0/src/lib.rs", + "workspace_with_path_dep-0.1.0/python/Cargo.toml", + "workspace_with_path_dep-0.1.0/python/src/lib.rs", + "workspace_with_path_dep-0.1.0/transitive_lib/Cargo.toml", + "workspace_with_path_dep-0.1.0/transitive_lib/src/lib.rs", "workspace_with_path_dep-0.1.0/PKG-INFO", ], None, - "sdist-workspace-with-path-dep", + "sdist-workspace-with-path-dep-git", )) } @@ -627,6 +686,7 @@ fn workspace_with_path_dep_sdist() { fn workspace_inheritance_sdist() { handle_result(other::test_source_distribution( "test-crates/workspace-inheritance/python", + SdistGenerator::Cargo, vec![ "workspace_inheritance-0.1.0/local_dependencies/generic_lib/Cargo.toml", "workspace_inheritance-0.1.0/local_dependencies/generic_lib/src/lib.rs",