rustPlatform.fetchCargoVendor: init

doc/languages-frameworks/rust.section.md

@@ -64,10 +64,18 @@ hash using `nix-hash --to-sri --type sha256 "<original sha256>"`.
 ```
 
 Exception: If the application has cargo `git` dependencies, the `cargoHash`
-approach will not work, and you will need to copy the `Cargo.lock` file of the application
-to nixpkgs and continue with the next section for specifying the options of the `cargoLock`
-section.
+approach will not work by default. In this case, you can set `useFetchCargoVendor = true`
+to use an improved fetcher that supports handling `git` dependencies.
 
+```nix
+{
+  useFetchCargoVendor = true;
+  cargoHash = "sha256-RqPVFovDaD2rW31HyETJfQ0qVwFxoGEvqkIgag3H6KU=";
+}
+```
+
+If this method still does not work, you can resort to copying the `Cargo.lock` file into nixpkgs
+and importing it as described in the [next section](#importing-a-cargo.lock-file).
 
 Both types of hashes are permitted when contributing to nixpkgs. The
 Cargo hash is obtained by inserting a fake checksum into the
@@ -462,6 +470,17 @@ also be used:
   the `Cargo.lock`/`Cargo.toml` files need to be patched before
   vendoring.
 
+In case the lockfile contains cargo `git` dependencies, you can use
+`fetchCargoVendor` instead.
+```nix
+{
+  cargoDeps = rustPlatform.fetchCargoVendor {
+    inherit src;
+    hash = "sha256-RqPVFovDaD2rW31HyETJfQ0qVwFxoGEvqkIgag3H6KU=";
+  };
+}
+```
+
 If a `Cargo.lock` file is available, you can alternatively use the
 `importCargoLock` function. In contrast to `fetchCargoTarball`, this
 function does not require a hash (unless git dependencies are used)

pkgs/build-support/rust/build-rust-package/default.nix

@@ -1,6 +1,7 @@
 { lib
 , importCargoLock
 , fetchCargoTarball
+, fetchCargoVendor
 , stdenv
 , callPackage
 , cargoBuildHook
@@ -36,6 +37,7 @@
 , cargoDepsHook ? ""
 , buildType ? "release"
 , meta ? {}
+, useFetchCargoVendor ? false
 , cargoLock ? null
 , cargoVendorDir ? null
 , checkType ? buildType
@@ -67,6 +69,12 @@ let
   cargoDeps =
     if cargoVendorDir != null then null
     else if cargoLock != null then importCargoLock cargoLock
+    else if useFetchCargoVendor then (fetchCargoVendor {
+      inherit src srcs sourceRoot preUnpack unpackPhase postUnpack;
+      name = cargoDepsName;
+      patches = cargoPatches;
+      hash = args.cargoHash;
+    } // depsExtraArgs)
     else fetchCargoTarball ({
       inherit src srcs sourceRoot preUnpack unpackPhase postUnpack cargoUpdateHook;
       name = cargoDepsName;

pkgs/build-support/rust/fetch-cargo-tarball/default.nix

@@ -120,8 +120,9 @@ stdenv.mkDerivation (
           echo
           echo "ERROR: The Cargo.lock contains git dependencies"
           echo
-          echo "This is currently not supported in the fixed-output derivation fetcher."
-          echo "Use cargoLock.lockFile / importCargoLock instead."
+          echo "This is not supported in the default fixed-output derivation fetcher."
+          echo "Set \`useFetchCargoVendor = true\` / use fetchCargoVendor"
+          echo "or use cargoLock.lockFile / importCargoLock instead."
           echo
 
           exit 1

pkgs/build-support/rust/fetch-cargo-vendor-util.py

@@ -0,0 +1,284 @@
+import functools
+import hashlib
+import json
+import multiprocessing as mp
+import re
+import shutil
+import subprocess
+import sys
+import tomllib
+from pathlib import Path
+from typing import Any, TypedDict, cast
+
+import requests
+
+eprint = functools.partial(print, file=sys.stderr)
+
+
+def load_toml(path: Path) -> dict[str, Any]:
+    with open(path, "rb") as f:
+        return tomllib.load(f)
+
+
+def download_file_with_checksum(url: str, destination_path: Path) -> str:
+    sha256_hash = hashlib.sha256()
+    with requests.get(url, stream=True) as response:
+        if not response.ok:
+            raise Exception(f"Failed to fetch file from {url}. Status code: {response.status_code}")
+        with open(destination_path, "wb") as file:
+            for chunk in response.iter_content(1024):  # Download in chunks
+                if chunk:  # Filter out keep-alive chunks
+                    file.write(chunk)
+                    sha256_hash.update(chunk)
+
+    # Compute the final checksum
+    checksum = sha256_hash.hexdigest()
+    return checksum
+
+
+def get_download_url_for_tarball(pkg: dict[str, Any]) -> str:
+    # TODO: support other registries
+    #       maybe fetch config.json from the registry root and get the dl key
+    #       See: https://doc.rust-lang.org/cargo/reference/registry-index.html#index-configuration
+    if pkg["source"] != "registry+https://github.com/rust-lang/crates.io-index":
+        raise Exception("Only the default crates.io registry is supported.")
+
+    return f"https://crates.io/api/v1/crates/{pkg["name"]}/{pkg["version"]}/download"
+
+
+def download_tarball(pkg: dict[str, Any], out_dir: Path) -> None:
+
+    url = get_download_url_for_tarball(pkg)
+    filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"
+
+    # TODO: allow legacy checksum specification, see importCargoLock for example
+    #       also, don't forget about the other usage of the checksum
+    expected_checksum = pkg["checksum"]
+
+    tarball_out_dir = out_dir / "tarballs" / filename
+    eprint(f"Fetching {url} -> tarballs/{filename}")
+
+    calculated_checksum = download_file_with_checksum(url, tarball_out_dir)
+
+    if calculated_checksum != expected_checksum:
+        raise Exception(f"Hash mismatch! File fetched from {url} had checksum {calculated_checksum}, expected {expected_checksum}.")
+
+
+def download_git_tree(url: str, git_sha_rev: str, out_dir: Path) -> None:
+
+    tree_out_dir = out_dir / "git" / git_sha_rev
+    eprint(f"Fetching {url}#{git_sha_rev} -> git/{git_sha_rev}")
+
+    cmd = ["nix-prefetch-git", "--builder", "--quiet", "--url", url, "--rev", git_sha_rev, "--out", str(tree_out_dir)]
+    subprocess.check_output(cmd)
+
+
+GIT_SOURCE_REGEX = re.compile("git\\+(?P<url>[^?]+)(\\?(?P<type>rev|tag|branch)=(?P<value>.*))?#(?P<git_sha_rev>.*)")
+
+
+class GitSourceInfo(TypedDict):
+    url: str
+    type: str | None
+    value: str | None
+    git_sha_rev: str
+
+
+def parse_git_source(source: str) -> GitSourceInfo:
+    match = GIT_SOURCE_REGEX.match(source)
+    if match is None:
+        raise Exception(f"Unable to process git source: {source}.")
+    return cast(GitSourceInfo, match.groupdict(default=None))
+
+
+def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None:
+    cargo_toml = load_toml(lockfile_path)
+
+    git_packages: list[dict[str, Any]] = []
+    registry_packages: list[dict[str, Any]] = []
+
+    for pkg in cargo_toml["package"]:
+        # ignore local dependenices
+        if "source" not in pkg.keys():
+            eprint(f"Skipping local dependency: {pkg["name"]}")
+            continue
+        source = pkg["source"]
+
+        if source.startswith("git+"):
+            git_packages.append(pkg)
+        elif source.startswith("registry+"):
+            registry_packages.append(pkg)
+        else:
+            raise Exception(f"Can't process source: {source}.")
+
+    git_sha_rev_to_url: dict[str, str] = {}
+    for pkg in git_packages:
+        source_info = parse_git_source(pkg["source"])
+        git_sha_rev_to_url[source_info["git_sha_rev"]] = source_info["url"]
+
+    out_dir.mkdir(exist_ok=True)
+    shutil.copy(lockfile_path, out_dir / "Cargo.lock")
+
+    # create a pool with at most 10 concurrent jobs
+    with mp.Pool(min(10, mp.cpu_count())) as pool:
+
+        if len(git_packages) != 0:
+            (out_dir / "git").mkdir()
+            # run download jobs in parallel
+            git_args_gen = ((url, git_sha_rev, out_dir) for git_sha_rev, url in git_sha_rev_to_url.items())
+            pool.starmap(download_git_tree, git_args_gen)
+
+        if len(registry_packages) != 0:
+            (out_dir / "tarballs").mkdir()
+            # run download jobs in parallel
+            tarball_args_gen = ((pkg, out_dir) for pkg in registry_packages)
+            pool.starmap(download_tarball, tarball_args_gen)
+
+
+def get_manifest_metadata(manifest_path: Path) -> dict[str, Any]:
+    cmd = ["cargo", "metadata", "--format-version", "1", "--no-deps", "--manifest-path", str(manifest_path)]
+    output = subprocess.check_output(cmd)
+    return json.loads(output)
+
+
+def try_get_crate_manifest_path_from_mainfest_path(manifest_path: Path, crate_name: str) -> Path | None:
+    metadata = get_manifest_metadata(manifest_path)
+
+    for pkg in metadata["packages"]:
+        if pkg["name"] == crate_name:
+            return Path(pkg["manifest_path"])
+
+    return None
+
+
+def find_crate_manifest_in_tree(tree: Path, crate_name: str) -> Path:
+    # in some cases Cargo.toml is not located at the top level, so we also look at subdirectories
+    manifest_paths = tree.glob("**/Cargo.toml")
+
+    for manifest_path in manifest_paths:
+        res = try_get_crate_manifest_path_from_mainfest_path(manifest_path, crate_name)
+        if res is not None:
+            return res
+
+    raise Exception(f"Couldn't find manifest for crate {crate_name} inside {tree}.")
+
+
+def copy_and_patch_git_crate_subtree(git_tree: Path, crate_name: str, crate_out_dir: Path) -> None:
+    crate_manifest_path = find_crate_manifest_in_tree(git_tree, crate_name)
+    crate_tree = crate_manifest_path.parent
+
+    eprint(f"Copying to {crate_out_dir}")
+    shutil.copytree(crate_tree, crate_out_dir)
+    crate_out_dir.chmod(0o755)
+
+    with open(crate_manifest_path, "r") as f:
+        manifest_data = f.read()
+
+    if "workspace" in manifest_data:
+        crate_manifest_metadata = get_manifest_metadata(crate_manifest_path)
+        workspace_root = Path(crate_manifest_metadata["workspace_root"])
+
+        root_manifest_path = workspace_root / "Cargo.toml"
+        manifest_path = crate_out_dir / "Cargo.toml"
+
+        manifest_path.chmod(0o644)
+        eprint(f"Patching {manifest_path}")
+
+        cmd = ["replace-workspace-values", str(manifest_path), str(root_manifest_path)]
+        subprocess.check_output(cmd)
+
+
+def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> None:
+    eprint(f"Unpacking to {crate_out_dir}")
+    crate_out_dir.mkdir()
+    cmd = ["tar", "xf", str(tarball_path), "-C", str(crate_out_dir), "--strip-components=1"]
+    subprocess.check_output(cmd)
+
+
+def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
+    lockfile_path = vendor_staging_dir / "Cargo.lock"
+    out_dir.mkdir(exist_ok=True)
+    shutil.copy(lockfile_path, out_dir / "Cargo.lock")
+
+    cargo_toml = load_toml(lockfile_path)
+
+    config_lines = [
+        '[source.vendored-sources]',
+        'directory = "@vendor@"',
+        '[source.crates-io]',
+        'replace-with = "vendored-sources"',
+    ]
+
+    seen_source_keys = set()
+    for pkg in cargo_toml["package"]:
+
+        # ignore local dependenices
+        if "source" not in pkg.keys():
+            continue
+
+        source: str = pkg["source"]
+
+        dir_name = f"{pkg["name"]}-{pkg["version"]}"
+        crate_out_dir = out_dir / dir_name
+
+        if source.startswith("git+"):
+
+            source_info = parse_git_source(pkg["source"])
+            git_sha_rev = source_info["git_sha_rev"]
+            git_tree = vendor_staging_dir / "git" / git_sha_rev
+
+            copy_and_patch_git_crate_subtree(git_tree, pkg["name"], crate_out_dir)
+
+            # git based crates allow having no checksum information
+            with open(crate_out_dir / ".cargo-checksum.json", "w") as f:
+                json.dump({"files": {}}, f)
+
+            source_key = source[0:source.find("#")]
+
+            if source_key in seen_source_keys:
+                continue
+
+            seen_source_keys.add(source_key)
+
+            config_lines.append(f'[source."{source_key}"]')
+            config_lines.append(f'git = "{source_info["url"]}"')
+            if source_info["type"] is not None:
+                config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"')
+            config_lines.append('replace-with = "vendored-sources"')
+
+        elif source.startswith("registry+"):
+
+            filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"
+            tarball_path = vendor_staging_dir / "tarballs" / filename
+
+            extract_crate_tarball_contents(tarball_path, crate_out_dir)
+
+            # non-git based crates need the package checksum at minimum
+            with open(crate_out_dir / ".cargo-checksum.json", "w") as f:
+                json.dump({"files": {}, "package": pkg["checksum"]}, f)
+
+        else:
+            raise Exception(f"Can't process source: {source}.")
+
+    (out_dir / ".cargo").mkdir()
+    with open(out_dir / ".cargo" / "config.toml", "w") as config_file:
+        config_file.writelines(line + "\n" for line in config_lines)
+
+
+def main() -> None:
+    subcommand = sys.argv[1]
+
+    subcommand_func_dict = {
+        "create-vendor-staging": lambda: create_vendor_staging(lockfile_path=Path(sys.argv[2]), out_dir=Path(sys.argv[3])),
+        "create-vendor": lambda: create_vendor(vendor_staging_dir=Path(sys.argv[2]), out_dir=Path(sys.argv[3]))
+    }
+
+    subcommand_func = subcommand_func_dict.get(subcommand)
+
+    if subcommand_func is None:
+        raise Exception(f"Unknown subcommand: '{subcommand}'. Must be one of {list(subcommand_func_dict.keys())}")
+
+    subcommand_func()
+
+
+if __name__ == "__main__":
+    main()