Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(nix): Fix dependency caching #1421

Merged
merged 5 commits into from
May 30, 2023
Merged

chore(nix): Fix dependency caching #1421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
db0eb38
chore: Move the git information to improve caching
phated May 26, 2023
bed7f2c
change some rust code
phated May 26, 2023
b0c2e4d
cache changes
phated May 26, 2023
3139ca1
remove rust changes
phated May 26, 2023
b3449bb
Update .github/workflows/test.yml
phated May 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,4 @@ jobs:
- name: Export cache from nix store
if: steps.nix-store-cache.outputs.cache-hit != 'true'
run: |
nix copy --to "file:///tmp/nix-cache?compression=zstd&parallel-compression=true" .#cargo-artifacts
nix copy --to "file:///tmp/nix-cache?compression=zstd&parallel-compression=true" .#native-cargo-artifacts
130 changes: 88 additions & 42 deletions flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,28 +68,41 @@

craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain;

environment = {
sharedEnvironment = {
# We enable backtraces on any failure for help with debugging
RUST_BACKTRACE = "1";
};

nativeEnvironment = sharedEnvironment // {
# rust-bindgen needs to know the location of libclang
LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
};

wasmEnvironment = sharedEnvironment // {
# We set the environment variable because barretenberg must be compiled in a special way for wasm
BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin";
};

testEnvironment = sharedEnvironment // {
# Barretenberg fails if tests are run on multiple threads, so we set the test thread
# count to 1 throughout the entire project
#
# Note: Setting this allows for consistent behavior across build and shells, but is mostly
# hidden from the developer - i.e. when they see the command being run via `nix flake check`
RUST_TEST_THREADS = "1";

# We enable backtraces on any failure for help with debugging
RUST_BACKTRACE = "1";

# We set the environment variable because barretenberg must be compiled in a special way for wasm
BARRETENBERG_BIN_DIR = "${pkgs.barretenberg-wasm}/bin";
};

# The `self.rev` property is only available when the working tree is not dirty
GIT_COMMIT = if (self ? rev) then self.rev else "unknown";
GIT_DIRTY = if (self ? rev) then "false" else "true";

# We use `.nr` and `.toml` files in tests so we need to create a special source
# filter to include those files in addition to usual rust/cargo source files
noirFilter = path: _type: builtins.match ".*nr$" path != null;
tomlFilter = path: _type: builtins.match ".*toml$" path != null;
sourceFilter = path: type:
(noirFilter path type) || (tomlFilter path type) || (craneLib.filterCargoSources path type);

# As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses
# gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++
stdenv =
Expand All @@ -98,21 +111,35 @@
else
pkgs.llvmPackages.stdenv;

# Combine the environment and other configuration needed for crane to build our Rust packages
commonArgs = environment // {
pname = "noir";
extraBuildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [
# Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156
pkgs.libiconv
pkgs.darwin.apple_sdk.frameworks.Security
];

sharedArgs = {
# x-release-please-start-version
version = "0.6.0";
# x-release-please-end

# Use our custom stdenv to build and test our Rust project
inherit stdenv;
src = pkgs.lib.cleanSourceWith {
src = craneLib.path ./.;
filter = sourceFilter;
};

src = ./.;
# TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell.
cargoClippyExtraArgs = "--all-targets -- -D warnings";

# Running checks don't do much more than compiling itself and increase
# the build time by a lot, so we disable them throughout all our flakes
doCheck = false;
# TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell.
cargoTestExtraArgs = "--workspace";
};

# Combine the environment and other configuration needed for crane to build our Rust packages
nativeArgs = nativeEnvironment // sharedArgs // {
pname = "noir-native";

# Use our custom stdenv to build and test our Rust project
inherit stdenv;

nativeBuildInputs = [
# This provides the pkg-config tool to find barretenberg & other native libraries
Expand All @@ -124,18 +151,21 @@
buildInputs = [
pkgs.llvmPackages.openmp
pkgs.barretenberg
] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
# Need libiconv and apple Security on Darwin. See https://github.com/ipetkov/crane/issues/156
pkgs.libiconv
pkgs.darwin.apple_sdk.frameworks.Security
];
] ++ extraBuildInputs;
};

# Combine the environment and other configuration needed for crane to build with the wasm feature
wasmArgs = wasmEnvironment // sharedArgs // {
pname = "noir-wasm";

inherit GIT_COMMIT;
inherit GIT_DIRTY;
# We disable the default "plonk_bn254" feature and enable the "plonk_bn254_wasm" feature
cargoExtraArgs = "--no-default-features --features='plonk_bn254_wasm'";

buildInputs = [ ] ++ extraBuildInputs;
};

# The `port` is parameterized to support parallel test runs without colliding static servers
testArgs = port: {
testArgs = port: testEnvironment // {
# We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server
# This is necessary because the Nix sandbox has no network access and downloading during tests would fail
TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}";
Expand All @@ -160,44 +190,60 @@
};

# Build *just* the cargo dependencies, so we can reuse all of that work between runs
cargoArtifacts = craneLib.buildDepsOnly commonArgs;
native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs;
wasm-cargo-artifacts = craneLib.buildDepsOnly wasmArgs;

noir-native = craneLib.buildPackage (nativeArgs // {
inherit GIT_COMMIT GIT_DIRTY;

noir = craneLib.buildPackage (commonArgs // {
inherit cargoArtifacts;
cargoArtifacts = native-cargo-artifacts;

# We don't want to run checks or tests when just building the project
doCheck = false;
});

noir-wasm = craneLib.buildPackage (wasmArgs // {
inherit GIT_COMMIT GIT_DIRTY;

cargoArtifacts = wasm-cargo-artifacts;

# We don't want to run checks or tests when just building the project
doCheck = false;
});
in
rec {
checks = {
cargo-clippy = craneLib.cargoClippy (commonArgs // {
inherit cargoArtifacts;
cargo-clippy = craneLib.cargoClippy (nativeArgs // {
inherit GIT_COMMIT GIT_DIRTY;

# TODO(#1198): It'd be nice to include these flags when running `cargo clippy` in a devShell.
cargoClippyExtraArgs = "--all-targets -- -D warnings";
cargoArtifacts = native-cargo-artifacts;
});

cargo-test = craneLib.cargoTest (commonArgs // (testArgs 8000) // {
inherit cargoArtifacts;

# TODO(#1198): It'd be nice to include this flag when running `cargo test` in a devShell.
cargoTestExtraArgs = "--workspace";
cargo-test = craneLib.cargoTest (nativeArgs // (testArgs 8000) // {
inherit GIT_COMMIT GIT_DIRTY;

# It's unclear why doCheck needs to be enabled for tests to run but not clippy
doCheck = true;
cargoArtifacts = native-cargo-artifacts;
});
};

packages.default = noir;
packages = {
default = noir-native;

# We expose the `cargo-artifacts` derivation so we can cache our cargo dependencies in CI
packages.cargo-artifacts = cargoArtifacts;
inherit noir-native;
inherit noir-wasm;

# We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI
inherit native-cargo-artifacts;
inherit wasm-cargo-artifacts;
};

# TODO(#1197): Look into installable apps with Nix flakes
# apps.default = flake-utils.lib.mkApp { drv = nargo; };

# Setup the environment to match the stdenv from `nix build` & `nix flake check`, and
# combine it with the environment settings, the inputs from our checks derivations,
# and extra tooling via `nativeBuildInputs`
devShells.default = pkgs.mkShell.override { inherit stdenv; } (environment // {
devShells.default = pkgs.mkShell.override { inherit stdenv; } (nativeEnvironment // wasmEnvironment // testEnvironment // {
inputsFrom = builtins.attrValues checks;

nativeBuildInputs = with pkgs; [
Expand Down