From da85938e6b6a6b893f7b36b64fba30122763f833 Mon Sep 17 00:00:00 2001 From: Adam Spofford <93943719+adamspofford-dfinity@users.noreply.github.com> Date: Tue, 25 Oct 2022 14:34:19 -0700 Subject: [PATCH] chore: Rewrite build script in Rust (#2706) The build script currently requires a shell script to curl all the bundled assets and pack them into tarballs. This PR rewrites the script in Rust, which not only does all the fetching in parallel but supports Windows. The format of the sources manifest has been changed from a shell script to a .toml file, as the latter is far more easily consumed from Rust and will help eliminate some edge cases like `motoko-base`. Given Windows assets, DFX now builds on Windows. Linux assets are substituted as a stopgap since technically any external program can be used through WSL, but we probably want to have a proper Windows asset set eventually. --- .github/workflows/lint.yml | 2 +- .github/workflows/prepare-dfx-assets.yml | 6 +- .github/workflows/shellcheck.yml | 3 +- Cargo.lock | 6 +- Cargo.toml | 3 + nix/sources.json | 16 +- scripts/dfx-asset-sources.sh | 54 ------ scripts/prepare-dfx-assets.sh | 161 ----------------- scripts/update-replica.sh | 8 +- scripts/write-dfx-asset-sources.sh | 68 +++---- src/dfx/Cargo.toml | 7 +- src/dfx/assets/README.adoc | 4 +- src/dfx/assets/build.rs | 94 +++++----- src/dfx/assets/dfx-asset-sources.toml | 110 +++++++++++ src/dfx/assets/prepare_assets.rs | 221 +++++++++++++++++++++++ 15 files changed, 437 insertions(+), 326 deletions(-) delete mode 100644 scripts/dfx-asset-sources.sh delete mode 100755 scripts/prepare-dfx-assets.sh create mode 100644 src/dfx/assets/dfx-asset-sources.toml create mode 100644 src/dfx/assets/prepare_assets.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f58093b59e..809665f417 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: rust: [ '1.60.0' ] - os: [ ubuntu-latest, macos-latest ] + os: [ ubuntu-latest, macos-latest, windows-latest ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/prepare-dfx-assets.yml b/.github/workflows/prepare-dfx-assets.yml index 807b577795..b6c7ef0d00 100644 --- a/.github/workflows/prepare-dfx-assets.yml +++ b/.github/workflows/prepare-dfx-assets.yml @@ -34,6 +34,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target + !target/*/build/dfx-*/out/dfx-assets key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Install Rust run: | @@ -41,8 +42,9 @@ jobs: rustup component add clippy rustup default ${{ matrix.rust }} - - name: Prepare dfx assets - run: scripts/prepare-dfx-assets.sh "$(mktemp -d)" + - name: Run build script + run: | + cargo check aggregate: name: prepare-dfx-assets:required diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 4d7c53aa91..2a9581ad9a 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -36,5 +36,4 @@ jobs: run: $HOME/bin/shellcheck-v0.7.1/shellcheck e2e/**/*.*sh - name: Check release script run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/release.sh - - name: Check asset prep script - run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/prepare-dfx-assets.sh + diff --git a/Cargo.lock b/Cargo.lock index 8f685a8685..945170aa8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,7 @@ dependencies = [ "time", "tiny-bip39", "tokio", + "toml", "url", "walkdir", "wasmparser 0.87.0", @@ -3522,9 +3523,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ "autocfg", "bytes", @@ -3532,7 +3533,6 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", diff --git a/Cargo.toml b/Cargo.toml index 18c2f830bc..4e27f41615 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,9 @@ lto = true [profile.dev.package.argon2] opt-level = 3 +[profile.dev.package.miniz_oxide] +opt-level = 3 + [profile.release.package.ic-frontend-canister] opt-level = 'z' diff --git a/nix/sources.json b/nix/sources.json index 32ce604365..eb3c08a849 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -12,18 +12,18 @@ "url": "https://github.com/RustSec/advisory-db/archive/fe7b79e8ebf9ad03660e573e0c2a1ed4e80dee5d.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "canister-sandbox-x86_64-darwin": { + "canister_sandbox-x86_64-darwin": { "builtin": false, - "description": "The canister-sandbox binary. It must be updated together with the replica binary.", + "description": "The canister_sandbox binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "07d5jf6y6a7kl3s4pgmjs7xsr93cvn6lp56gyr7zxziab81wp1i3", "type": "file", "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-darwin/canister_sandbox.gz" }, - "canister-sandbox-x86_64-linux": { + "canister_sandbox-x86_64-linux": { "builtin": false, - "description": "The canister-sandbox binary. It must be updated together with the replica binary.", + "description": "The canister_sandbox binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "1vyxwlvh59irx0j4rbjdyrj2j7hhhnc1c5552yc0pbmip9xalzkr", "type": "file", @@ -178,7 +178,7 @@ }, "replica-x86_64-darwin": { "builtin": false, - "description": "The replica binary. It must be updated together with the canister-sandbox binary.", + "description": "The replica binary. It must be updated together with the canister_sandbox binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "06mvsa9ibj3waj600digpgnipj74lc020lgvvamzzvpqpvwh6hag", "type": "file", @@ -187,14 +187,14 @@ }, "replica-x86_64-linux": { "builtin": false, - "description": "The replica binary. It must be updated together with the canister-sandbox binary.", + "description": "The replica binary. It must be updated together with the canister_sandbox binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "1c71c3zzljdjgbgx85qgijf75ccqv9vm083sidycnrh8m1lrzh27", "type": "file", "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-linux/replica.gz" }, - "sandbox-launcher-x86_64-darwin": { + "sandbox_launcher-x86_64-darwin": { "builtin": false, "description": "The sandbox_launcher binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", @@ -203,7 +203,7 @@ "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-darwin/sandbox_launcher.gz" }, - "sandbox-launcher-x86_64-linux": { + "sandbox_launcher-x86_64-linux": { "builtin": false, "description": "The sandbox_launcher binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", diff --git a/scripts/dfx-asset-sources.sh b/scripts/dfx-asset-sources.sh deleted file mode 100644 index 8ab7e51053..0000000000 --- a/scripts/dfx-asset-sources.sh +++ /dev/null @@ -1,54 +0,0 @@ -# generated by write-dfx-asset-sources.sh -IC_REF_X86_64_DARWIN_SHA256=07fb2cf2a570d6d90259ce165e6d5b9bbc408508743291be7890d8cb01c8a178 -IC_REF_X86_64_DARWIN_URL=https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-darwin.tar.gz -IC_REF_X86_64_LINUX_SHA256=d6f66f45b3fc904bea7810d3795b7761115b4635c3846991c4516ecaa42e2d36 -IC_REF_X86_64_LINUX_URL=https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-linux.tar.gz -ICX_PROXY_X86_64_DARWIN_SHA256=5783bba5021cf43149bc118789cea29f6462fd97dd78bdb776f8782ea7813d27 -ICX_PROXY_X86_64_DARWIN_URL=https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz -ICX_PROXY_X86_64_LINUX_SHA256=7a5612a1fb7512d22dcd37627a9d626fbc282b172665a832fe2cc2b243789fa1 -ICX_PROXY_X86_64_LINUX_URL=https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz -IC_ADMIN_X86_64_DARWIN_SHA256=303fe743a47180d1a196f7c4cf6acc9d17530768fd1417af30f615a2f92ccd3e -IC_ADMIN_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-admin.gz -IC_ADMIN_X86_64_LINUX_SHA256=c201463f2498dbf980fb3b1fb5ec75f46e7e8464d7530c0423d889ead09de44e -IC_ADMIN_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-admin.gz -IC_BTC_ADAPTER_X86_64_DARWIN_SHA256=b1e26a1b242b29ab61346d8a5856199d735d75ef6aa3ab5af94de250a4797164 -IC_BTC_ADAPTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-btc-adapter.gz -IC_BTC_ADAPTER_X86_64_LINUX_SHA256=26941f1fb7897faa72239871a27a18c1bccdd538fef13911b028f7568ef28c71 -IC_BTC_ADAPTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-btc-adapter.gz -IC_CANISTER_HTTP_ADAPTER_X86_64_DARWIN_SHA256=d9dc8a228b3507f0e1df73781d9738eecd216cbfda7a1f5cb063cbf2a2a7b54b -IC_CANISTER_HTTP_ADAPTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-canister-http-adapter.gz -IC_CANISTER_HTTP_ADAPTER_X86_64_LINUX_SHA256=610a565ec150be9cb29f69b99cecf56f25e131ca740bf41c5b309be140fa0da8 -IC_CANISTER_HTTP_ADAPTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-canister-http-adapter.gz -IC_NNS_INIT_X86_64_DARWIN_SHA256=4086229540cced5742acb8b9885a841e7c47a8e3d7b59a393034d79f7cb6d2a4 -IC_NNS_INIT_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-nns-init.gz -IC_NNS_INIT_X86_64_LINUX_SHA256=71cacac1053c07e4b607948662674c564e6f134b68f5febe49a8e421b7ab2768 -IC_NNS_INIT_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-nns-init.gz -IC_STARTER_X86_64_DARWIN_SHA256=b3b5dcc554b9419f5929cce069e84d8d5070661d8a1a60b278c39272f5916c4e -IC_STARTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-starter.gz -IC_STARTER_X86_64_LINUX_SHA256=0bee932157078e868649365485755e9055391a6dcb05e369f617a95c05c461e6 -IC_STARTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-starter.gz -MOTOKO_X86_64_DARWIN_SHA256=0fd98250ce7f2c6dbbaa9ccae141ad60d9542bd4e915e0fb4f82a235db666c88 -MOTOKO_X86_64_DARWIN_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-macos-0.7.1.tar.gz -MOTOKO_X86_64_LINUX_SHA256=132b13907425d3a7bccf10f48a420140244e663f0b48b200908c295a71a3a3c7 -MOTOKO_X86_64_LINUX_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-linux64-0.7.1.tar.gz -# The replica and canister_sandbox binaries must have the same revision. -REPLICA_X86_64_DARWIN_SHA256=4f4103f9bef8eeffabdafb512000a3e4c81bedbb2f36008c547cc81593d2bb1a -REPLICA_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/replica.gz -REPLICA_X86_64_LINUX_SHA256=47c09f69a80866cb7c8b7a205077da98b1729c8c0f17d4df7ab249faff60e1b0 -REPLICA_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz -# The replica and canister_sandbox binaries must have the same revision. -CANISTER_SANDBOX_X86_64_DARWIN_SHA256=2386cb035a2afefe4ff6cf944b8ddd6ca4acfbd1b2be4bf4a0f328e38d93a51d -CANISTER_SANDBOX_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz -CANISTER_SANDBOX_X86_64_LINUX_SHA256=797eaa7abab1ae0b9817a514169885101e2964f64dae4c24e839a60237e5ddef -CANISTER_SANDBOX_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/canister_sandbox.gz -SANDBOX_LAUNCHER_X86_64_DARWIN_SHA256=a5f1cb3dc57be908fe5bd880a42160f4b13f62b9c3acc5baef9e7a2518b53ad1 -SANDBOX_LAUNCHER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz -SANDBOX_LAUNCHER_X86_64_LINUX_SHA256=9fb47f54a2690b0a5de3c5908f09282323548181c36d20f55e2fd816f72994ba -SANDBOX_LAUNCHER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/sandbox_launcher.gz -SNS_X86_64_DARWIN_SHA256=0402538652ba6296ac60e4fef255691226ae20cbd5d20297f585369565d1f035 -SNS_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/sns.gz -SNS_X86_64_LINUX_SHA256=2f04439436171ba96912ac7967fcd3a275d68b71267f07f221c7381f54fb0ef2 -SNS_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/sns.gz -MOTOKO_BASE_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz -MOTOKO_BASE_SHA256=efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1 -REPLICA_REV=3e1be1316341811db5c9300935c4236bfab8fa2a diff --git a/scripts/prepare-dfx-assets.sh b/scripts/prepare-dfx-assets.sh deleted file mode 100755 index b2f748d41f..0000000000 --- a/scripts/prepare-dfx-assets.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -SDK_ROOT_DIR="$( cd -- "$(dirname -- "$( dirname -- "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )" - -# shellcheck disable=SC1090 -source "$SDK_ROOT_DIR/scripts/dfx-asset-sources.sh" - -DFX_ASSETS_FINAL_DIR=${1?'Must specify a destination directory.'} - -DFX_ASSETS_TEMP_DIR=$(mktemp -d) -BINARY_CACHE_TEMP_DIR=$(mktemp -d) -DOWNLOAD_TEMP_DIR=$(mktemp -d) - -function cleanup { - rm -rf "$DFX_ASSETS_TEMP_DIR" "$BINARY_CACHE_TEMP_DIR" "$DOWNLOAD_TEMP_DIR" -} -trap cleanup EXIT - -# We use x86_64 even on Apple M1 (arm64), through rosetta -MACHINE=x86_64 -case "$OSTYPE" in - darwin*) PLATFORM="darwin" ;; - linux*) PLATFORM="linux" ;; - *) echo "Unsupported OS type: $OSTYPE" ; exit 1;; -esac - -add_canisters() { - tar -czf "$DFX_ASSETS_TEMP_DIR"/assetstorage_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./assetstorage.did ./assetstorage.wasm.gz - tar -czf "$DFX_ASSETS_TEMP_DIR"/wallet_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./wallet.did ./wallet.wasm - tar -czf "$DFX_ASSETS_TEMP_DIR"/ui_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./ui.did ./ui.wasm -} - -download_url_and_check_sha() { - URL="$1" - EXPECTED_SHA256="$2" - LOCAL_PATH="$3" - - echo "Downloading $URL with expected sha256=$EXPECTED_SHA256 to $LOCAL_PATH" - - curl --fail --location --output "$LOCAL_PATH" "$URL" - - ACTUAL_SHA256=$(shasum -a 256 "$LOCAL_PATH" | cut -f 1 -d ' ') - - if [ "$EXPECTED_SHA256" != "$ACTUAL_SHA256" ]; then - echo "SHA256 mismatch for $URL: expected $EXPECTED_SHA256, got $ACTUAL_SHA256" - exit 1 - fi -} - -get_variable() { - NAME="$1" - PART="$2" - - VAR_NAME=$(echo "${NAME}_${MACHINE}_${PLATFORM}_${PART}" | tr '[:lower:]-' '[:upper:]_') - VAR_VALUE=${!VAR_NAME} - - echo "$VAR_VALUE" -} - -download_binary() { - NAME="$1" - SHA256=$(get_variable "$NAME" "SHA256") - URL=$(get_variable "$NAME" "URL") - - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/$NAME.gz" - BINARY_CACHE_PATH="$BINARY_CACHE_TEMP_DIR/$NAME" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - gunzip -c "$DOWNLOAD_PATH" >"$BINARY_CACHE_PATH" - chmod 0500 "$BINARY_CACHE_PATH" -} - -download_tarball() { - NAME="$1" - - SHA256=$(get_variable "$NAME" "SHA256") - URL=$(get_variable "$NAME" "URL") - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/$NAME.tar.gz" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - # -k: some archives contain r-x ".", and on linux the default behavior is to overwrite the - # metadata. We only want to extract new files anyway. - tar -xkvf "$DOWNLOAD_PATH" -C "$BINARY_CACHE_TEMP_DIR" -} - -download_ic_ref() { - download_tarball "ic-ref" - chmod 0500 "$BINARY_CACHE_TEMP_DIR/ic-ref" -} - -download_icx_proxy() { - download_tarball "icx-proxy" - - chmod 0500 "$BINARY_CACHE_TEMP_DIR/icx-proxy" -} - -download_motoko_binaries() { - download_tarball "motoko" - - for a in mo-doc mo-ide moc; - do - chmod 0500 "$BINARY_CACHE_TEMP_DIR/$a" - done -} - -download_motoko_base() { - URL="$MOTOKO_BASE_URL" - SHA256="$MOTOKO_BASE_SHA256" - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/motoko-base-tarball.tar.gz" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - mkdir "$DOWNLOAD_TEMP_DIR/motoko-base" - tar -xkvf "$DOWNLOAD_PATH" -C "$DOWNLOAD_TEMP_DIR/motoko-base" - - cp -R "$DOWNLOAD_TEMP_DIR/motoko-base/src/" "$BINARY_CACHE_TEMP_DIR/base" - chmod 0755 "$BINARY_CACHE_TEMP_DIR/base" - find "$BINARY_CACHE_TEMP_DIR/base" -type f -exec touch {} \; -exec chmod 0644 {} \; - - chmod -R 0744 "$DOWNLOAD_TEMP_DIR/motoko-base" - rm -rf "$DOWNLOAD_TEMP_DIR/motoko-base" -} - -add_binary_cache() { - download_binary "ic-admin" - download_binary "ic-btc-adapter" - download_binary "ic-canister-http-adapter" - download_binary "ic-nns-init" - download_binary "replica" - download_binary "canister_sandbox" - download_binary "sandbox_launcher" - download_binary "ic-starter" - download_binary "sns" - download_ic_ref - download_icx_proxy - download_motoko_binaries - download_motoko_base - - tar -czf "$DFX_ASSETS_TEMP_DIR"/binary_cache.tgz -C "$BINARY_CACHE_TEMP_DIR" . -} - -echo "Building $DFX_ASSETS_FINAL_DIR" - -add_canisters -add_binary_cache - -if [ -d "$DFX_ASSETS_FINAL_DIR" ] -then - ( - cd "$DFX_ASSETS_FINAL_DIR" - rm -f binary_cache.tgz assetstorage_canister.tgz wallet_canister.tgz ui_canister.tgz - ) - rmdir "$DFX_ASSETS_FINAL_DIR" -fi -mv "$DFX_ASSETS_TEMP_DIR" "$DFX_ASSETS_FINAL_DIR" - -echo "Built $DFX_ASSETS_FINAL_DIR" diff --git a/scripts/update-replica.sh b/scripts/update-replica.sh index 5367094ddd..d13d2f87a5 100755 --- a/scripts/update-replica.sh +++ b/scripts/update-replica.sh @@ -22,10 +22,10 @@ niv update ic-starter-x86_64-darwin -a rev=$SHA niv update ic-starter-x86_64-linux -a rev=$SHA niv update replica-x86_64-darwin -a rev=$SHA niv update replica-x86_64-linux -a rev=$SHA -niv update canister-sandbox-x86_64-darwin -a rev=$SHA -niv update canister-sandbox-x86_64-linux -a rev=$SHA -niv update sandbox-launcher-x86_64-darwin -a rev=$SHA -niv update sandbox-launcher-x86_64-linux -a rev=$SHA +niv update canister_sandbox-x86_64-darwin -a rev=$SHA +niv update canister_sandbox-x86_64-linux -a rev=$SHA +niv update sandbox_launcher-x86_64-darwin -a rev=$SHA +niv update sandbox_launcher-x86_64-linux -a rev=$SHA niv update sns-x86_64-darwin -a rev=$SHA niv update sns-x86_64-linux -a rev=$SHA diff --git a/scripts/write-dfx-asset-sources.sh b/scripts/write-dfx-asset-sources.sh index abd897d144..0bfd8434ae 100755 --- a/scripts/write-dfx-asset-sources.sh +++ b/scripts/write-dfx-asset-sources.sh @@ -16,7 +16,7 @@ which curl >/dev/null || ( echo "Please install curl in order to run this script SDK_ROOT_DIR="$( cd -- "$(dirname -- "$( dirname -- "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )" -DFX_ASSET_SOURCES="$SDK_ROOT_DIR/scripts/dfx-asset-sources.sh" +DFX_ASSET_SOURCES="$SDK_ROOT_DIR/src/dfx/assets/dfx-asset-sources.toml" NIX_SOURCES_JSON="$SDK_ROOT_DIR/nix/sources.json" read_sha256_from_nix_sources() { @@ -39,17 +39,16 @@ read_rev_from_nix_sources() { jq -r .'"'"$KEY"'".rev' "$NIX_SOURCES_JSON" } -normalize_varname() { - echo "$1" | tr '[:lower:]-' '[:upper:]_' -} - -write_sha256() { - KEY="$1" - SHA256=$(read_sha256_from_nix_sources "$KEY") - - NAME=$(normalize_varname "${KEY}_SHA256") - - echo "$NAME=$SHA256" >>"$DFX_ASSET_SOURCES" +write_entry() { + NAME="$1" + PLATFORM="$2" + KEY="${3:-"${NAME}-${PLATFORM}"}" + URL=$(read_url_from_nix_sources "$KEY") + SHA256="${4:-"$(read_sha256_from_nix_sources "$KEY")"}" + cat >>"$DFX_ASSET_SOURCES" <<<" +[${PLATFORM}.${NAME}] +url = '${URL}' +sha256 = '${SHA256}'" } calculate_sha256() { @@ -73,50 +72,27 @@ calculate_sha256() { echo "SHA256 mismatch for $URL: expected $EXPECTED_BASE32_SHA256, got $ACTUAL_BASE32_SHA256" exit 1 fi - - NAME=$(normalize_varname "${KEY}_SHA256") - - echo "$NAME=$SHA256" >>"$DFX_ASSET_SOURCES" - -} - -write_url() { - KEY="$1" - URL=$(read_url_from_nix_sources "$KEY") - - NAME=$(normalize_varname "${KEY}_URL") - - echo "$NAME=$URL" >>"$DFX_ASSET_SOURCES" + echo "$SHA256" } write_replica_rev() { REV=$(read_rev_from_nix_sources "replica-x86_64-darwin") - echo "REPLICA_REV=$REV" >>"$DFX_ASSET_SOURCES" -} - -write_var() { - VALUE=$(jq -r .'"'"$1"'"."'"$2"'"' "$NIX_SOURCES_JSON") - NAME=$(normalize_varname "${1}_${2}") - echo "$NAME=$VALUE" >>"$DFX_ASSET_SOURCES" + echo "replica-rev = '$REV'" >>"$DFX_ASSET_SOURCES" } echo "# generated by write-dfx-asset-sources.sh" >"$DFX_ASSET_SOURCES" -for name in "ic-ref" "icx-proxy" "ic-admin" "ic-btc-adapter" "ic-canister-http-adapter" "ic-nns-init" "ic-starter" "motoko" "replica" "canister-sandbox" "sandbox-launcher" "sns"; +write_replica_rev +motoko_base_sha="$(calculate_sha256 "motoko-base")" +for platform in "darwin" "linux"; do - if [[ "$name" == "replica" || "$name" == "canister-sandbox" ]]; then - echo "# The replica and canister_sandbox binaries must have the same revision." >>"$DFX_ASSET_SOURCES" - fi - for platform in "darwin" "linux"; + for name in "ic-ref" "icx-proxy" "ic-admin" "ic-btc-adapter" "ic-canister-http-adapter" "ic-nns-init" "ic-starter" "motoko" "replica" "canister_sandbox" "sandbox_launcher" "sns"; do - write_sha256 "${name}-x86_64-${platform}" - write_url "${name}-x86_64-${platform}" + if [[ "$name" == "replica" || "$name" == "canister_sandbox" ]]; then + echo "# The replica and canister_sandbox binaries must have the same revision." >>"$DFX_ASSET_SOURCES" + fi + write_entry "$name" "x86_64-${platform}" done + write_entry "motoko-base" "x86_64-${platform}" "motoko-base" "$motoko_base_sha" done - -write_url "motoko-base" -calculate_sha256 "motoko-base" - -write_replica_rev - diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index abdd4d2f4f..5422371c53 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -10,10 +10,15 @@ name = "dfx" path = "src/main.rs" [build-dependencies] +bytes = "1" flate2 = "1.0.11" -hex = "0.4.2" +hex = "0.4.3" +reqwest = "0.11.9" +serde = { version = "1.0", features = ["derive"] } sha2 = "0.10.6" tar = "0.4.26" +tokio = { version = "1.17.0", features = ["full"] } +toml = "0.5.9" walkdir = "2.3.2" [dependencies] diff --git a/src/dfx/assets/README.adoc b/src/dfx/assets/README.adoc index 6336d14a96..ddc45435a9 100644 --- a/src/dfx/assets/README.adoc +++ b/src/dfx/assets/README.adoc @@ -1,11 +1,11 @@ = Assets -The `files/` directory contains all the files of a new project. This is tar gzipped at build time +The `new_project_*/` directories contain all the files of a new project. These are tar gzipped at build time and injected into the binary. The following strings are replaced: -- `{project_name}` => the project name. +- `{project_name}` / `+__project_name__+` => the project name. - `{dfx_version}` => the DFX version used to create the project. Also, files that start with `+++__dot__+++` will be replaced with `.`. diff --git a/src/dfx/assets/build.rs b/src/dfx/assets/build.rs index 0702dbb611..d06c31c4e1 100644 --- a/src/dfx/assets/build.rs +++ b/src/dfx/assets/build.rs @@ -1,17 +1,21 @@ use flate2::write::GzEncoder; use flate2::Compression; +use serde::Deserialize; use sha2::{Digest, Sha256}; +use std::collections::HashMap; use std::fs::{read_to_string, File}; -use std::io::{BufRead, Read, Write}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs, io}; +use std::{env, fs}; use walkdir::WalkDir; +mod prepare_assets; + const INPUTS: &[&str] = &[ "nix/sources.json", - "scripts/dfx-asset-sources.sh", - "scripts/prepare-dfx-assets.sh", + "src/dfx/assets/prepare_assets.rs", + "src/dfx/assets/build.rs", "src/distributed/assetstorage.did", "src/distributed/assetstorage.wasm.gz", "src/distributed/ui.did", @@ -42,13 +46,34 @@ fn get_project_root_path() -> PathBuf { .expect("Unable to determine project root") } -fn find_assets() -> PathBuf { +#[derive(Deserialize, Clone)] +struct Source { + url: String, + sha256: String, +} + +impl Source { + fn sha256(&self) -> Vec { + hex::decode(&self.sha256).expect("Invalid SHA-256") + } +} + +#[derive(Deserialize)] +struct Sources { + #[serde(rename = "x86_64-linux")] + x86_64_linux: HashMap, + #[serde(rename = "x86_64-darwin")] + x86_64_darwin: HashMap, + #[serde(rename = "replica-rev")] + replica_rev: String, +} + +fn find_assets(sources: Sources) -> PathBuf { println!("cargo:rerun-if-env-changed=DFX_ASSETS"); if let Ok(a) = env::var("DFX_ASSETS") { PathBuf::from(a) } else { let project_root_path = get_project_root_path(); - let prepare_script_path = project_root_path.join("scripts/prepare-dfx-assets.sh"); for input in INPUTS { println!( "cargo:rerun-if-changed={}", @@ -69,18 +94,15 @@ fn find_assets() -> PathBuf { } } - let result = Command::new(&prepare_script_path) - .arg(&dfx_assets_path) - .output() - .expect("unable to run prepare script"); - if !result.status.success() { - println!( - "cargo:error=unable to run {}:", - prepare_script_path.to_string_lossy() - ); - println!("cargo:error={}", String::from_utf8_lossy(&result.stderr)); - std::process::exit(1) - } + let source_set = match ( + &*env::var("CARGO_CFG_TARGET_ARCH").unwrap(), + &*env::var("CARGO_CFG_TARGET_OS").unwrap(), + ) { + ("x86_64" | "aarch64", "macos") => sources.x86_64_darwin, // rosetta + ("x86_64", "linux" | "windows") => sources.x86_64_linux, + (arch, os) => panic!("Unsupported OS type {arch}-{os}"), + }; + prepare_assets::prepare(&dfx_assets_path, source_set); fs::write(last_hash_of_inputs_path, hash_of_inputs) .expect("unable to write last hash of inputs"); @@ -128,11 +150,8 @@ fn write_archive_accessor(fn_name: &str, f: &mut File) { f.write_all( format!( " - pub fn {fn_name}() -> Result>>>> {{ - let mut v = Vec::new(); - v.extend_from_slice(include_bytes!(\"{fn_name}.tgz\")); - - let tar = GzDecoder::new(std::io::Cursor::new(v)); + pub fn {fn_name}() -> Result>> {{ + let tar = GzDecoder::new(&include_bytes!(\"{fn_name}.tgz\")[..]); let archive = Archive::new(tar); Ok(archive) }} @@ -197,7 +216,7 @@ fn get_git_hash() -> Result { )) } -fn add_assets() { +fn add_assets(sources: Sources) { let out_dir = env::var("OUT_DIR").unwrap(); let loader_path = Path::new(&out_dir).join("load_assets.rs"); let mut f = File::create(&loader_path).unwrap(); @@ -205,7 +224,7 @@ fn add_assets() { f.write_all( b" use flate2::read::GzDecoder; - use std::io::{Cursor, Result}; + use std::io::Result; use std::vec::Vec; use tar::Archive; @@ -213,7 +232,7 @@ fn add_assets() { ) .unwrap(); - let dfx_assets = find_assets(); + let dfx_assets = find_assets(sources); add_asset_archive("binary_cache", &mut f, &dfx_assets); add_asset_archive("assetstorage_canister", &mut f, &dfx_assets); add_asset_archive("wallet_canister", &mut f, &dfx_assets); @@ -253,25 +272,16 @@ fn define_dfx_version() { } } -fn define_replica_rev() { - let pathname = get_project_root_path().join("scripts/dfx-asset-sources.sh"); - let file = File::open(pathname).expect("Unable to read scripts/dfx-asset-sources.sh"); - let reader = io::BufReader::new(file); - - let prefix = "REPLICA_REV="; - - let replica_rev_line = reader - .lines() - .map(|line| line.expect("Could not parse line")) - .find(|line| line.starts_with(prefix)) - .expect("No REPLICA_REV in scripts/dfx-asset-sources.sh"); - let replica_rev = &replica_rev_line[prefix.len()..]; - +fn define_replica_rev(replica_rev: &str) { println!("cargo:rustc-env=DFX_ASSET_REPLICA_REV={}", replica_rev); } fn main() { - add_assets(); + let sources: Sources = toml::from_slice( + &fs::read("assets/dfx-asset-sources.toml").expect("unable to read dfx-asset-sources.toml"), + ) + .expect("unable to parse dfx-asset-sources.toml"); + define_replica_rev(&sources.replica_rev); + add_assets(sources); define_dfx_version(); - define_replica_rev(); } diff --git a/src/dfx/assets/dfx-asset-sources.toml b/src/dfx/assets/dfx-asset-sources.toml new file mode 100644 index 0000000000..7294342489 --- /dev/null +++ b/src/dfx/assets/dfx-asset-sources.toml @@ -0,0 +1,110 @@ +# generated by write-dfx-asset-sources.sh +replica-rev = '3e1be1316341811db5c9300935c4236bfab8fa2a' + +[x86_64-darwin.ic-ref] +url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-darwin.tar.gz' +sha256 = '07fb2cf2a570d6d90259ce165e6d5b9bbc408508743291be7890d8cb01c8a178' + +[x86_64-darwin.icx-proxy] +url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz' +sha256 = '5783bba5021cf43149bc118789cea29f6462fd97dd78bdb776f8782ea7813d27' + +[x86_64-darwin.ic-admin] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-admin.gz' +sha256 = '303fe743a47180d1a196f7c4cf6acc9d17530768fd1417af30f615a2f92ccd3e' + +[x86_64-darwin.ic-btc-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-btc-adapter.gz' +sha256 = 'b1e26a1b242b29ab61346d8a5856199d735d75ef6aa3ab5af94de250a4797164' + +[x86_64-darwin.ic-canister-http-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-canister-http-adapter.gz' +sha256 = 'd9dc8a228b3507f0e1df73781d9738eecd216cbfda7a1f5cb063cbf2a2a7b54b' + +[x86_64-darwin.ic-nns-init] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-nns-init.gz' +sha256 = '4086229540cced5742acb8b9885a841e7c47a8e3d7b59a393034d79f7cb6d2a4' + +[x86_64-darwin.ic-starter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-starter.gz' +sha256 = 'b3b5dcc554b9419f5929cce069e84d8d5070661d8a1a60b278c39272f5916c4e' + +[x86_64-darwin.motoko] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-macos-0.7.1.tar.gz' +sha256 = '0fd98250ce7f2c6dbbaa9ccae141ad60d9542bd4e915e0fb4f82a235db666c88' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-darwin.replica] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/replica.gz' +sha256 = '4f4103f9bef8eeffabdafb512000a3e4c81bedbb2f36008c547cc81593d2bb1a' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-darwin.canister_sandbox] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz' +sha256 = '2386cb035a2afefe4ff6cf944b8ddd6ca4acfbd1b2be4bf4a0f328e38d93a51d' + +[x86_64-darwin.sandbox_launcher] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz' +sha256 = 'a5f1cb3dc57be908fe5bd880a42160f4b13f62b9c3acc5baef9e7a2518b53ad1' + +[x86_64-darwin.sns] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/sns.gz' +sha256 = '0402538652ba6296ac60e4fef255691226ae20cbd5d20297f585369565d1f035' + +[x86_64-darwin.motoko-base] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz' +sha256 = 'efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1' + +[x86_64-linux.ic-ref] +url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-linux.tar.gz' +sha256 = 'd6f66f45b3fc904bea7810d3795b7761115b4635c3846991c4516ecaa42e2d36' + +[x86_64-linux.icx-proxy] +url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz' +sha256 = '7a5612a1fb7512d22dcd37627a9d626fbc282b172665a832fe2cc2b243789fa1' + +[x86_64-linux.ic-admin] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-admin.gz' +sha256 = 'c201463f2498dbf980fb3b1fb5ec75f46e7e8464d7530c0423d889ead09de44e' + +[x86_64-linux.ic-btc-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-btc-adapter.gz' +sha256 = '26941f1fb7897faa72239871a27a18c1bccdd538fef13911b028f7568ef28c71' + +[x86_64-linux.ic-canister-http-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-canister-http-adapter.gz' +sha256 = '610a565ec150be9cb29f69b99cecf56f25e131ca740bf41c5b309be140fa0da8' + +[x86_64-linux.ic-nns-init] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-nns-init.gz' +sha256 = '71cacac1053c07e4b607948662674c564e6f134b68f5febe49a8e421b7ab2768' + +[x86_64-linux.ic-starter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-starter.gz' +sha256 = '0bee932157078e868649365485755e9055391a6dcb05e369f617a95c05c461e6' + +[x86_64-linux.motoko] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-linux64-0.7.1.tar.gz' +sha256 = '132b13907425d3a7bccf10f48a420140244e663f0b48b200908c295a71a3a3c7' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-linux.replica] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz' +sha256 = '47c09f69a80866cb7c8b7a205077da98b1729c8c0f17d4df7ab249faff60e1b0' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-linux.canister_sandbox] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/canister_sandbox.gz' +sha256 = '797eaa7abab1ae0b9817a514169885101e2964f64dae4c24e839a60237e5ddef' + +[x86_64-linux.sandbox_launcher] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/sandbox_launcher.gz' +sha256 = '9fb47f54a2690b0a5de3c5908f09282323548181c36d20f55e2fd816f72994ba' + +[x86_64-linux.sns] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/sns.gz' +sha256 = '2f04439436171ba96912ac7967fcd3a275d68b71267f07f221c7381f54fb0ef2' + +[x86_64-linux.motoko-base] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz' +sha256 = 'efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1' diff --git a/src/dfx/assets/prepare_assets.rs b/src/dfx/assets/prepare_assets.rs new file mode 100644 index 0000000000..bd33f844f7 --- /dev/null +++ b/src/dfx/assets/prepare_assets.rs @@ -0,0 +1,221 @@ +use std::{ + collections::HashMap, + fs::File, + io::{self, BufWriter}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use flate2::{bufread::GzDecoder, write::GzEncoder, Compression}; +use reqwest::Client; +use sha2::{Digest, Sha256}; +use tar::{Archive, Builder, EntryType, Header}; +use tokio::task::{spawn, spawn_blocking, JoinSet}; + +use crate::Source; + +#[tokio::main] +pub(crate) async fn prepare(out_dir: &Path, source_set: HashMap) { + std::fs::create_dir_all(out_dir).expect("error creating output directory"); + let out_dir_ = out_dir.to_owned(); + let copy_join = spawn_blocking(|| copy_canisters(out_dir_)); + let out_dir = out_dir.to_owned(); + make_binary_cache(out_dir, source_set).await; + copy_join.await.unwrap(); +} + +fn copy_canisters(out_dir: PathBuf) { + let distributed = Path::new("../distributed"); + for can in ["assetstorage", "wallet", "ui"] { + let mut tar = Builder::new(GzEncoder::new( + BufWriter::new(File::create(out_dir.join(format!("{can}_canister.tgz"))).unwrap()), + Compression::new(6), + )); + for ext in [ + ".did", + if can == "assetstorage" { + ".wasm.gz" + } else { + ".wasm" + }, + ] { + let filename = format!("{can}{ext}"); + let input_file = File::open(distributed.join(&filename)).unwrap(); + let metadata = input_file.metadata().unwrap(); + let mut header = Header::new_gnu(); + header.set_mode(0o644); + header.set_size(metadata.len()); + tar.append_data(&mut header, &filename, input_file).unwrap(); + } + tar.finish().unwrap(); + } +} + +async fn make_binary_cache(out_dir: PathBuf, sources: HashMap) { + let sources = Arc::new(sources); + let client = Client::builder() + .timeout(Duration::from_secs(300)) + .build() + .unwrap(); + let mo_base = spawn(download_mo_base(client.clone(), sources.clone())); + let bins = spawn(download_binaries(client.clone(), sources.clone())); + let bin_tars = spawn(download_bin_tarballs(client.clone(), sources.clone())); + let (mo_base, bins, bin_tars) = tokio::try_join!(mo_base, bins, bin_tars).unwrap(); + spawn_blocking(|| write_binary_cache(out_dir, mo_base, bins, bin_tars)) + .await + .unwrap(); +} + +fn write_binary_cache( + out_dir: PathBuf, + mo_base: HashMap, + bins: HashMap, + mut bin_tars: HashMap, +) { + let mut tar = Builder::new(GzEncoder::new( + BufWriter::new(File::create(out_dir.join("binary_cache.tgz")).unwrap()), + Compression::new(6), + )); + for (path, bin) in bins.into_iter().chain( + ["icx-proxy", "ic-ref", "moc", "mo-doc", "mo-ide"] + .map(|bin| (bin.into(), bin_tars.remove(Path::new(bin)).unwrap())), + ) { + let mut header = Header::new_gnu(); + header.set_size(bin.len() as u64); + header.set_mode(0o500); + tar.append_data(&mut header, path, bin.reader()).unwrap(); + } + + for (path, file) in bin_tars { + let mut header = Header::new_gnu(); + header.set_size(file.len() as u64); + header.set_mode(0o644); + tar.append_data(&mut header, path, file.reader()).unwrap(); + } + let mut base_hdr = Header::new_gnu(); + base_hdr.set_entry_type(EntryType::dir()); + base_hdr.set_mode(0o755); + base_hdr.set_size(0); + tar.append_data(&mut base_hdr, "base", io::empty()).unwrap(); + for (path, file) in mo_base { + let mut header = Header::new_gnu(); + header.set_mode(0o644); + header.set_size(file.len() as u64); + tar.append_data(&mut header, Path::new("base").join(path), file.reader()) + .unwrap(); + } + tar.finish().unwrap(); +} + +async fn download_and_check_sha(client: Client, source: Source) -> Bytes { + let response = client.get(&source.url).send().await.unwrap(); + response.error_for_status_ref().unwrap(); + let content = response.bytes().await.unwrap(); + let sha = Sha256::digest(&content); + assert_eq!( + sha[..], + source.sha256()[..], + "sha256 hash for {} did not match", + source.url + ); + content +} + +async fn download_binaries( + client: Client, + sources: Arc>, +) -> HashMap { + let mut joinset = JoinSet::new(); + for bin in [ + "ic-admin", + "ic-btc-adapter", + "ic-canister-http-adapter", + "ic-nns-init", + "replica", + "canister_sandbox", + "sandbox_launcher", + "ic-starter", + "sns", + ] { + let source = sources + .get(bin) + .unwrap_or_else(|| panic!("Cannot find source for {bin}")) + .clone(); + let client_ = client.clone(); + joinset.spawn(async move { (bin, download_and_check_sha(client_, source).await) }); + } + let mut map = HashMap::new(); + while let Some(res) = joinset.join_next().await { + let (bin, content) = res.unwrap(); + let decompressed = spawn_blocking(|| { + let mut buf = BytesMut::new(); + io::copy( + &mut GzDecoder::new(content.reader()), + &mut (&mut buf).writer(), + ) + .unwrap(); + buf.freeze() + }) + .await + .unwrap(); + map.insert(bin.into(), decompressed); + } + map +} + +async fn download_bin_tarballs( + client: Client, + sources: Arc>, +) -> HashMap { + let mut map = HashMap::new(); + let [motoko, icx_proxy, ic_ref] = ["motoko", "icx-proxy", "ic-ref"].map(|pkg| { + let client = client.clone(); + let source = sources[pkg].clone(); + spawn(download_and_check_sha(client, source)) + }); + let (motoko, icx_proxy, ic_ref) = tokio::try_join!(motoko, icx_proxy, ic_ref).unwrap(); + for tar in [motoko, icx_proxy, ic_ref] { + tar_xzf(&tar, |path, content| { + map.insert(path, content); + }); + } + map +} + +async fn download_mo_base( + client: Client, + sources: Arc>, +) -> HashMap { + let source = sources["motoko-base"].clone(); + let mo_base = download_and_check_sha(client, source).await; + let mut map = HashMap::new(); + tar_xzf(&mo_base, |path, content| { + let path = path.strip_prefix(".").unwrap_or(&path); // normalize ./x to x + if let Ok(file) = path.strip_prefix("src") { + map.insert(file.to_owned(), content); + } + }); + map +} + +fn tar_xzf(gz: &[u8], mut each: impl FnMut(PathBuf, Bytes)) { + let mut tar = Archive::new(GzDecoder::new(gz)); + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry.path().unwrap_or_else(|e| { + panic!( + "Malformed file path {}: {e}", + String::from_utf8_lossy(&entry.path_bytes()) + ) + }); + let path = path.strip_prefix(".").unwrap_or(&path).to_owned(); + let mut content = BytesMut::with_capacity(entry.header().size().unwrap() as usize); + io::copy(&mut entry, &mut (&mut content).writer()).unwrap(); + each(path, content.freeze()); + } +}