chore: Rewrite build script in Rust (#2706)

The build script currently requires a shell script to curl all the bundled assets and pack them into tarballs. This PR rewrites the script in Rust, which not only does all the fetching in parallel but supports Windows. 

The format of the sources manifest has been changed from a shell script to a .toml file, as the latter is far more easily consumed from Rust and will help eliminate some edge cases like `motoko-base`.

Given Windows assets, DFX now builds on Windows. Linux assets are substituted as a stopgap since technically any external program can be used through WSL, but we probably want to have a proper Windows asset set eventually.

.github/workflows/lint.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         rust: [ '1.60.0' ]
-        os: [ ubuntu-latest, macos-latest ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/prepare-dfx-assets.yml

@@ -34,15 +34,17 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             target
+            !target/*/build/dfx-*/out/dfx-assets
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
       - name: Install Rust
         run: |
           rustup update ${{ matrix.rust }} --no-self-update
           rustup component add clippy
           rustup default ${{ matrix.rust }}
 
-      - name: Prepare dfx assets
-        run: scripts/prepare-dfx-assets.sh "$(mktemp -d)"
+      - name: Run build script
+        run: |
+          cargo check
 
   aggregate:
     name: prepare-dfx-assets:required

.github/workflows/shellcheck.yml

@@ -36,5 +36,4 @@ jobs:
         run: $HOME/bin/shellcheck-v0.7.1/shellcheck e2e/**/*.*sh
       - name: Check release script
         run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/release.sh
-      - name: Check asset prep script
-        run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/prepare-dfx-assets.sh
+

Cargo.toml

@@ -30,6 +30,9 @@ lto = true
 [profile.dev.package.argon2]
 opt-level = 3
 
+[profile.dev.package.miniz_oxide]
+opt-level = 3
+
 [profile.release.package.ic-frontend-canister]
 opt-level = 'z'
 

nix/sources.json

@@ -12,18 +12,18 @@
         "url": "https://github.com/RustSec/advisory-db/archive/fe7b79e8ebf9ad03660e573e0c2a1ed4e80dee5d.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "canister-sandbox-x86_64-darwin": {
+    "canister_sandbox-x86_64-darwin": {
         "builtin": false,
-        "description": "The canister-sandbox binary. It must be updated together with the replica binary.",
+        "description": "The canister_sandbox binary. It must be updated together with the replica binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",
         "sha256": "07d5jf6y6a7kl3s4pgmjs7xsr93cvn6lp56gyr7zxziab81wp1i3",
         "type": "file",
         "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz",
         "url_template": "https://download.dfinity.systems/blessed/ic/<rev>/sdk-release/x86_64-darwin/canister_sandbox.gz"
     },
-    "canister-sandbox-x86_64-linux": {
+    "canister_sandbox-x86_64-linux": {
         "builtin": false,
-        "description": "The canister-sandbox binary. It must be updated together with the replica binary.",
+        "description": "The canister_sandbox binary. It must be updated together with the replica binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",
         "sha256": "1vyxwlvh59irx0j4rbjdyrj2j7hhhnc1c5552yc0pbmip9xalzkr",
         "type": "file",
@@ -178,7 +178,7 @@
     },
     "replica-x86_64-darwin": {
         "builtin": false,
-        "description": "The replica binary. It must be updated together with the canister-sandbox binary.",
+        "description": "The replica binary. It must be updated together with the canister_sandbox binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",
         "sha256": "06mvsa9ibj3waj600digpgnipj74lc020lgvvamzzvpqpvwh6hag",
         "type": "file",
@@ -187,14 +187,14 @@
     },
     "replica-x86_64-linux": {
         "builtin": false,
-        "description": "The replica binary. It must be updated together with the canister-sandbox binary.",
+        "description": "The replica binary. It must be updated together with the canister_sandbox binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",
         "sha256": "1c71c3zzljdjgbgx85qgijf75ccqv9vm083sidycnrh8m1lrzh27",
         "type": "file",
         "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz",
         "url_template": "https://download.dfinity.systems/blessed/ic/<rev>/sdk-release/x86_64-linux/replica.gz"
     },
-    "sandbox-launcher-x86_64-darwin": {
+    "sandbox_launcher-x86_64-darwin": {
         "builtin": false,
         "description": "The sandbox_launcher binary. It must be updated together with the replica binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",
@@ -203,7 +203,7 @@
         "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz",
         "url_template": "https://download.dfinity.systems/blessed/ic/<rev>/sdk-release/x86_64-darwin/sandbox_launcher.gz"
     },
-    "sandbox-launcher-x86_64-linux": {
+    "sandbox_launcher-x86_64-linux": {
         "builtin": false,
         "description": "The sandbox_launcher binary. It must be updated together with the replica binary.",
         "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a",

scripts/update-replica.sh

@@ -22,10 +22,10 @@ niv update ic-starter-x86_64-darwin -a rev=$SHA
 niv update ic-starter-x86_64-linux -a rev=$SHA
 niv update replica-x86_64-darwin -a rev=$SHA
 niv update replica-x86_64-linux -a rev=$SHA
-niv update canister-sandbox-x86_64-darwin -a rev=$SHA
-niv update canister-sandbox-x86_64-linux -a rev=$SHA
-niv update sandbox-launcher-x86_64-darwin -a rev=$SHA
-niv update sandbox-launcher-x86_64-linux -a rev=$SHA
+niv update canister_sandbox-x86_64-darwin -a rev=$SHA
+niv update canister_sandbox-x86_64-linux -a rev=$SHA
+niv update sandbox_launcher-x86_64-darwin -a rev=$SHA
+niv update sandbox_launcher-x86_64-linux -a rev=$SHA
 niv update sns-x86_64-darwin -a rev=$SHA
 niv update sns-x86_64-linux -a rev=$SHA