feat(bench): Re-enable cargo bench

.github/workflows/benchmarks.yml

@@ -0,0 +1,51 @@
+name: Run Benchmarks
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs daily at midnight
+  pull_request:
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: fetch submodules
+        run: git submodule update --init --recursive
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-wasi
+      - name: Setup build env
+        run: |
+          os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
+          ./scripts/setup-$os.sh
+        shell: bash
+      - name: Run Benchmarks
+        run: |
+          set -o pipefail
+          cargo bench -p containerd-shim-benchmarks -- --output-format bencher | tee output.txt
+      # Download previous benchmark result from cache (if exists)
+      - name: Cache benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/[email protected]
+        with:
+          name: Criterion.rs Benchmark
+          tool: 'cargo'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # my experimental local benchmarking seems to have a 20% margin of error. 
+          # So I set the alert threshold to 130% of the previous benchmark result.
+          # If the current benchmark result is more than 130% of the previous benchmark result, it will fail.
+          alert-threshold: '130%'
+          fail-on-alert: true
+          alert-comment-cc-users: '@runwasi-committers'
+          # Enable Job Summary
+          summary-always: true
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json

Cargo.toml

@@ -6,7 +6,8 @@ members = [
     "crates/oci-tar-builder",
     "crates/containerd-shim-wasmedge",
     "crates/containerd-shim-wasmtime",
-    "crates/containerd-shim-wasmer"
+    "crates/containerd-shim-wasmer",
+    "benches/containerd-shim-benchmarks",
 ]
 resolver = "2"
 
@@ -46,5 +47,9 @@ windows-sys = "0.52"
 serial_test = "2"
 tracing = "0.1"
 
+# wasmtime
+wasmtime = { version = "21.0.1", features = ["async"] }
+wasmtime-wasi = { version = "21.0.1" }
+
 [profile.release]
 panic = "abort"

Makefile

@@ -315,6 +315,10 @@ test/k3s-oci-%: dist/img-oci.tar bin/k3s dist-%
 .PHONY: test/k3s/clean
 test/k3s/clean: bin/k3s/clean;
 
+.PHONY: bench
+bench:
+	$(CARGO) bench -p containerd-shim-benchmarks
+
 .PHONY: clean
 clean:
 	-rm -rf dist

benches/containerd-shim-benchmarks/Cargo.toml

@@ -6,18 +6,18 @@ edition.workspace = true
 [dependencies]
 anyhow = { workspace = true }
 chrono = { workspace = true }
-containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm" }
-containerd-shim-wasmedge = { path = "../../crates/containerd-shim-wasmedge" }
+containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm", features = ["testing"] }
 containerd-shim-wasmtime = { path = "../../crates/containerd-shim-wasmtime" }
 libc = { workspace = true }
 oci-spec = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tempfile = "3.8"
+wasmtime = { workspace = true }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 
 [[bench]]
-name = "webassembly-benchmarks"
+name = "wasmtime-benchmarks"
 harness = false