Assets stdlib

.github/workflows/test.yml

@@ -25,6 +25,7 @@ jobs:
         env:
             GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified
             PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         outputs:
             # These outputs are used to pass information along to the next job
             should_run_tests: ${{ steps.should_run_tests.outputs.should_run_tests }} # We only want the next job to run the tests once we have finished deploying to PyPI and GitHub
@@ -35,6 +36,11 @@ jobs:
               with:
                   ref: ${{ contains(github.head_ref, 'release--') && github.event.pull_request.head.ref || github.ref }} # This is necessary for this job to be able to commit and push to the origin remote properly
                   token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }} # A personal GitHub token is setup as a secret so that committing and pushing to GitHub from the Action will trigger another workflow
+            - if: contains(github.head_ref, 'release--')
+              name: Install dfx
+              run: |
+                  DFXVM_INIT_YES=true DFX_VERSION=0.19.0 sh -ci "$(curl -fsSL https://sdk.dfinity.org/install.sh)"
+                  echo "$HOME/.local/share/dfx/bin" >> $GITHUB_PATH
             # TODO we should use some Action-specific bot account
             - if: contains(github.head_ref, 'release--')
               run: git config --global user.name 'Jordan Last'
@@ -142,6 +148,7 @@ jobs:
         runs-on: ubuntu-latest
         env:
             ETHEREUM_URL: ${{ secrets.ETHEREUM_URL }}
+            KYBRA_COMPILE_RUST_PYTHON_STDLIB: ${{ matrix.kybra_source == 'repo' }}
         strategy:
             fail-fast: false # We want to see which example tests succeed and which ones fail, we don't want one example test to cancel the rest
             matrix:

kybra/__main__.py

@@ -22,7 +22,7 @@
 def main():
     args = parse_args_or_exit(sys.argv)
     paths = create_paths(args)
-    is_verbose = args["flags"]["verbose"]
+    is_verbose = args["flags"]["verbose"] or os.environ.get("KYBRA_VERBOSE") == "true"
 
     subprocess.run(
         [

kybra/build_wasm_binary_or_exit.py

@@ -1,7 +1,9 @@
+import os
 import shutil
 import subprocess
 import sys
 
+import kybra
 from kybra.colors import red
 from kybra.timed import timed_inline
 from kybra.types import Paths
@@ -11,32 +13,150 @@
 def build_wasm_binary_or_exit(
     paths: Paths, canister_name: str, cargo_env: dict[str, str], verbose: bool = False
 ):
-    compile_python_stdlib(paths, cargo_env, verbose)
+    compile_or_download_rust_python_stdlib(paths, cargo_env, verbose)
     compile_generated_rust_code(paths, canister_name, cargo_env, verbose)
     copy_wasm_to_dev_location(paths, canister_name)
     run_wasi2ic_on_wasm(paths, canister_name, cargo_env, verbose)
     generate_and_create_candid_file(paths, canister_name, cargo_env, verbose)
 
 
-def compile_python_stdlib(paths: Paths, cargo_env: dict[str, str], verbose: bool):
+def compile_or_download_rust_python_stdlib(
+    paths: Paths, cargo_env: dict[str, str], verbose: bool
+):
+    if os.environ.get("KYBRA_COMPILE_RUST_PYTHON_STDLIB") == "true":
+        compile_rust_python_stdlib(paths, cargo_env, verbose)
+    else:
+        rust_python_stdlib_global_path = (
+            f"{paths['global_kybra_version_dir']}/rust_python_stdlib"
+        )
+        download_rust_python_stdlib(
+            rust_python_stdlib_global_path, paths, cargo_env, verbose
+        )
+        copy_rust_python_stdlib_global_to_staging(rust_python_stdlib_global_path, paths)
+
+
+def compile_rust_python_stdlib(paths: Paths, cargo_env: dict[str, str], verbose: bool):
+    rust_python_global_path = f"{paths['global_kybra_version_dir']}/RustPython"
+
+    if not os.path.exists(rust_python_global_path):
+        clone_and_checkout_rust_python(paths, cargo_env, verbose)
+
+    copy_rust_python_lib_global_to_staging(rust_python_global_path, paths)
+
+    rust_python_stdlib_staging_path = f"{paths['canister']}/rust_python_stdlib"
+
+    create_rust_python_stdlib_staging_directory(rust_python_stdlib_staging_path)
+    compile_and_write_rust_python_stdlib_to_staging(
+        rust_python_stdlib_staging_path, paths, cargo_env, verbose
+    )
+
+
+def clone_and_checkout_rust_python(
+    paths: Paths, cargo_env: dict[str, str], verbose: bool
+):
+    run_subprocess(
+        ["git", "clone", "https://github.com/RustPython/RustPython.git"],
+        cargo_env,
+        verbose,
+        cwd=paths["global_kybra_version_dir"],
+    )
+
+    run_subprocess(
+        ["git", "checkout", "f12875027ce425297c07cbccb9be77514ed46157"],
+        cargo_env,
+        verbose,
+        cwd=f"{paths['global_kybra_version_dir']}/RustPython",
+    )
+
+
+def copy_rust_python_lib_global_to_staging(rust_python_global_path: str, paths: Paths):
     shutil.copytree(
-        f"{paths['global_kybra_version_dir']}/RustPython/Lib",
+        f"{rust_python_global_path}/Lib",
         f"{paths['canister']}/Lib",
     )
 
+
+def create_rust_python_stdlib_staging_directory(rust_python_stdlib_staging_path: str):
+    os.makedirs(rust_python_stdlib_staging_path)
+
+    shutil.copy(
+        os.path.dirname(kybra.__file__) + "/compiler/LICENSE-RustPython",
+        f"{rust_python_stdlib_staging_path}/LICENSE-RustPython",
+    )
+
+    shutil.copy(
+        os.path.dirname(kybra.__file__) + "/compiler/python_3_10_13_licenses.pdf",
+        f"{rust_python_stdlib_staging_path}/python_3_10_13_licenses.pdf",
+    )
+
+
+def compile_and_write_rust_python_stdlib_to_staging(
+    rust_python_stdlib_staging_path: str,
+    paths: Paths,
+    cargo_env: dict[str, str],
+    verbose: bool,
+):
     run_subprocess(
         [
             f"{paths['global_kybra_rust_bin_dir']}/cargo",
             "run",
             f"--manifest-path={paths['canister']}/kybra_compile_python_stdlib/Cargo.toml",
             f"--package=kybra_compile_python_stdlib",
-            f"{paths['canister']}/python_stdlib",
+            f"{rust_python_stdlib_staging_path}/stdlib",
         ],
         cargo_env,
         verbose,
     )
 
 
+def download_rust_python_stdlib(
+    rust_python_stdlib_global_path: str,
+    paths: Paths,
+    cargo_env: dict[str, str],
+    verbose: bool,
+):
+    if not os.path.exists(rust_python_stdlib_global_path):
+        download_rust_python_stdlib_tar_gz(paths, cargo_env, verbose)
+        extract_and_decompress_rust_python_stdlib_tar_gz(paths, cargo_env, verbose)
+
+
+def download_rust_python_stdlib_tar_gz(
+    paths: Paths, cargo_env: dict[str, str], verbose: bool
+):
+    run_subprocess(
+        [
+            "curl",
+            "-Lf",
+            f"https://github.com/demergent-labs/kybra/releases/download/{kybra.__version__}/rust_python_stdlib.tar.gz",
+            "-o",
+            "rust_python_stdlib.tar.gz",
+        ],
+        cargo_env,
+        verbose,
+        cwd=paths["global_kybra_version_dir"],
+    )
+
+
+def copy_rust_python_stdlib_global_to_staging(
+    rust_python_stdlib_global_path: str, paths: Paths
+):
+    shutil.copytree(
+        rust_python_stdlib_global_path,
+        f"{paths['canister']}/rust_python_stdlib",
+    )
+
+
+def extract_and_decompress_rust_python_stdlib_tar_gz(
+    paths: Paths, cargo_env: dict[str, str], verbose: bool
+):
+    run_subprocess(
+        ["tar", "-xvf", "rust_python_stdlib.tar.gz"],
+        cargo_env,
+        verbose,
+        cwd=paths["global_kybra_version_dir"],
+    )
+
+
 def compile_generated_rust_code(
     paths: Paths, canister_name: str, cargo_env: dict[str, str], verbose: bool
 ):
@@ -96,9 +216,13 @@ def generate_and_create_candid_file(
 
 
 def run_subprocess(
-    args: list[str], env: dict[str, str], verbose: bool, throw: bool = True
+    args: list[str],
+    env: dict[str, str],
+    verbose: bool,
+    throw: bool = True,
+    cwd: str | None = None,
 ) -> bytes:
-    result = subprocess.run(args, env=env, capture_output=not verbose)
+    result = subprocess.run(args, env=env, capture_output=not verbose, cwd=cwd)
 
     if result.returncode != 0:
         if throw == True: