From 2f11ddf51f99844f009f8e3c5406583779cbc97c Mon Sep 17 00:00:00 2001 From: Angel M De Miguel Date: Wed, 25 Jan 2023 13:43:09 +0100 Subject: [PATCH 1/4] feat: create methods to manage runtimes. Refactor common structs and modules --- Cargo.lock | 473 +++++++++++++++++++++++++++- Cargo.toml | 3 + src/fetch.rs | 26 ++ src/main.rs | 2 + src/runtimes/manager.rs | 59 ++++ src/runtimes/metadata.rs | 151 +++++++-- src/runtimes/mod.rs | 5 +- src/runtimes/modules/data.rs | 59 ---- src/runtimes/modules/javascript.rs | 15 +- src/runtimes/modules/mod.rs | 2 - src/runtimes/remote_file.rs | 24 -- src/runtimes/repository.rs | 49 --- src/runtimes/runtime.rs | 48 +-- src/store.rs | 85 +++++ src/workers/worker.rs | 2 +- tests/data/metadata/repository.toml | 6 +- tests/data/metadata/runtime.toml | 6 +- 17 files changed, 784 insertions(+), 231 deletions(-) create mode 100644 src/fetch.rs create mode 100644 src/runtimes/manager.rs delete mode 100644 src/runtimes/modules/data.rs delete mode 100644 src/runtimes/remote_file.rs delete mode 100644 src/runtimes/repository.rs create mode 100644 src/store.rs diff --git a/Cargo.lock b/Cargo.lock index 0b919336..afde3bb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash", - "base64", + "base64 0.13.1", "bitflags", "brotli", "bytes", @@ -327,6 +327,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bincode" version = "1.3.3" @@ -376,7 +382,16 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "digest", + "digest 0.10.6", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", ] [[package]] @@ -650,6 +665,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpp_demangle" version = "0.3.5" @@ -856,13 +887,22 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer", + "block-buffer 0.10.3", "crypto-common", "subtle", ] @@ -976,6 +1016,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + [[package]] name = "fd-lock" version = "3.0.8" @@ -1013,6 +1062,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1033,6 +1097,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.25" @@ -1162,6 +1235,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.8" @@ -1173,6 +1252,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "http-range" version = "0.1.5" @@ -1197,6 +1287,43 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.3.0" @@ -1224,6 +1351,15 @@ dependencies = [ "serde", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "io-extras" version = "0.17.1" @@ -1312,6 +1448,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72167d68f5fce3b8655487b8038691a3c9984ee769590f93f2a631f4ad64e4f5" +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1486,6 +1631,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.2" @@ -1537,6 +1700,57 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -1800,6 +2014,52 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -1852,12 +2112,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645926f31b250a2dca3c232496c2d898d91036e45ca0e97e0e2390c54e11be36" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.16" @@ -1924,7 +2216,20 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.6", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", ] [[package]] @@ -1935,7 +2240,17 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.6", +] + +[[package]] +name = "sha256" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e334db67871c14c18fc066ad14af13f9fdf5f9a91c61af432d1e3a39c8c6a141" +dependencies = [ + "hex", + "sha2 0.9.9", ] [[package]] @@ -2050,6 +2365,20 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -2148,6 +2477,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -2171,6 +2510,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2204,6 +2549,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -2263,6 +2614,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -2286,6 +2643,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2335,6 +2702,72 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + [[package]] name = "wasm-encoder" version = "0.20.0" @@ -2357,7 +2790,7 @@ name = "wasm-workers-rs" version = "0.6.0" dependencies = [ "anyhow", - "base64", + "base64 0.13.1", "http", "serde", "serde_json", @@ -2371,15 +2804,18 @@ dependencies = [ "actix-files", "actix-web", "anyhow", - "base64", + "base64 0.13.1", "blake3", "clap 4.0.32", "env_logger 0.9.3", "lazy_static", "regex", + "reqwest", "serde", "serde_json", + "sha256", "toml", + "url", "wasi-common", "wasmtime", "wasmtime-wasi", @@ -2443,14 +2879,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1e77abcf538af42517e188c109e4b50ecf6c0ee4d77ede76a438e0306b934dc" dependencies = [ "anyhow", - "base64", + "base64 0.13.1", "bincode", "directories-next", "file-per-thread-logger", "log", "rustix", "serde", - "sha2", + "sha2 0.10.6", "toml", "windows-sys", "zstd", @@ -2654,6 +3090,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.3.0" @@ -2795,6 +3241,15 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "winx" version = "0.34.0" diff --git a/Cargo.toml b/Cargo.toml index b6c56eb8..2143af35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,9 @@ toml = "0.5.9" base64 = "0.13.1" clap = { version = "4.0.10", features = ["derive"] } regex = "1" +url = "2.3.1" +reqwest = { version = "0.11" } +sha256 = "1.1.1" [workspace] members = [ diff --git a/src/fetch.rs b/src/fetch.rs new file mode 100644 index 00000000..13c2a851 --- /dev/null +++ b/src/fetch.rs @@ -0,0 +1,26 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::runtimes::metadata::Checksum; +use anyhow::Result; + +// TODO: Remove it when implementing the manager +#[allow(dead_code)] +/// Fetch the contents of a given file and validates it +/// using the Sha256. +pub async fn fetch>(file: T) -> Result> { + let body: Vec = reqwest::get(file.as_ref()).await?.bytes().await?.into(); + + Ok(body) +} + +// TODO: Remove it when implementing the manager +#[allow(dead_code)] +/// Fetch the contents of a given file and validates it +/// using the Sha256. +pub async fn fetch_and_validate>(file: T, checksum: &Checksum) -> Result> { + let body: Vec = fetch(file).await?; + checksum.validate(&body)?; + + Ok(body) +} diff --git a/src/main.rs b/src/main.rs index 52d8d67a..4cba4b3a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,8 +6,10 @@ extern crate lazy_static; mod config; mod data; +mod fetch; mod router; mod runtimes; +mod store; mod workers; use actix_files::{Files, NamedFile}; diff --git a/src/runtimes/manager.rs b/src/runtimes/manager.rs new file mode 100644 index 00000000..9e93757f --- /dev/null +++ b/src/runtimes/manager.rs @@ -0,0 +1,59 @@ +//// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + metadata::{RemoteFile, Runtime as RuntimeMetadata}, + modules::{javascript::JavaScriptRuntime, native::NativeRuntime}, + runtime::Runtime, +}; +use crate::{fetch::fetch_and_validate, store::Store}; +use anyhow::{anyhow, Result}; +use std::path::Path; + +// A collection of methods to manage runtimes + +/// Initializes a runtime based on the file extension. In the future, +/// This will contain a more complete struct that will identify local +/// runtimes. +pub fn init_runtime(path: &Path) -> Result> { + if let Some(ext) = path.extension() { + let ext_as_str = ext.to_str().unwrap(); + + match ext_as_str { + "js" => Ok(Box::new(JavaScriptRuntime::new(path.to_path_buf())?)), + "wasm" => Ok(Box::new(NativeRuntime::new(path.to_path_buf()))), + _ => Err(anyhow!(format!( + "The '{}' extension does not have an associated runtime", + ext_as_str + ))), + } + } else { + Err(anyhow!("The given file does not have a valid extension")) + } +} + +// TODO: Remove it when implementing the full logic +#[allow(dead_code)] +// Install a given runtime based on its metadata +pub async fn install_runtime(repository: &str, metadata: &RuntimeMetadata) -> Result<()> { + let store = Store::new(&["runtimes", repository, &metadata.name, &metadata.version])?; + + // Install the different files + download_file(&metadata.binary, &store).await?; + + if let Some(polyfill) = &metadata.polyfill { + download_file(polyfill, &store).await?; + } + + if let Some(template) = &metadata.template { + download_file(template, &store).await?; + } + + Ok(()) +} + +// TODO: Remove it when implementing the full logic +async fn download_file(file: &RemoteFile, store: &Store) -> Result<()> { + let contents = fetch_and_validate(&file.url, &file.checksum).await?; + store.write(&[&file.filename], &contents) +} diff --git a/src/runtimes/metadata.rs b/src/runtimes/metadata.rs index 89e6f534..a9acf09a 100644 --- a/src/runtimes/metadata.rs +++ b/src/runtimes/metadata.rs @@ -1,10 +1,49 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -use super::{remote_file::RemoteFile, runtime::RuntimeStatus}; +use crate::fetch::fetch; use anyhow::{anyhow, Result}; use serde::Deserialize; -use std::collections::HashMap; +use sha256::digest as sha256_digest; +use std::{collections::HashMap, fmt}; +use url::Url; + +/// A Repository contains the list of runtimes available on it. +/// This file is used by wws to properly show the list of available +/// repos and install them. +/// +/// By default, this repository class rely on the +/// [WebAssembly Language Runtimes](https://github.com/vmware-labs/webassembly-language-runtimes) +/// repository. It looks for a repository.toml file in the Git repo. +#[derive(Deserialize)] +pub struct Repository { + /// Version of the repository file + pub version: u32, + /// The list of runtimes available in the repository + pub runtimes: Vec, +} + +// TODO: Remove it when implementing the manager +#[allow(dead_code)] +impl Repository { + /// Reads and parses the metadata from a slice of bytes. It will return + /// a result as the deserialization may fail. + pub fn from_slice(data: &[u8]) -> Result { + toml::from_slice::(data).map_err(|err| { + println!("Err: {:?}", err); + anyhow!("wws could not deserialize the repository metadata") + }) + } + + /// Retrieve a repository from a remote file. It will download the content + /// using reqwest and initializing the repository with it. + pub async fn from_remote_file(repository_url: &str) -> Result { + let url = Url::parse(repository_url)?; + let data = fetch(&url).await?; + + Repository::from_slice(&data) + } +} // TODO: Remove it when implementing the manager #[allow(dead_code)] @@ -17,52 +56,116 @@ use std::collections::HashMap; /// different pieces like polyfills files, templates, /// arguments, etc. #[derive(Deserialize)] -pub struct RuntimeMetadata<'a> { +pub struct Runtime { /// Name of the runtime (like ruby, python, etc) - name: &'a str, + pub name: String, /// Specific version of the runtime - version: &'a str, + pub version: String, /// Current status in the repository - status: RuntimeStatus, + pub status: RuntimeStatus, /// Associated extensions - extensions: Vec<&'a str>, + pub extensions: Vec, /// Arguments to pass to the Wasm module via WASI - args: Vec<&'a str>, + pub args: Vec, /// A list of environment variables that must be configured /// for the runtime to work. - envs: Option>, + pub envs: Option>, /// The reference to a remote binary (url + checksum) - binary: RemoteFile<'a>, + pub binary: RemoteFile, /// The reference to a remote polyfill file (url + checksum) - polyfill: Option>, - /// The refernmece to a template file for the worker. It will wrap the + pub polyfill: Option, + /// The reference to a template file for the worker. It will wrap the /// source code into a template that can include imports, /// function calls, etc. - template: Option>, + pub template: Option, } -// TODO: Remove it when implementing the manager -#[allow(dead_code)] -impl<'a> RuntimeMetadata<'a> { - /// Reads and parses the metadata from a slice of bytes. It will return - /// a result as the deserialization may fail. - pub fn from_slice(data: &'a [u8]) -> Result { - toml::from_slice::(data) - .map_err(|_| anyhow!("wws could not deserialize the runtime metadata")) +// Implement the Display trait to runtime metadata properly in the CLI +impl fmt::Display for Runtime { + /// Prints the runtime as a row for a list of runtimes + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}\t{}\t{}\t{}", + &self.name, + &self.version, + self.extensions.join(", "), + &self.binary.url + ) + } +} + +/// Define the status of a runtime in a target repository +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RuntimeStatus { + Active, + Yanked, + Deprecated, + Unknown, +} + +impl From<&str> for RuntimeStatus { + /// Create a RuntimeStatus variant from a &str. It uses predefined + /// values + fn from(value: &str) -> Self { + match value { + "active" => RuntimeStatus::Active, + "yanked" => RuntimeStatus::Yanked, + "deprecated" => RuntimeStatus::Deprecated, + _ => RuntimeStatus::Unknown, + } + } +} + +/// A file represents a combination of both a remote URL, filename +/// and checksum. +#[derive(Deserialize)] +pub struct RemoteFile { + /// URL pointing to the file + pub url: String, + /// Checksum to validate the given file + pub checksum: Checksum, + /// Provide a filename + pub filename: String, +} + +/// A list of available checksums. For now, we will support only sha256 +#[derive(Deserialize)] +#[serde(rename_all = "lowercase", tag = "type")] +pub enum Checksum { + Sha256 { value: String }, +} + +impl Checksum { + /// Validate the provided slice of bytes with the given checksum. + /// Depending on the type, it will calculate a different digest. + pub fn validate(&self, bytes: &[u8]) -> Result<()> { + match self { + Checksum::Sha256 { value } if value == &sha256_digest(bytes) => Ok(()), + _ => Err(anyhow!("The checksums dont not match")), + } } } #[cfg(test)] mod tests { - use crate::runtimes::remote_file::Checksum; - use super::*; use std::{any::Any, fs}; + #[test] + fn parse_index_toml() { + let contents = fs::read("tests/data/metadata/repository.toml").unwrap(); + let repo = Repository::from_slice(&contents).unwrap(); + + assert_eq!(repo.version, 1); + assert_eq!(repo.runtimes.len(), 1); + } + #[test] fn parse_runtime_toml() { let contents = fs::read("tests/data/metadata/runtime.toml").unwrap(); - let metadata = RuntimeMetadata::from_slice(&contents).unwrap(); + let metadata = toml::from_slice::(&contents).unwrap(); assert_eq!(metadata.name, "ruby"); assert_eq!(metadata.version, "3.2.0+20230118-8aec06d"); diff --git a/src/runtimes/mod.rs b/src/runtimes/mod.rs index c2303134..42a81444 100644 --- a/src/runtimes/mod.rs +++ b/src/runtimes/mod.rs @@ -1,8 +1,7 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -mod metadata; +pub mod manager; +pub mod metadata; mod modules; -mod remote_file; -mod repository; pub mod runtime; diff --git a/src/runtimes/modules/data.rs b/src/runtimes/modules/data.rs deleted file mode 100644 index 9baa96be..00000000 --- a/src/runtimes/modules/data.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -// Define multiple utils functions that runtimes -// will use. This includes methods to create shared -// folders, provide a hashes for the functions, etc. - -use anyhow::Result; -use std::{fs, path::Path}; - -/// This is a temporary folder in which runtimes can prepare -/// and store certain data. For example, the JS runtime have -/// to mount a folder with the source code. To avoid moun†ing -/// a folder that may include multiple files, it stores in -/// .wws/js/XXX/index.js the worker file. -const TMP_FOLDER: &str = ".wws"; - -/// Maange the required data to run a worker in a specific -/// runtime. As stated before, some runtimes may require to write -/// temporary files on a folder. Note that Wasm VMs require -/// to mount a folder, not a file. To keep workers isolated -/// between others, we will prepare specific folders with the -/// source code only. -pub struct Data { - /// The folder inside the main TMP folder. - pub folder: String, -} - -impl Data { - /// Creates a new temp folder for the given language. This will - /// allow later on to write files in that folder. - pub fn new(lang_folder: String, source_path: &Path) -> Result { - let hash = Self::file_hash(source_path)?; - let folder = format!("{}/{}/{}", TMP_FOLDER, lang_folder, hash); - - fs::create_dir_all(&folder)?; - - Ok(Self { folder }) - } - - /// Write a source file into the temp language folder - pub fn write_source(&self, source_path: &Path) -> Result { - let ext = source_path - .extension() - .unwrap_or_default() - .to_str() - .unwrap_or_default(); - - fs::copy(source_path, format!("{}/index.{}", self.folder, ext)).map_err(anyhow::Error::msg) - } - - /// Geenrate a file hash based on the blake3 implementation. This will - /// allow to have multiple folders that don't collide between them. - pub fn file_hash(path: &Path) -> Result { - let content = fs::read(path)?; - - Ok(blake3::hash(&content).to_string()) - } -} diff --git a/src/runtimes/modules/javascript.rs b/src/runtimes/modules/javascript.rs index 54aa9e68..8a1caaea 100644 --- a/src/runtimes/modules/javascript.rs +++ b/src/runtimes/modules/javascript.rs @@ -1,8 +1,8 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -use super::data::Data; use crate::runtimes::runtime::Runtime; +use crate::store::Store; use anyhow::Result; use std::{fs, path::PathBuf}; use wasmtime_wasi::Dir; @@ -14,8 +14,8 @@ static JS_ENGINE_WASM: &[u8] = pub struct JavaScriptRuntime { /// Path of the given module path: PathBuf, - /// Utils to make temporary files - data: Data, + /// Utils to store temporary files for this runtime + store: Store, } impl JavaScriptRuntime { @@ -25,9 +25,10 @@ impl JavaScriptRuntime { /// automatically pick and run it. We use the Data struct for /// this purpose pub fn new(path: PathBuf) -> Result { - let data = Data::new(String::from("js"), &path)?; + let hash = Store::file_hash(&path)?; + let store = Store::new(&["workers", "js", &hash])?; - Ok(Self { path, data }) + Ok(Self { path, store }) } } @@ -38,7 +39,7 @@ impl Runtime for JavaScriptRuntime { /// file into an isolated and separate folder. Then, we will mount /// it during the [prepare_wasi_ctx] call. fn prepare(&self) -> Result<()> { - self.data.write_source(&self.path)?; + self.store.copy(&self.path, &["index.js"])?; Ok(()) } @@ -46,7 +47,7 @@ impl Runtime for JavaScriptRuntime { /// Mount the source code in the WASI context so it can be /// processed by the engine fn prepare_wasi_ctx(&self, builder: WasiCtxBuilder) -> Result { - let source = fs::File::open(&self.data.folder)?; + let source = fs::File::open(&self.store.folder)?; Ok(builder.preopened_dir(Dir::from_std_file(source), "/src")?) } diff --git a/src/runtimes/modules/mod.rs b/src/runtimes/modules/mod.rs index f6ca7d95..9764954c 100644 --- a/src/runtimes/modules/mod.rs +++ b/src/runtimes/modules/mod.rs @@ -1,7 +1,5 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -mod data; - pub(super) mod javascript; pub(super) mod native; diff --git a/src/runtimes/remote_file.rs b/src/runtimes/remote_file.rs deleted file mode 100644 index c14a7b14..00000000 --- a/src/runtimes/remote_file.rs +++ /dev/null @@ -1,24 +0,0 @@ -use serde::Deserialize; - -/// A file represents a combination of both a remote URL and -/// a checksum. It will provide utils to download and validate -/// the file. -/// -/// If the checksum is not present, the validation will return -/// always an Ok result. -/// -#[derive(Deserialize)] -pub struct RemoteFile<'a> { - /// URL pointing to the file - pub url: &'a str, - /// Checksum to validate the given file - pub checksum: Checksum<'a>, -} - -/// A list of available checksums. For now, -/// we will support only sha256 -#[derive(Deserialize)] -#[serde(rename_all = "lowercase", tag = "type")] -pub enum Checksum<'a> { - Sha256 { value: &'a str }, -} diff --git a/src/runtimes/repository.rs b/src/runtimes/repository.rs deleted file mode 100644 index d19d0710..00000000 --- a/src/runtimes/repository.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2022 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use super::metadata::RuntimeMetadata; -use anyhow::{anyhow, Result}; -use serde::Deserialize; - -// TODO: Remove it when implementing the manager -#[allow(dead_code)] -/// A Repository contains the list of runtimes available on it. -/// This file is used by wws to properly show the list of available -/// repos and install them. -/// -/// By default, this repository class rely on the [WebAssembly Language Runtimes](https://github.com/vmware-labs/webassembly-language-runtimes) -/// repository. It looks for a repository.toml file in the Git repo. -#[derive(Deserialize)] -pub struct Repository<'a> { - /// Version of the repository file - pub version: u32, - /// The list of runtimes available in the repository - #[serde(borrow)] - pub runtimes: Vec>, -} - -// TODO: Remove it when implementing the manager -#[allow(dead_code)] -impl<'a> Repository<'a> { - /// Reads and parses the metadata from a slice of bytes. It will return - /// a result as the deserialization may fail. - pub fn from_slice(data: &'a [u8]) -> Result { - toml::from_slice::(data) - .map_err(|_| anyhow!("wws could not deserialize the repository metadata")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - #[test] - fn parse_index_toml() { - let contents = fs::read("tests/data/metadata/repository.toml").unwrap(); - let repo = Repository::from_slice(&contents).unwrap(); - - assert_eq!(repo.version, 1); - assert_eq!(repo.runtimes.len(), 1); - } -} diff --git a/src/runtimes/runtime.rs b/src/runtimes/runtime.rs index b9842cc6..5f337ea3 100644 --- a/src/runtimes/runtime.rs +++ b/src/runtimes/runtime.rs @@ -1,35 +1,9 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -use super::modules::{javascript::JavaScriptRuntime, native::NativeRuntime}; -use anyhow::{anyhow, Result}; -use serde::Deserialize; -use std::path::Path; +use anyhow::Result; use wasmtime_wasi::WasiCtxBuilder; -/// Define the status of a runtime in a target repository -#[derive(Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum RuntimeStatus { - Active, - Yanked, - Deprecated, - Unknown, -} - -impl From<&str> for RuntimeStatus { - /// Create a RuntimeStatus variant from a &str. It uses predefined - /// values - fn from(value: &str) -> Self { - match value { - "active" => RuntimeStatus::Active, - "yanked" => RuntimeStatus::Yanked, - "deprecated" => RuntimeStatus::Deprecated, - _ => RuntimeStatus::Unknown, - } - } -} - /// Define the behavior a Runtime must have. This includes methods /// to initialize the environment for the given runtime as well as /// the Wasi Context to process the request. @@ -56,23 +30,3 @@ pub trait Runtime { /// runtime like JS or Python. fn module_bytes(&self) -> Result>; } - -/// Initializes a runtime based on the file extension. In the future, -/// This will contain a more complete struct that will identify local -/// runtimes. -pub fn init_runtime(path: &Path) -> Result> { - if let Some(ext) = path.extension() { - let ext_as_str = ext.to_str().unwrap(); - - match ext_as_str { - "js" => Ok(Box::new(JavaScriptRuntime::new(path.to_path_buf())?)), - "wasm" => Ok(Box::new(NativeRuntime::new(path.to_path_buf()))), - _ => Err(anyhow!(format!( - "The '{}' extension does not have an associated runtime", - ext_as_str - ))), - } - } else { - Err(anyhow!("The given file does not have a valid extension")) - } -} diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 00000000..cd590d42 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,85 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Define a common place to store the data associated to the +// user project. wws requires to install runtimes, create +// temporary files with workers metadata, etc. +// +// This struct provide the basics to interact with that folder +// in both Unix and Windows systems. + +use anyhow::Result; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// This is a temporary folder in which runtimes can prepare +/// and store certain data. For example, the JS runtime have +/// to mount a folder with the source code. To avoid moun†ing +/// a folder that may include multiple files, it stores in +/// .wws/js/XXX/index.js the worker file. +const STORE_FOLDER: &str = ".wws"; + +/// Struct to initialize, create and interact with files inside +/// the store. All paths are considered &[&str] to ensure we +/// generate the paths properly on Windows and Unix. +pub struct Store { + /// The base folder for this instance. Every time we initialize + /// a store, it will ensure all the files are scoped to the given + /// folder. + pub folder: PathBuf, +} + +// TODO: Remove it when implementing the full logic +#[allow(dead_code)] +impl Store { + /// Instance a new store and creates the root folder. The root path is + /// used to scope the files inside the STORE_FOLDER folder. + pub fn new(root: &[&str]) -> Result { + let folder = Self::build_root_path(root); + + // Try to create the directory + fs::create_dir_all(&folder)?; + + Ok(Self { folder }) + } + + /// Write a specific file inside the configured root folder + pub fn write(&self, path: &[&str], contents: &[u8]) -> Result<()> { + let file_path = self.build_folder_path(path); + fs::write(file_path, contents)?; + + Ok(()) + } + + /// Copy file inside the configured root folder + pub fn copy(&self, source: &Path, dest: &[&str]) -> Result<()> { + let file_path = self.build_folder_path(dest); + fs::copy(source, file_path)?; + + Ok(()) + } + + /// This method builds a path in the context of the instance folder + fn build_folder_path(&self, source: &[&str]) -> PathBuf { + source + .iter() + .fold(self.folder.clone(), |acc, comp| acc.join(comp)) + } + + /// Generate a file hash based on the blake3 implementation + pub fn file_hash(path: &Path) -> Result { + let content = fs::read(path)?; + + Ok(blake3::hash(&content).to_string()) + } + + /// Build a valid path for multiple platforms. It takes advantages of the + /// Path methods + fn build_root_path(source: &[&str]) -> PathBuf { + source + .iter() + .fold(PathBuf::from(STORE_FOLDER), |acc, comp| acc.join(comp)) + } +} diff --git a/src/workers/worker.rs b/src/workers/worker.rs index 980926ae..7b359fa0 100644 --- a/src/workers/worker.rs +++ b/src/workers/worker.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::wasm_io::{WasmInput, WasmOutput}; -use crate::runtimes::runtime::{init_runtime, Runtime}; +use crate::runtimes::{manager::init_runtime, runtime::Runtime}; use actix_web::HttpRequest; use anyhow::Result; use std::{collections::HashMap, path::Path}; diff --git a/tests/data/metadata/repository.toml b/tests/data/metadata/repository.toml index a1ead61e..ea64bf9d 100644 --- a/tests/data/metadata/repository.toml +++ b/tests/data/metadata/repository.toml @@ -5,7 +5,7 @@ name = "ruby" version = "3.2.0+20230118-8aec06d" status = "active" args = [ "--", "/src/index.rb" ] -binary = { url = "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/ruby%2F3.2.0%2B20230118-8aec06d/ruby-3.2.0.wasm", checksum = { type = "sha256", value = "e2d91cff05ec59ed9c88aadbd3b477842092054bf24c5d944d5ad6dbafdd3b32" } } +binary = { filename = "ruby.wasm", url = "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/ruby%2F3.2.0%2B20230118-8aec06d/ruby-3.2.0.wasm", checksum = { type = "sha256", value = "e2d91cff05ec59ed9c88aadbd3b477842092054bf24c5d944d5ad6dbafdd3b32" } } extensions = [ "rb" ] -polyfill = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/poly.rb", checksum = { type = "sha256", value = "2ba09117ed20a05480615b2aaaf6c7cd7f61fa06f4919777d773df2fcbc736cf" } } -template = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/template.txt", checksum = { type = "sha256", value = "6d808b4747cf30f82665a38a47e1176513bbdd6ad558c09db03d719e33ad2da0" } } \ No newline at end of file +polyfill = { filename = "poly.rb", url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/poly.rb", checksum = { type = "sha256", value = "2ba09117ed20a05480615b2aaaf6c7cd7f61fa06f4919777d773df2fcbc736cf" } } +template = { filename = "template.txt", url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/template.txt", checksum = { type = "sha256", value = "6d808b4747cf30f82665a38a47e1176513bbdd6ad558c09db03d719e33ad2da0" } } \ No newline at end of file diff --git a/tests/data/metadata/runtime.toml b/tests/data/metadata/runtime.toml index 5e6c37f3..c3cb0951 100644 --- a/tests/data/metadata/runtime.toml +++ b/tests/data/metadata/runtime.toml @@ -2,7 +2,7 @@ name = "ruby" version = "3.2.0+20230118-8aec06d" status = "active" args = [ "--", "/src/index.rb" ] -binary = { url = "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/ruby%2F3.2.0%2B20230118-8aec06d/ruby-3.2.0.wasm", checksum = { type = "sha256", value = "e2d91cff05ec59ed9c88aadbd3b477842092054bf24c5d944d5ad6dbafdd3b32" } } +binary = { url = "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/ruby%2F3.2.0%2B20230118-8aec06d/ruby-3.2.0.wasm", filename = "ruby.wasm", checksum = { type = "sha256", value = "e2d91cff05ec59ed9c88aadbd3b477842092054bf24c5d944d5ad6dbafdd3b32" } } extensions = [ "rb" ] -polyfill = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/poly.rb", checksum = { type = "sha256", value = "2ba09117ed20a05480615b2aaaf6c7cd7f61fa06f4919777d773df2fcbc736cf" } } -template = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/template.txt", checksum = { type = "sha256", value = "6d808b4747cf30f82665a38a47e1176513bbdd6ad558c09db03d719e33ad2da0" } } \ No newline at end of file +polyfill = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/poly.rb", filename = "poly.rb", checksum = { type = "sha256", value = "2ba09117ed20a05480615b2aaaf6c7cd7f61fa06f4919777d773df2fcbc736cf" } } +template = { url = "https://raw.githubusercontent.com/Angelmmiguel/wws-index-test/main/ruby/template.txt", filename = "template.txt", checksum = { type = "sha256", value = "6d808b4747cf30f82665a38a47e1176513bbdd6ad558c09db03d719e33ad2da0" } } \ No newline at end of file From a3ac08196139bbe7720ec52030355bbda5ca27e5 Mon Sep 17 00:00:00 2001 From: Angel M De Miguel Date: Thu, 26 Jan 2023 11:47:53 +0100 Subject: [PATCH 2/4] feat: upgrade TOML to 0.6.0 and allow serializing metadata --- Cargo.lock | 56 ++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 +- src/config.rs | 7 ++--- src/runtimes/metadata.rs | 25 +++++++++--------- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afde3bb3..3bb4db1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1672,6 +1672,15 @@ dependencies = [ "nom", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -2196,6 +2205,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2510,6 +2528,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729bfd096e40da9c001f778f5cdecbd2957929a24e10e5883d9392220a751581" +dependencies = [ + "indexmap", + "nom8", + "serde", + "serde_spanned", + "toml_datetime", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -2814,7 +2866,7 @@ dependencies = [ "serde", "serde_json", "sha256", - "toml", + "toml 0.6.0", "url", "wasi-common", "wasmtime", @@ -2887,7 +2939,7 @@ dependencies = [ "rustix", "serde", "sha2 0.10.6", - "toml", + "toml 0.5.10", "windows-sys", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 2143af35..d64ea3a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ env_logger = "0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.85" wax = "0.5.0" -toml = "0.5.9" +toml = "0.6.0" base64 = "0.13.1" clap = { version = "4.0.10", features = ["derive"] } regex = "1" diff --git a/src/config.rs b/src/config.rs index dc910567..37afcf10 100644 --- a/src/config.rs +++ b/src/config.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Deserializer}; use std::collections::HashMap; use std::path::PathBuf; use std::{env, fs}; -use toml::from_slice; +use toml::from_str; /// Workers configuration. These files are optional when no configuration change is required. #[derive(Deserialize, Clone)] @@ -46,9 +46,10 @@ impl Config { /// namespace = "todos" /// ``` pub fn try_from_file(path: PathBuf) -> Result { - let contents = fs::read(&path).expect("The configuration file was not properly loaded"); + let contents = + fs::read_to_string(&path).expect("The configuration file was not properly loaded"); - let try_config: Result = from_slice(&contents); + let try_config: Result = from_str(&contents); match try_config { Ok(c) => Ok(c), diff --git a/src/runtimes/metadata.rs b/src/runtimes/metadata.rs index a9acf09a..5c0bb88a 100644 --- a/src/runtimes/metadata.rs +++ b/src/runtimes/metadata.rs @@ -3,7 +3,7 @@ use crate::fetch::fetch; use anyhow::{anyhow, Result}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha256::digest as sha256_digest; use std::{collections::HashMap, fmt}; use url::Url; @@ -28,8 +28,8 @@ pub struct Repository { impl Repository { /// Reads and parses the metadata from a slice of bytes. It will return /// a result as the deserialization may fail. - pub fn from_slice(data: &[u8]) -> Result { - toml::from_slice::(data).map_err(|err| { + pub fn from_str(data: &str) -> Result { + toml::from_str::(data).map_err(|err| { println!("Err: {:?}", err); anyhow!("wws could not deserialize the repository metadata") }) @@ -40,8 +40,9 @@ impl Repository { pub async fn from_remote_file(repository_url: &str) -> Result { let url = Url::parse(repository_url)?; let data = fetch(&url).await?; + let str_data = String::from_utf8(data)?; - Repository::from_slice(&data) + Repository::from_str(&str_data) } } @@ -55,7 +56,7 @@ impl Repository { /// a source code as a worker. The configuration includes /// different pieces like polyfills files, templates, /// arguments, etc. -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct Runtime { /// Name of the runtime (like ruby, python, etc) pub name: String, @@ -96,7 +97,7 @@ impl fmt::Display for Runtime { } /// Define the status of a runtime in a target repository -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Clone)] #[serde(rename_all = "lowercase")] pub enum RuntimeStatus { Active, @@ -120,7 +121,7 @@ impl From<&str> for RuntimeStatus { /// A file represents a combination of both a remote URL, filename /// and checksum. -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct RemoteFile { /// URL pointing to the file pub url: String, @@ -131,7 +132,7 @@ pub struct RemoteFile { } /// A list of available checksums. For now, we will support only sha256 -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Clone)] #[serde(rename_all = "lowercase", tag = "type")] pub enum Checksum { Sha256 { value: String }, @@ -155,8 +156,8 @@ mod tests { #[test] fn parse_index_toml() { - let contents = fs::read("tests/data/metadata/repository.toml").unwrap(); - let repo = Repository::from_slice(&contents).unwrap(); + let contents = fs::read_to_string("tests/data/metadata/repository.toml").unwrap(); + let repo = Repository::from_str(&contents).unwrap(); assert_eq!(repo.version, 1); assert_eq!(repo.runtimes.len(), 1); @@ -164,8 +165,8 @@ mod tests { #[test] fn parse_runtime_toml() { - let contents = fs::read("tests/data/metadata/runtime.toml").unwrap(); - let metadata = toml::from_slice::(&contents).unwrap(); + let contents = fs::read_to_string("tests/data/metadata/runtime.toml").unwrap(); + let metadata = toml::from_str::(&contents).unwrap(); assert_eq!(metadata.name, "ruby"); assert_eq!(metadata.version, "3.2.0+20230118-8aec06d"); From fb0189e9cc9062169613c31296ac52a6a8d07521 Mon Sep 17 00:00:00 2001 From: Angel M De Miguel Date: Thu, 26 Jan 2023 12:21:59 +0100 Subject: [PATCH 3/4] fix: store temporary files (.wws) in the project folder --- src/router/route.rs | 2 +- src/runtimes/manager.rs | 18 ++++++++++++++---- src/runtimes/modules/javascript.rs | 5 +++-- src/store.rs | 8 ++++---- src/workers/worker.rs | 4 ++-- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/router/route.rs b/src/router/route.rs index 4f8c6a80..3bc1c0d9 100644 --- a/src/router/route.rs +++ b/src/router/route.rs @@ -54,7 +54,7 @@ impl Route { /// /// This method also initializes the Runner and loads the Config if available. pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str) -> Self { - let worker = Worker::new(&filepath).unwrap(); + let worker = Worker::new(base_path, &filepath).unwrap(); // Load configuration let mut config_path = filepath.clone(); diff --git a/src/runtimes/manager.rs b/src/runtimes/manager.rs index 9e93757f..cffd5676 100644 --- a/src/runtimes/manager.rs +++ b/src/runtimes/manager.rs @@ -15,12 +15,15 @@ use std::path::Path; /// Initializes a runtime based on the file extension. In the future, /// This will contain a more complete struct that will identify local /// runtimes. -pub fn init_runtime(path: &Path) -> Result> { +pub fn init_runtime(project_root: &Path, path: &Path) -> Result> { if let Some(ext) = path.extension() { let ext_as_str = ext.to_str().unwrap(); match ext_as_str { - "js" => Ok(Box::new(JavaScriptRuntime::new(path.to_path_buf())?)), + "js" => Ok(Box::new(JavaScriptRuntime::new( + project_root, + path.to_path_buf(), + )?)), "wasm" => Ok(Box::new(NativeRuntime::new(path.to_path_buf()))), _ => Err(anyhow!(format!( "The '{}' extension does not have an associated runtime", @@ -35,8 +38,15 @@ pub fn init_runtime(path: &Path) -> Result> { // TODO: Remove it when implementing the full logic #[allow(dead_code)] // Install a given runtime based on its metadata -pub async fn install_runtime(repository: &str, metadata: &RuntimeMetadata) -> Result<()> { - let store = Store::new(&["runtimes", repository, &metadata.name, &metadata.version])?; +pub async fn install_runtime( + project_root: &Path, + repository: &str, + metadata: &RuntimeMetadata, +) -> Result<()> { + let store = Store::new( + project_root, + &["runtimes", repository, &metadata.name, &metadata.version], + )?; // Install the different files download_file(&metadata.binary, &store).await?; diff --git a/src/runtimes/modules/javascript.rs b/src/runtimes/modules/javascript.rs index 8a1caaea..1f6b72bc 100644 --- a/src/runtimes/modules/javascript.rs +++ b/src/runtimes/modules/javascript.rs @@ -4,6 +4,7 @@ use crate::runtimes::runtime::Runtime; use crate::store::Store; use anyhow::Result; +use std::path::Path; use std::{fs, path::PathBuf}; use wasmtime_wasi::Dir; use wasmtime_wasi::WasiCtxBuilder; @@ -24,9 +25,9 @@ impl JavaScriptRuntime { /// mount the JS file into /src/index.js and the runtime will /// automatically pick and run it. We use the Data struct for /// this purpose - pub fn new(path: PathBuf) -> Result { + pub fn new(project_root: &Path, path: PathBuf) -> Result { let hash = Store::file_hash(&path)?; - let store = Store::new(&["workers", "js", &hash])?; + let store = Store::new(project_root, &["workers", "js", &hash])?; Ok(Self { path, store }) } diff --git a/src/store.rs b/src/store.rs index cd590d42..363e10aa 100644 --- a/src/store.rs +++ b/src/store.rs @@ -36,8 +36,8 @@ pub struct Store { impl Store { /// Instance a new store and creates the root folder. The root path is /// used to scope the files inside the STORE_FOLDER folder. - pub fn new(root: &[&str]) -> Result { - let folder = Self::build_root_path(root); + pub fn new(project_root: &Path, folder: &[&str]) -> Result { + let folder = Self::build_root_path(project_root, folder); // Try to create the directory fs::create_dir_all(&folder)?; @@ -77,9 +77,9 @@ impl Store { /// Build a valid path for multiple platforms. It takes advantages of the /// Path methods - fn build_root_path(source: &[&str]) -> PathBuf { + fn build_root_path(root: &Path, source: &[&str]) -> PathBuf { source .iter() - .fold(PathBuf::from(STORE_FOLDER), |acc, comp| acc.join(comp)) + .fold(root.join(STORE_FOLDER), |acc, comp| acc.join(comp)) } } diff --git a/src/workers/worker.rs b/src/workers/worker.rs index 7b359fa0..007dc65f 100644 --- a/src/workers/worker.rs +++ b/src/workers/worker.rs @@ -24,9 +24,9 @@ pub struct Worker { impl Worker { /// Creates a new Worker - pub fn new(path: &Path) -> Result { + pub fn new(project_root: &Path, path: &Path) -> Result { let engine = Engine::default(); - let runtime = init_runtime(path)?; + let runtime = init_runtime(project_root, path)?; let bytes = runtime.module_bytes()?; let module = Module::from_binary(&engine, &bytes)?; From 50ab01e40fe2c39ddf285dfb82afb38d184ed77a Mon Sep 17 00:00:00 2001 From: Angel M De Miguel Date: Thu, 26 Jan 2023 12:40:48 +0100 Subject: [PATCH 4/4] fix: bubble config loading error and remove the Display impl --- src/config.rs | 8 ++++---- src/runtimes/metadata.rs | 17 +---------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/config.rs b/src/config.rs index 37afcf10..4b252268 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::data::kv::KVConfigData; +use anyhow::{anyhow, Result}; use serde::{Deserialize, Deserializer}; use std::collections::HashMap; use std::path::PathBuf; @@ -45,15 +46,14 @@ impl Config { /// [data.kv] /// namespace = "todos" /// ``` - pub fn try_from_file(path: PathBuf) -> Result { - let contents = - fs::read_to_string(&path).expect("The configuration file was not properly loaded"); + pub fn try_from_file(path: PathBuf) -> Result { + let contents = fs::read_to_string(&path)?; let try_config: Result = from_str(&contents); match try_config { Ok(c) => Ok(c), - Err(err) => Err(format!( + Err(err) => Err(anyhow!( "Error reading the configuration file at {}: {}", &path.to_str().unwrap_or("?"), err diff --git a/src/runtimes/metadata.rs b/src/runtimes/metadata.rs index 5c0bb88a..1463a5ca 100644 --- a/src/runtimes/metadata.rs +++ b/src/runtimes/metadata.rs @@ -5,7 +5,7 @@ use crate::fetch::fetch; use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; use sha256::digest as sha256_digest; -use std::{collections::HashMap, fmt}; +use std::collections::HashMap; use url::Url; /// A Repository contains the list of runtimes available on it. @@ -81,21 +81,6 @@ pub struct Runtime { pub template: Option, } -// Implement the Display trait to runtime metadata properly in the CLI -impl fmt::Display for Runtime { - /// Prints the runtime as a row for a list of runtimes - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{}\t{}\t{}\t{}", - &self.name, - &self.version, - self.extensions.join(", "), - &self.binary.url - ) - } -} - /// Define the status of a runtime in a target repository #[derive(Deserialize, Serialize, Clone)] #[serde(rename_all = "lowercase")]