feat(vm)!: create vm builtin [fixes BRND-16] (#2320)

.github/workflows/e2e.yml

@@ -57,6 +57,7 @@ jobs:
       !github.event.pull_request.head.repo.fork
     uses: fluencelabs/decider/.github/workflows/tests.yml@main
     with:
+      ref: "feature/brnd-19"
       test-cargo-dependencies: |
         [
           {

Cargo.toml

@@ -104,6 +104,7 @@ chain-connector = { path = "crates/chain-connector" }
 types = { path = "crates/types" }
 core-distributor = { path = "crates/core-distributor" }
 log-format = { path = "crates/log-format" }
+vm-utils = { path = "crates/vm-utils" }
 
 # spell
 fluence-spell-dtos = "=0.7.5"

aquamarine/src/plumber.rs

@@ -655,15 +655,15 @@ where
 mod tests {
     use std::collections::HashMap;
     use std::convert::Infallible;
-    use std::path::PathBuf;
+    use std::path::{Path, PathBuf};
     use std::task::Waker;
     use std::{sync::Arc, task::Context};
 
     use avm_server::{AVMMemoryStats, CallResults, ParticleParameters};
     use fluence_keypair::KeyPair;
     use fluence_libp2p::RandomPeerId;
     use futures::task::noop_waker_ref;
-    use workers::{KeyStorage, PeerScopes, Workers};
+    use workers::{KeyStorage, PeerScopes, Workers, WorkersConfig};
 
     use particle_args::Args;
     use particle_execution::{FunctionOutcome, ParticleFunction, ParticleParams, ServiceFunction};
@@ -678,6 +678,7 @@ mod tests {
     use async_trait::async_trait;
     use avm_server::avm_runner::RawAVMOutcome;
     use core_distributor::dummy::DummyCoreDistibutor;
+    use fs_utils::create_dirs;
     use marine_wasmtime_backend::{WasmtimeConfig, WasmtimeWasmBackend};
     use particle_services::{PeerScope, WasmBackendConfig};
     use tracing::Span;
@@ -759,7 +760,7 @@ mod tests {
         }
     }
 
-    async fn plumber() -> Plumber<VMMock, Arc<MockF>> {
+    async fn plumber(dir: &Path) -> Plumber<VMMock, Arc<MockF>> {
         let avm_wasm_config: WasmtimeConfig = WasmBackendConfig::default().into();
         let avm_wasm_backend =
             WasmtimeWasmBackend::new(avm_wasm_config).expect("Could not create wasm backend");
@@ -768,8 +769,11 @@ mod tests {
         let builtin_mock = Arc::new(MockF);
 
         let root_key_pair: KeyPair = KeyPair::generate_ed25519();
-        let key_pair_path: PathBuf = "keypair".into();
-        let workers_path: PathBuf = "workers".into();
+        let key_pair_path: PathBuf = dir.join("keypair");
+        let workers_path: PathBuf = dir.join("workers");
+
+        create_dirs(&[&key_pair_path, &workers_path]).unwrap();
+
         let key_storage = KeyStorage::from_path(key_pair_path.clone(), root_key_pair.clone())
             .await
             .expect("Could not load key storage");
@@ -786,24 +790,24 @@ mod tests {
             key_storage.clone(),
         );
 
+        let workers_config = WorkersConfig::new(32, None);
+
         let (workers, _receiver) = Workers::from_path(
+            workers_config,
             workers_path.clone(),
             key_storage.clone(),
             core_distributor,
             thread_pinner,
-            32,
         )
         .await
         .expect("Could not load worker registry");
 
         let workers = Arc::new(workers);
 
-        let tmp_dir = tempfile::tempdir().expect("Could not create temp dir");
-        let tmp_path = tmp_dir.path();
         let data_store = ParticleDataStore::new(
-            tmp_path.join("particles"),
-            tmp_path.join("vault"),
-            tmp_path.join("anomaly"),
+            dir.join("particles"),
+            dir.join("vault"),
+            dir.join("anomaly"),
         );
         data_store
             .initialize()
@@ -828,7 +832,6 @@ mod tests {
         let mut particle = Particle::default();
         particle.timestamp = ts;
         particle.ttl = ttl;
-
         particle
     }
 
@@ -840,9 +843,10 @@ mod tests {
     #[ignore]
     #[tokio::test]
     async fn remove_expired() {
+        let tmp_dir = tempfile::tempdir().expect("Could not create temp dir");
         set_mock_time(real_time::now_ms());
 
-        let mut plumber = plumber().await;
+        let mut plumber = plumber(tmp_dir.path()).await;
 
         let particle = particle(now_ms(), 1);
         let deadline = Deadline::from(&particle);
@@ -877,10 +881,11 @@ mod tests {
     /// Checks that expired particle won't create an actor
     #[tokio::test]
     async fn ignore_expired() {
+        let tmp_dir = tempfile::tempdir().expect("Could not create temp dir");
         set_mock_time(real_time::now_ms());
         // set_mock_time(1000);
 
-        let mut plumber = plumber().await;
+        let mut plumber = plumber(tmp_dir.path()).await;
         let particle = particle(now_ms() - 100, 99);
         let deadline = Deadline::from(&particle);
         assert!(deadline.is_expired(now_ms()));

crates/chain-connector/src/connector.rs

@@ -512,7 +512,7 @@ impl HttpChainConnector {
         }
         .abi_encode();
         tracing::debug!(target: "chain-connector", "Registering worker {worker_id} for deal {deal_id} with cu_id {cu_id}");
-        self.send_tx(data, &deal_id.as_str()).await
+        self.send_tx(data, deal_id.as_str()).await
     }
 
     fn difficulty_params(&self) -> ArrayParams {

crates/core-distributor/Cargo.toml

@@ -20,7 +20,6 @@ bimap = { version = "0.6.3", features = ["serde"] }
 newtype_derive = "0.1.6"
 nonempty.workspace = true
 tokio = { workspace = true, features = ["fs", "rt", "sync", "macros", "tracing"] }
-async-trait.workspace = true
 enum_dispatch.workspace = true
 num_cpus.workspace = true
 parking_lot.workspace = true
@@ -33,7 +32,6 @@ rand = { workspace = true }
 hex.workspace = true
 serde_with = { workspace = true }
 hex-utils = { workspace = true, features = ["serde_with"] }
-derivative = { workspace = true }
 mockall = { version = "0.12.1", optional = true }
 
 [dev-dependencies]

crates/created-swarm/src/swarm.rs

@@ -307,8 +307,8 @@ pub struct SwarmConfig {
     pub pool_size: Option<usize>,
     pub builtins_dir: Option<PathBuf>,
     pub spell_base_dir: Option<PathBuf>,
-    pub allowed_binaries: Vec<String>,
-    pub allowed_effectors: HashMap<String, HashMap<String, String>>,
+    pub allowed_binaries: Vec<PathBuf>,
+    pub allowed_effectors: HashMap<String, HashMap<String, PathBuf>>,
     pub enabled_system_services: Vec<String>,
     pub extend_system_services: Vec<system_services::PackageDistro>,
     pub override_system_services_config: Option<system_services_config::SystemServicesConfig>,
@@ -338,7 +338,7 @@ impl SwarmConfig {
             pool_size: <_>::default(),
             builtins_dir: None,
             spell_base_dir: None,
-            allowed_binaries: vec!["/usr/bin/ipfs".to_string(), "/usr/bin/curl".to_string()],
+            allowed_binaries: vec!["/usr/bin/ipfs".into(), "/usr/bin/curl".into()],
             allowed_effectors: HashMap::new(),
             enabled_system_services: vec![],
             extend_system_services: vec![],

crates/nox-tests/tests/modules.rs

@@ -29,7 +29,7 @@ async fn test_add_module_mounted_binaries() {
     let swarms = make_swarms_with_cfg(1, move |mut cfg| {
         cfg.allowed_effectors = hashmap! {
             "bafkreiepzclggkt57vu7yrhxylfhaafmuogtqly7wel7ozl5k2ehkd44oe".to_string() => hashmap! {
-                "ls".to_string() => "/bin/ls".to_string()
+                "ls".to_string() => "/bin/ls".into()
             }
         };
         cfg

crates/nox-tests/tests/services.rs

@@ -154,7 +154,7 @@ async fn test_system_service_override() {
 #[tokio::test]
 async fn create_service_from_config() {
     let swarms = make_swarms_with_cfg(1, move |mut cfg| {
-        cfg.allowed_binaries = vec!["/does/not/exist".to_string()];
+        cfg.allowed_binaries = vec!["/does/not/exist".into()];
         cfg
     })
     .await;

crates/server-config/src/defaults.rs

@@ -270,28 +270,28 @@ pub fn default_decider_worker_gas() -> u64 {
     210_000
 }
 
-pub fn default_ipfs_binary_path() -> String {
-    "/usr/bin/ipfs".to_string()
+pub fn default_ipfs_binary_path() -> PathBuf {
+    "/usr/bin/ipfs".into()
 }
 
-pub fn default_curl_binary_path() -> String {
-    "/usr/bin/curl".to_string()
+pub fn default_curl_binary_path() -> PathBuf {
+    "/usr/bin/curl".into()
 }
 
 pub fn default_decider_network_id() -> u64 {
     // 80001 = polygon mumbai
     80001
 }
 
-pub fn default_effectors() -> HashMap<String, (String, HashMap<String, String>)> {
+pub fn default_effectors() -> HashMap<String, (String, HashMap<String, PathBuf>)> {
     hashmap! {
         "curl".to_string() => ("bafkreids22lgia5bqs63uigw4mqwhsoxvtnkpfqxqy5uwyyerrldsr32ce".to_string(), hashmap! {
             "curl".to_string() => default_curl_binary_path(),
         })
     }
 }
 
-pub fn default_binaries_mapping() -> BTreeMap<String, String> {
+pub fn default_binaries_mapping() -> BTreeMap<String, PathBuf> {
     btreemap! {
         "curl".to_string() => default_curl_binary_path(),
         "ipfs".to_string() => default_ipfs_binary_path(),

crates/server-config/src/node_config.rs

@@ -181,6 +181,8 @@ pub struct UnresolvedNodeConfig {
 
     #[serde(default)]
     pub network: Network,
+
+    pub vm: Option<VmConfig>,
 }
 
 #[serde_as]
@@ -281,6 +283,7 @@ impl UnresolvedNodeConfig {
             chain_listener_config: self.chain_listener_config,
             services: self.services,
             network: self.network,
+            vm: self.vm,
         };
 
         Ok(result)
@@ -450,7 +453,7 @@ pub struct NodeConfig {
     #[serde(serialize_with = "peer_id::serde::serialize")]
     pub management_peer_id: PeerId,
 
-    pub allowed_effectors: HashMap<Hash, HashMap<String, String>>,
+    pub allowed_effectors: HashMap<Hash, HashMap<String, PathBuf>>,
 
     pub dev_mode_config: DevModeConfig,
 
@@ -465,6 +468,8 @@ pub struct NodeConfig {
     pub services: ServicesConfig,
 
     pub network: Network,
+
+    pub vm: Option<VmConfig>,
 }
 
 #[derive(Clone, Deserialize, Serialize, Derivative, Copy)]
@@ -678,7 +683,7 @@ pub struct EffectorsConfig(HashMap<EffectorModuleName, EffectorConfig>);
 pub struct EffectorConfig {
     #[derivative(Debug(format_with = "std::fmt::Display::fmt"))]
     wasm_cid: Hash,
-    allowed_binaries: HashMap<String, String>,
+    allowed_binaries: HashMap<String, PathBuf>,
 }
 
 fn default_effectors_config() -> EffectorsConfig {
@@ -704,7 +709,13 @@ pub struct DevModeConfig {
     pub enable: bool,
     /// Mounted binaries mapping: binary name (used in the effector modules) to binary path
     #[serde(default = "default_binaries_mapping")]
-    pub binaries: BTreeMap<String, String>,
+    pub binaries: BTreeMap<String, PathBuf>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
+pub struct VmConfig {
+    pub libvirt_uri: String,
+    pub bridge_name: String,
 }
 
 fn default_dev_mode_config() -> DevModeConfig {