From 92421f02355d048fdee2c3478b8348d340a2aaff Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Mon, 16 Sep 2024 21:39:19 +0000 Subject: [PATCH] deprecate: remove 'shared' mode The 'shared' mode in runwasi is not actively maintained and tested. This commit is removing it. The 'shared' mode will soon be replaced with the Sandboxer API described in #385 Signed-off-by: jiaxiao zhou --- Cargo.lock | 5 - README.md | 66 +---- .../Cargo.toml | 5 - crates/containerd-shim-wasm/build.rs | 35 --- .../containerd-shim-wasm/protos/sandbox.proto | 41 --- crates/containerd-shim-wasm/src/lib.rs | 1 - .../containerd-shim-wasm/src/sandbox/cli.rs | 40 +-- .../src/sandbox/manager.rs | 262 ------------------ .../containerd-shim-wasm/src/sandbox/mod.rs | 4 - .../src/sandbox/shim/local.rs | 19 +- .../src/sandbox/shim/mod.rs | 1 - crates/containerd-shim-wasm/src/services.rs | 6 - 12 files changed, 5 insertions(+), 480 deletions(-) delete mode 100644 crates/containerd-shim-wasm/build.rs delete mode 100644 crates/containerd-shim-wasm/protos/sandbox.proto delete mode 100644 crates/containerd-shim-wasm/src/sandbox/manager.rs delete mode 100644 crates/containerd-shim-wasm/src/services.rs diff --git a/Cargo.lock b/Cargo.lock index 40b21f81d..acdb0c5b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,13 +766,8 @@ name = "containerd-shim-wasm-test-modules" version = "0.4.1" dependencies = [ "anyhow", - "env_logger", "lazy_static", "libc", - "log", - "oci-spec", - "serde_json", - "tempfile", "wat", ] diff --git a/README.md b/README.md index 722d875a5..c0517fad5 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,9 @@ on the CNCF slack. ## Usage -runwasi is intended to be consumed as a library to be linked to from your own wasm host implementation. +runwasi is intended to be consumed as a library to be linked to from your own wasm host implementation. It creates one shim process per container or k8s pod. -There are two modes of operation supported: - -1. "Normal" mode where there is 1 shim process per container or k8s pod. -2. "Shared" mode where there is a single manager service running all shims in process. - -In either case you need to implement a trait to teach runwasi how to use your wasm host. +You need to implement a trait to teach runwasi how to use your wasm host. There are two ways to do this: * implementing the `sandbox::Instance` trait @@ -119,45 +114,6 @@ I encourage you to checkout how that is implemented. The shim binary just needs to be installed into `$PATH` (as seen by the containerd process) with a binary name like `containerd-shim-myshim-v1`. -For the shared mode: - -```rust -use containerd_shim_wasm::sandbox::{Local, ManagerService, Instance}; -use containerd_shim_wasm::services::sandbox_ttrpc::{create_manager, Manager}; -use std::sync::Arc; -use ttrpc::{self, Server}; -/// ... - -struct MyInstance { - /// ... -} - -impl Instance for MyInstance { - // ... -} - -fn main() { - let s: ManagerService> = - ManagerService::new(Engine::new(Config::new().interruptable(true)).unwrap()); - let s = Arc::new(Box::new(s) as Box); - let service = create_manager(s); - - let mut server = Server::new() - .bind("unix:///run/io.containerd.myshim.v1/manager.sock") - .unwrap() - .register_service(service); - - server.start().unwrap(); - let (_tx, rx) = std::sync::mpsc::channel::<()>(); - rx.recv().unwrap(); -} -``` - -This will be the host daemon that you startup and manage on your own. -You can use the provided `containerd-shim-myshim-v1` binary as the shim to specify in containerd. - -Shared mode requires precise control over real threads and as such should not be used with an async runtime. - Check out these projects that build on top of runwasi: - [spinkube/containerd-shim-spin](https://github.com/spinkube/containerd-shim-spin) - [deislabs/containerd-wasm-shims](https://github.com/deislabs/containerd-wasm-shims) @@ -174,24 +130,6 @@ And make sure the shim binary must be in $PATH (that is the $PATH that container This shim runs one per pod. -- **containerd-shim-[ wasmedge | wasmtime | wasmer ]d-v1** - -A cli used to connect containerd to the `containerd-[ wasmedge | wasmtime | wasmer ]d` sandbox daemon. -When containerd requests for a container to be created, it fires up this shim binary which will connect to the `containerd-[ wasmedge | wasmtime | wasmer ]d` service running on the host. -The service will return a path to a unix socket which this shim binary will write back to containerd which containerd will use to connect to for shim requests. -This binary does not serve requests, it is only responsible for sending requests to the `containerd-[ wasmedge | wasmtime | wasmer ]d` daemon to create or destroy sandboxes. - -- **containerd-[ wasmedge | wasmtime | wasmer ]d** - -This is a sandbox manager that enables running 1 wasm host for the entire node instead of one per pod (or container). -When a container is created, a request is sent to this service to create a sandbox. -The "sandbox" is a containerd task service that runs in a new thread on its own unix socket, which we return back to containerd to connect to. - -The Wasmedge / Wasmtime / Wasmer engine is shared between all sandboxes in the service. - -To use this shim, specify `io.containerd.[ wasmedge | wasmtime | wasmer ]d.v1` as the runtime to use. -You will need to make sure the `containerd-[ wasmedge | wasmtime | wasmer ]d` daemon has already been started. - ## Contributing To begin contributing, learn to build and test the project or to add a new shim please read our [CONTRIBUTING.md](./CONTRIBUTING.md) diff --git a/crates/containerd-shim-wasm-test-modules/Cargo.toml b/crates/containerd-shim-wasm-test-modules/Cargo.toml index 7a3db8e63..a5f3572ce 100644 --- a/crates/containerd-shim-wasm-test-modules/Cargo.toml +++ b/crates/containerd-shim-wasm-test-modules/Cargo.toml @@ -7,12 +7,7 @@ license.workspace = true [dependencies] anyhow = { workspace = true } -env_logger = { workspace = true } libc = { workspace = true } -log = { workspace = true } -oci-spec = { workspace = true } -serde_json = { workspace = true } -tempfile = { workspace = true } [build-dependencies] anyhow = { workspace = true } diff --git a/crates/containerd-shim-wasm/build.rs b/crates/containerd-shim-wasm/build.rs deleted file mode 100644 index 3c22a1195..000000000 --- a/crates/containerd-shim-wasm/build.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::env::var_os; -use std::path::Path; - -use ttrpc_codegen::{Codegen, ProtobufCustomize}; - -fn main() { - let protos = ["protos/sandbox.proto"]; - println!("cargo:rerun-if-changed=protos/sandbox.proto"); - - let out_dir = var_os("OUT_DIR").unwrap(); - let out_dir = Path::new(&out_dir); - - Codegen::new() - .out_dir(out_dir) - .inputs(protos) - .include("protos") - .rust_protobuf() - .rust_protobuf_customize(ProtobufCustomize::default().gen_mod_rs(false)) - .run() - .expect("failed to generate code"); - - let sanbox_rs = out_dir.join("sandbox.rs"); - let sanbox_ttrpc_rs = out_dir.join("sandbox_ttrpc.rs"); - - std::fs::write( - out_dir.join("mod.rs"), - format!( - r#" -#[path = {sanbox_rs:?}] pub mod sandbox; -#[path = {sanbox_ttrpc_rs:?}] pub mod sandbox_ttrpc; -"#, - ), - ) - .expect("failed to generate module"); -} diff --git a/crates/containerd-shim-wasm/protos/sandbox.proto b/crates/containerd-shim-wasm/protos/sandbox.proto deleted file mode 100644 index 733ee2cc1..000000000 --- a/crates/containerd-shim-wasm/protos/sandbox.proto +++ /dev/null @@ -1,41 +0,0 @@ -syntax = "proto3"; - -package runwasi.services.sandbox.v1; -option go_package = "github.com/cpuguy83/runwasi/api/services/sandbox/v1;sandbox"; - -service Manager { - rpc Create(CreateRequest) returns (CreateResponse) {} - rpc Connect(ConnectRequest) returns (ConnectResponse) {} - rpc Delete(DeleteRequest) returns (DeleteResponse) {} -} - -message CreateRequest { - string namespace = 1; - string id = 2; - string ttrpc_address = 3; - string working_directory = 4; - string containerd_address = 5; -} - -message CreateResponse { - // Path of the socket file serving the shim API - string socket_path = 1; -} - -message ConnectRequest { - string id = 1; - string ttrpc_address = 2; -} - -message ConnectResponse { - // Path of the socket file serving the shim API - string socket_path = 1; -} - -message DeleteRequest { - string namespace = 1; - string id = 2; - string ttrpc_address = 3; -} - -message DeleteResponse {} diff --git a/crates/containerd-shim-wasm/src/lib.rs b/crates/containerd-shim-wasm/src/lib.rs index 5d7a33158..01cbcc179 100644 --- a/crates/containerd-shim-wasm/src/lib.rs +++ b/crates/containerd-shim-wasm/src/lib.rs @@ -5,7 +5,6 @@ pub mod container; pub mod sandbox; -pub mod services; #[cfg_attr(unix, path = "sys/unix/mod.rs")] #[cfg_attr(windows, path = "sys/windows/mod.rs")] diff --git a/crates/containerd-shim-wasm/src/sandbox/cli.rs b/crates/containerd-shim-wasm/src/sandbox/cli.rs index 732f0a0d6..80b90f3e7 100644 --- a/crates/containerd-shim-wasm/src/sandbox/cli.rs +++ b/crates/containerd-shim-wasm/src/sandbox/cli.rs @@ -1,16 +1,10 @@ use std::path::PathBuf; -use std::sync::mpsc::channel; -use std::sync::Arc; use containerd_shim::{parse, run, Config}; -use ttrpc::Server; -use crate::sandbox::manager::Shim; -use crate::sandbox::shim::Local; #[cfg(feature = "opentelemetry")] use crate::sandbox::shim::{otel_traces_enabled, OTLPConfig}; -use crate::sandbox::{Instance, ManagerService, ShimCli}; -use crate::services::sandbox_ttrpc::{create_manager, Manager}; +use crate::sandbox::{Instance, ShimCli}; pub mod r#impl { pub use git_version::git_version; @@ -115,37 +109,7 @@ fn shim_main_inner<'a, I>( let shim_version = shim_version.into().unwrap_or("v1"); let lower_name = name.to_lowercase(); - let shim_cli = format!("containerd-shim-{lower_name}-{shim_version}"); - let shim_client = format!("containerd-shim-{lower_name}d-{shim_version}"); - let shim_daemon = format!("containerd-{lower_name}d"); let shim_id = format!("io.containerd.{lower_name}.{shim_version}"); - match argv0.to_lowercase() { - s if s == shim_cli => { - run::>(&shim_id, config); - } - s if s == shim_client => { - run::(&shim_client, config); - } - s if s == shim_daemon => { - log::info!("starting up!"); - let s: ManagerService> = Default::default(); - let s = Arc::new(Box::new(s) as Box); - let service = create_manager(s); - - let mut server = Server::new() - .bind("unix:///run/io.containerd.wasmwasi.v1/manager.sock") - .expect("failed to bind to socket") - .register_service(service); - - server.start().expect("failed to start daemon"); - log::info!("server started!"); - let (_tx, rx) = channel::<()>(); - rx.recv().unwrap(); - } - _ => { - eprintln!("error: unrecognized binary name, expected one of {shim_cli}, {shim_client}, or {shim_daemon}."); - std::process::exit(1); - } - } + run::>(&shim_id, config); } diff --git a/crates/containerd-shim-wasm/src/sandbox/manager.rs b/crates/containerd-shim-wasm/src/sandbox/manager.rs deleted file mode 100644 index a9c48f26f..000000000 --- a/crates/containerd-shim-wasm/src/sandbox/manager.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! This experimental module implements a manager service which can be used to -//! manage multiple instances of a sandbox in-process. -//! The idea behind this module is to only need a single shim process for the entire node rather than one per pod/container. - -use std::collections::HashMap; -use std::env::current_dir; -use std::path::Path; -use std::sync::{Arc, RwLock}; -use std::thread; - -use anyhow::Context; -use containerd_shim::error::Error as ShimError; -use containerd_shim::protos::shim::shim_ttrpc::{create_task, Task}; -use containerd_shim::protos::ttrpc::{Client, Server}; -use containerd_shim::protos::TaskClient; -use containerd_shim::publisher::RemotePublisher; -use containerd_shim::{self as shim, api, TtrpcContext, TtrpcResult}; -use oci_spec::runtime::{self, Spec}; -use shim::Flags; -use ttrpc::context; - -use super::error::Error; -use super::instance::Instance; -use super::sandbox; -use crate::services::sandbox_ttrpc::{Manager, ManagerClient}; -use crate::sys::networking::setup_namespaces; - -/// Sandbox wraps an Instance and is used with the `Service` to manage multiple instances. -pub trait Sandbox: Task + Send + Sync { - type Instance: Instance; - - fn new( - namespace: String, - containerd_address: String, - id: String, - engine: ::Engine, - publisher: RemotePublisher, - ) -> Self; -} - -/// Service is a manager service which can be used to manage multiple instances of a sandbox in-process. -pub struct Service { - sandboxes: RwLock>, - engine: ::Engine, - phantom: std::marker::PhantomData, -} - -impl Service { - pub fn new(engine: ::Engine) -> Self { - Self { - sandboxes: RwLock::new(HashMap::new()), - engine, - phantom: std::marker::PhantomData, - } - } -} - -impl Default for Service -where - ::Engine: Default, -{ - fn default() -> Self { - Self::new(Default::default()) - } -} - -impl Manager for Service { - fn create( - &self, - _ctx: &TtrpcContext, - req: sandbox::CreateRequest, - ) -> TtrpcResult { - let mut sandboxes = self.sandboxes.write().unwrap(); - - if sandboxes.contains_key(&req.id) { - return Err(Error::AlreadyExists(req.id).into()); - } - - let sock = format!("unix://{}/shim.sock", &req.working_directory); - - let publisher = RemotePublisher::new(req.ttrpc_address)?; - - let sb = T::new( - req.namespace.clone(), - req.containerd_address.clone(), - req.id.clone(), - self.engine.clone(), - publisher, - ); - let task_service = create_task(Arc::new(Box::new(sb))); - let mut server = Server::new().bind(&sock)?.register_service(task_service); - - sandboxes.insert(req.id.clone(), sock.clone()); - - let cfg = Spec::load( - Path::new(&req.working_directory) - .join("config.json") - .to_str() - .unwrap(), - ) - .map_err(|err| Error::InvalidArgument(format!("could not load runtime spec: {}", err)))?; - - let (tx, rx) = std::sync::mpsc::channel::>(); - - let id = &req.id; - - let _ = thread::Builder::new() - .name(format!("{}-sandbox-create", id)) - .spawn(move || { - let r = start_sandbox(cfg, &mut server); - tx.send(r).context("could not send sandbox result").unwrap(); - }) - .context("failed to spawn sandbox thread") - .map_err(Error::from)?; - - rx.recv() - .context("could not receive sandbox result") - .map_err(Error::from)??; - Ok(sandbox::CreateResponse { - socket_path: sock, - ..Default::default() - }) - } - - fn delete( - &self, - _ctx: &TtrpcContext, - req: sandbox::DeleteRequest, - ) -> TtrpcResult { - let mut sandboxes = self.sandboxes.write().unwrap(); - if !sandboxes.contains_key(&req.id) { - return Err(Error::NotFound(req.id).into()); - } - let sock = sandboxes.remove(&req.id).unwrap(); - let c = Client::connect(&sock)?; - let tc = TaskClient::new(c); - - tc.shutdown( - context::Context::default(), - &api::ShutdownRequest { - id: req.id, - now: true, - ..Default::default() - }, - )?; - - Ok(sandbox::DeleteResponse::default()) - } -} - -// Note that this changes the current thread's state. -// You probably want to run this in a new thread. -fn start_sandbox(cfg: runtime::Spec, server: &mut Server) -> Result<(), Error> { - setup_namespaces(&cfg)?; - - server.start_listen().context("could not start listener")?; - Ok(()) -} - -/// Shim implements the containerd-shim CLI for connecting to a Manager service. -pub struct Shim { - id: String, - namespace: String, -} - -impl Task for Shim {} - -impl shim::Shim for Shim { - type T = Self; - - fn new(_runtime_id: &str, args: &Flags, _config: &mut shim::Config) -> Self { - Shim { - id: args.id.to_string(), - namespace: args.namespace.to_string(), - } - } - - fn start_shim(&mut self, opts: containerd_shim::StartOpts) -> shim::Result { - let dir = current_dir().map_err(|err| ShimError::Other(err.to_string()))?; - let spec = Spec::load(dir.join("config.json").to_str().unwrap()).map_err(|err| { - shim::Error::InvalidArgument(format!("error loading runtime spec: {}", err)) - })?; - - let default = HashMap::new() as HashMap; - let annotations = spec.annotations().as_ref().unwrap_or(&default); - - let sandbox = annotations - .get("io.kubernetes.cri.sandbox-id") - .unwrap_or(&opts.id) - .to_string(); - - let client = Client::connect("unix:///run/io.containerd.wasmwasi.v1/manager.sock")?; - let mc = ManagerClient::new(client); - - let addr = match mc.create( - context::Context::default(), - &sandbox::CreateRequest { - id: sandbox.clone(), - working_directory: dir.as_path().to_str().unwrap().to_string(), - ttrpc_address: opts.ttrpc_address.clone(), - ..Default::default() - }, - ) { - Ok(res) => res.socket_path, - Err(_) => { - let res = mc.connect( - context::Context::default(), - &sandbox::ConnectRequest { - id: sandbox, - ttrpc_address: opts.ttrpc_address, - ..Default::default() - }, - )?; - res.socket_path - } - }; - - shim::util::write_address(&addr)?; - - Ok(addr) - } - - fn wait(&mut self) { - todo!() - } - - fn create_task_service(&self, _publisher: RemotePublisher) -> Self::T { - todo!() // but not really, haha - } - - fn delete_shim(&mut self) -> shim::Result { - let dir = current_dir().map_err(|err| ShimError::Other(err.to_string()))?; - let spec = Spec::load(dir.join("config.json").to_str().unwrap()).map_err(|err| { - shim::Error::InvalidArgument(format!("error loading runtime spec: {}", err)) - })?; - - let default = HashMap::new() as HashMap; - let annotations = spec.annotations().as_ref().unwrap_or(&default); - - let sandbox = annotations - .get("io.kubernetes.cri.sandbox-id") - .unwrap_or(&self.id) - .to_string(); - if sandbox != self.id { - return Ok(api::DeleteResponse::default()); - } - - let client = Client::connect("unix:///run/io.containerd.wasmwasi.v1/manager.sock")?; - let mc = ManagerClient::new(client); - mc.delete( - context::Context::default(), - &sandbox::DeleteRequest { - id: sandbox, - namespace: self.namespace.clone(), - ..Default::default() - }, - )?; - - // TODO: write pid, exit code, etc to disk so we can use it here. - Ok(api::DeleteResponse::default()) - } -} diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 872715717..157bd9009 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -1,19 +1,15 @@ //! Abstracts the sandboxing environment and execution context for a container. -use crate::services::sandbox; - pub mod cli; pub mod error; pub mod instance; pub mod instance_utils; -pub mod manager; pub mod shim; pub mod stdio; pub mod sync; pub use error::{Error, Result}; pub use instance::{Instance, InstanceConfig}; -pub use manager::{Sandbox as SandboxService, Service as ManagerService}; pub use shim::Cli as ShimCli; pub use stdio::Stdio; diff --git a/crates/containerd-shim-wasm/src/sandbox/shim/local.rs b/crates/containerd-shim-wasm/src/sandbox/shim/local.rs index dab46dd5b..41158f4c1 100644 --- a/crates/containerd-shim-wasm/src/sandbox/shim/local.rs +++ b/crates/containerd-shim-wasm/src/sandbox/shim/local.rs @@ -16,7 +16,6 @@ use containerd_shim::error::Error as ShimError; use containerd_shim::protos::events::task::{TaskCreate, TaskDelete, TaskExit, TaskIO, TaskStart}; use containerd_shim::protos::shim::shim_ttrpc::Task; use containerd_shim::protos::types::task::Status; -use containerd_shim::publisher::RemotePublisher; use containerd_shim::util::IntoOption; use containerd_shim::{DeleteResponse, ExitSignal, TtrpcContext, TtrpcResult}; use log::debug; @@ -25,7 +24,7 @@ use oci_spec::runtime::Spec; use crate::sandbox::instance::{Instance, InstanceConfig}; use crate::sandbox::shim::events::{EventSender, RemoteEventSender, ToTimestamp}; use crate::sandbox::shim::instance_data::InstanceData; -use crate::sandbox::{oci, Error, Result, SandboxService}; +use crate::sandbox::{oci, Error, Result}; use crate::sys::metrics::get_metrics; #[cfg(test)] @@ -343,22 +342,6 @@ impl Local { } } -impl SandboxService for Local { - type Instance = T; - #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))] - fn new( - namespace: String, - containerd_address: String, - _id: String, - engine: T::Engine, - publisher: RemotePublisher, - ) -> Self { - let events = RemoteEventSender::new(&namespace, publisher); - let exit = Arc::default(); - Local::::new(engine, events, exit, namespace, containerd_address) - } -} - impl Task for Local { #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))] fn create(&self, _: &TtrpcContext, req: CreateTaskRequest) -> TtrpcResult { diff --git a/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs b/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs index a5ce3eb1a..851f749a8 100644 --- a/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/shim/mod.rs @@ -12,6 +12,5 @@ mod otel; mod task_state; pub use cli::Cli; -pub(crate) use local::Local; #[cfg(feature = "opentelemetry")] pub use otel::{traces_enabled as otel_traces_enabled, Config as OTLPConfig}; diff --git a/crates/containerd-shim-wasm/src/services.rs b/crates/containerd-shim-wasm/src/services.rs deleted file mode 100644 index 6850a5f4f..000000000 --- a/crates/containerd-shim-wasm/src/services.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Generated service definitions from the protobuf definitions. - -// do not run clippy on generated files -#![allow(clippy::all)] - -include!(concat!(env!("OUT_DIR"), "/mod.rs"));