diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..ac4d9de29 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[target.'cfg(windows)'] +rustflags = [ + "-Adead_code", + "-Awarnings", +] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9f0f0cc8..b3d001b98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,10 @@ env: jobs: fmt: - runs-on: "ubuntu-latest" + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest"] + runs-on: ${{ matrix.os }} steps: - name: Free Disk Space (Ubuntu) uses: insightsengineering/disk-space-reclaimer@v1 @@ -27,18 +30,27 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: rustfmt, clippy + rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile - name: Setup build env - run: ./scripts/setup.sh + run: | + os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') + ./scripts/setup-$os.sh shell: bash - run: # needed to run rustfmt in nightly toolchain rustup toolchain install nightly --component rustfmt + - name: Set environment variables for Windows + if: runner.os == 'Windows' + run: | + # required until standalong is implemented for windows (https://github.com/WasmEdge/wasmedge-rust-sdk/issues/54) + echo "WASMEDGE_LIB_DIR=C:\Program Files\WasmEdge\lib" >> $env:GITHUB_ENV + echo "WASMEDGE_INCLUDE_DIR=C:\Program Files\WasmEdge\include" >> $env:GITHUB_ENV - name: Run checks run: make check build: strategy: matrix: - os: ["ubuntu-20.04", "ubuntu-22.04"] + os: ["ubuntu-20.04", "ubuntu-22.04", "windows-latest"] runs-on: ${{ matrix.os }} steps: - name: Free Disk Space (Ubuntu) @@ -53,18 +65,25 @@ jobs: docker-images: true - name: "check cgroup version" run: "mount | grep cgroup" + if: runner.os == 'Linux' - uses: actions/checkout@v3 - uses: actions-rust-lang/setup-rust-toolchain@v1 env: RUST_CACHE_KEY_OS: ${{ matrix.os }} + with: + rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile - name: Setup build env - run: ./scripts/setup.sh + run: | + os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') + ./scripts/setup-$os.sh shell: bash + - name: Set environment variables for Windows + if: runner.os == 'Windows' + run: | + echo "WASMEDGE_LIB_DIR=C:\Program Files\WasmEdge\lib" >> $env:GITHUB_ENV + echo "WASMEDGE_INCLUDE_DIR=C:\Program Files\WasmEdge\include" >> $env:GITHUB_ENV - name: Build - uses: actions-rs/cargo@v1 - with: - command: build - args: --all --verbose + run: make build - name: Validate docs run: ./scripts/validate-docs.sh - name: Run tests @@ -95,7 +114,7 @@ jobs: - name: setup rust-wasm target run: rustup target add wasm32-wasi - name: Setup build env - run: ./scripts/setup.sh + run: ./scripts/setup-linux.sh shell: bash - name: run run: make test/k8s @@ -132,7 +151,7 @@ jobs: - name: setup rust-wasm target run: rustup target add wasm32-wasi - name: Setup build env - run: ./scripts/setup.sh + run: ./scripts/setup-linux.sh shell: bash - name: run run: make test/k3s diff --git a/Makefile b/Makefile index 6be84159b..74b07b7c7 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,15 @@ ifeq ($(TARGET),release) RELEASE_FLAG = --release endif +FEATURES := --features libcontainer_default +WARNINGS := -D warnings +ifeq ($(OS), Windows_NT) +# need to turn off static/standalone for wasm-edge +FEATURES = --no-default-features +# turn of warnings until windows is fully supported #49 +WARNINGS = +endif + DOCKER_BUILD ?= docker buildx build KIND_CLUSTER_NAME ?= containerd-wasm @@ -17,25 +26,26 @@ KIND_CLUSTER_NAME ?= containerd-wasm .PHONY: build build: cargo build -p containerd-shim-wasm --features generate_bindings $(RELEASE_FLAG) - # compiling against libcontainer's default features (dependency on libseccomp) - cargo build -p containerd-shim-wasm --features libcontainer_default $(RELEASE_FLAG) - cargo build $(RELEASE_FLAG) + cargo build -p containerd-shim-wasm $(FEATURES) $(RELEASE_FLAG) + cargo build $(FEATURES) $(RELEASE_FLAG) .PHONY: check check: cargo +nightly fmt --all -- --check - cargo clippy --all --all-targets -- -D warnings + cargo clippy $(FEATURES) --all --all-targets -- $(WARNINGS) .PHONY: fix fix: cargo +nightly fmt --all - cargo clippy --fix --all --all-targets -- -D warnings + cargo clippy $(FEATURES) --fix --all --all-targets -- $(WARNINGS) .PHONY: test test: - RUST_LOG=trace cargo test --all --verbose -- --nocapture + RUST_LOG=trace cargo test $(FEATURES) --all --verbose -- --nocapture +ifneq ($(OS), Windows_NT) # run wasmedge test without the default `static` feature RUST_LOG=trace cargo test --package containerd-shim-wasmedge --verbose --no-default-features --features standalone -- --nocapture +endif .PHONY: install install: diff --git a/README.md b/README.md index 78f99d946..c5bc4ea83 100644 --- a/README.md +++ b/README.md @@ -146,12 +146,18 @@ You will need to make sure the `containerd-[ wasmedge | wasmtime ]d` daemon has #### Building -1. Install [youki dependencies](https://github.com/containers/youki#dependencies) +1. Install dependencies -If on ubuntu/debian you can use the following script. Refer to youki's documentation for other systems. +If on ubuntu/debian you can use the following script. Refer to [youki's](https://github.com/containers/youki#dependencies) documentation for other systems. ``` -./scripts/setup.sh +./scripts/setup-linux.sh +``` + +If on Windows use (use [git BASH](https://gitforwindows.org/) terminal which has shell emulator) + +``` +./scripts/setup-windows.sh ``` 2. Build diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 91dc35c2b..fe2f25b2e 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -16,20 +16,22 @@ containerd-shim = { workspace = true } anyhow = { workspace = true } serde_json = { workspace = true } oci-spec = { workspace = true } -command-fds = "0.2" serde = { workspace = true } thiserror = { workspace = true } protobuf = "3.2" ttrpc = { workspace = true } -nix = { workspace = true } chrono = { workspace = true } log = { workspace = true } -clone3 = "0.2" libc = { workspace = true } + +[target.'cfg(unix)'.dependencies] +clone3 = "0.2" caps = "0.5" +command-fds = "0.2" proc-mounts = "0.3" libcontainer = { workspace = true, optional = true, default-features = false } cgroups-rs = "0.3.3" +nix = { workspace = true } [build-dependencies] ttrpc-codegen = { version = "0.4.2", optional = true } @@ -50,4 +52,4 @@ libseccomp = ["libcontainer/libseccomp"] systemd = ["libcontainer/systemd"] cgroupsv2 = ["libcontainer/v2"] cgroupsv1 = ["libcontainer/v1"] -cgroupsv2_devices = ["libcontainer/cgroupsv2_devices"] \ No newline at end of file +cgroupsv2_devices = ["libcontainer/cgroupsv2_devices"] diff --git a/crates/containerd-shim-wasm/src/lib.rs b/crates/containerd-shim-wasm/src/lib.rs index 86973ce8c..84b78b09f 100644 --- a/crates/containerd-shim-wasm/src/lib.rs +++ b/crates/containerd-shim-wasm/src/lib.rs @@ -4,8 +4,10 @@ )] pub mod sandbox; - pub mod services; +#[cfg_attr(unix, path = "sys/unix.rs")] +#[cfg_attr(windows, path = "sys/windows.rs")] +pub mod sys; -#[cfg(feature = "libcontainer")] +#[cfg(all(feature = "libcontainer", not(target_os = "windows")))] pub mod libcontainer_instance; diff --git a/crates/containerd-shim-wasm/src/sandbox/error.rs b/crates/containerd-shim-wasm/src/sandbox/error.rs index 67f5d93c1..3eecd1f6f 100644 --- a/crates/containerd-shim-wasm/src/sandbox/error.rs +++ b/crates/containerd-shim-wasm/src/sandbox/error.rs @@ -39,6 +39,7 @@ pub enum Error { #[error("{0}")] Json(#[from] serde_json::Error), /// Error from the system + #[cfg(unix)] #[error("{0}")] Errno(#[from] nix::errno::Errno), } diff --git a/crates/containerd-shim-wasm/src/sandbox/instance.rs b/crates/containerd-shim-wasm/src/sandbox/instance.rs index 79656410a..e25de653b 100644 --- a/crates/containerd-shim-wasm/src/sandbox/instance.rs +++ b/crates/containerd-shim-wasm/src/sandbox/instance.rs @@ -5,12 +5,17 @@ use std::sync::{Arc, Condvar, Mutex}; use std::thread; use chrono::{DateTime, Utc}; -use libc::{SIGINT, SIGKILL, SIGTERM}; +use libc::{SIGINT, SIGTERM}; use super::error::Error; pub type ExitCode = Arc<(Mutex)>>, Condvar)>; +#[cfg(unix)] +use libc::SIGKILL; +#[cfg(windows)] +const SIGKILL: i32 = 9; + /// Generic options builder for creating a wasm instance. /// This is passed to the `Instance::new` method. #[derive(Clone)] @@ -216,6 +221,7 @@ mod noptests { use std::sync::mpsc::channel; use std::time::Duration; + #[cfg(unix)] use libc::SIGHUP; use super::*; @@ -259,6 +265,7 @@ mod noptests { Ok(()) } + #[cfg(unix)] #[test] fn test_op_kill_other() -> Result<(), Error> { let nop = Nop::new("".to_string(), None); diff --git a/crates/containerd-shim-wasm/src/sandbox/manager.rs b/crates/containerd-shim-wasm/src/sandbox/manager.rs index 960e8a0e7..5f5097f9d 100644 --- a/crates/containerd-shim-wasm/src/sandbox/manager.rs +++ b/crates/containerd-shim-wasm/src/sandbox/manager.rs @@ -4,8 +4,6 @@ use std::collections::HashMap; use std::env::current_dir; -use std::fs::File; -use std::os::unix::io::AsRawFd; use std::path::Path; use std::sync::{Arc, RwLock}; use std::thread; @@ -17,7 +15,6 @@ use containerd_shim::protos::ttrpc::{Client, Server}; use containerd_shim::protos::TaskClient; use containerd_shim::publisher::RemotePublisher; use containerd_shim::{self as shim, api, TtrpcContext, TtrpcResult}; -use nix::sched::{setns, unshare, CloneFlags}; use oci_spec::runtime; use shim::Flags; use ttrpc::context; @@ -26,6 +23,7 @@ use super::error::Error; use super::instance::Instance; use super::{oci, sandbox}; use crate::services::sandbox_ttrpc::{Manager, ManagerClient}; +use crate::sys::setup_namespaces; /// Sandbox wraps an Instance and is used with the `Service` to manage multiple instances. pub trait Sandbox: Task + Send + Sync { @@ -156,20 +154,7 @@ impl Manager for Service { // Note that this changes the current thread's state. // You probably want to run this in a new thread. fn start_sandbox(cfg: runtime::Spec, server: &mut Server) -> Result<(), Error> { - let namespaces = cfg.linux().as_ref().unwrap().namespaces().as_ref().unwrap(); - for ns in namespaces { - if ns.typ() == runtime::LinuxNamespaceType::Network { - if ns.path().is_some() { - let p = ns.path().clone().unwrap(); - let f = File::open(p).context("could not open network namespace")?; - setns(f.as_raw_fd(), CloneFlags::CLONE_NEWNET) - .context("error setting network namespace")?; - break; - } - - unshare(CloneFlags::CLONE_NEWNET).context("error unsharing network namespace")?; - } - } + setup_namespaces(&cfg)?; server.start_listen().context("could not start listener")?; Ok(()) diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index 08a3b40a2..1baddeda4 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -3,13 +3,12 @@ use std::collections::HashMap; use std::fs::File; use std::io::{ErrorKind, Write}; +#[cfg(unix)] use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process; use anyhow::Context; -use nix::sys::signal; -use nix::unistd::Pid; pub use oci_spec::runtime::Spec; use serde_json as json; @@ -96,14 +95,28 @@ pub fn setup_prestart_hooks(hooks: &Option) -> Result< // Based on OCI spec, the first argument of the args vector is the // arg0, which can be different from the path. For example, path // may be "/usr/bin/true" and arg0 is set to "true". However, rust - // command differenciates arg0 from args, where rust command arg + // command differentiates arg0 from args, where rust command arg // doesn't include arg0. So we have to make the split arg0 from the // rest of args. if let Some((arg0, args)) = hook.args().as_ref().and_then(|a| a.split_first()) { log::debug!("run_hooks arg0: {:?}, args: {:?}", arg0, args); - hook_command.arg0(arg0).args(args) + + #[cfg(unix)] + { + hook_command.arg0(arg0).args(args); + } + + #[cfg(windows)] + { + if !&hook.path().ends_with(arg0) { + return Err(crate::sandbox::Error::InvalidArgument("Running with arg0 as different name than executable is not supported on Windows due to rust std library process implementation.".to_string())); + } + + hook_command.args(args); + } } else { - hook_command.arg0(&hook.path().display().to_string()) + #[cfg(unix)] + hook_command.arg0(&hook.path().display().to_string()); }; let envs: HashMap = if let Some(env) = hook.env() { @@ -119,7 +132,6 @@ pub fn setup_prestart_hooks(hooks: &Option) -> Result< .stdin(process::Stdio::piped()) .spawn() .with_context(|| "Failed to execute hook")?; - let hook_process_pid = Pid::from_raw(hook_process.id() as i32); if let Some(stdin) = &mut hook_process.stdin { // We want to ignore BrokenPipe here. A BrokenPipe indicates @@ -135,7 +147,7 @@ pub fn setup_prestart_hooks(hooks: &Option) -> Result< if e.kind() != ErrorKind::BrokenPipe { // Not a broken pipe. The hook command may be waiting // for us. - let _ = signal::kill(hook_process_pid, signal::Signal::SIGKILL); + let _ = hook_process.kill(); } } } diff --git a/crates/containerd-shim-wasm/src/sandbox/shim.rs b/crates/containerd-shim-wasm/src/sandbox/shim.rs index a70055187..5713098a6 100644 --- a/crates/containerd-shim-wasm/src/sandbox/shim.rs +++ b/crates/containerd-shim-wasm/src/sandbox/shim.rs @@ -4,20 +4,19 @@ use std::collections::HashMap; use std::env::current_dir; -use std::fs::{self, canonicalize, create_dir_all, File, OpenOptions}; +use std::fs::{self, canonicalize, create_dir_all, DirBuilder, File, OpenOptions}; use std::ops::Not; -use std::os::unix::io::AsRawFd; +#[cfg(unix)] +use std::os::unix::fs::DirBuilderExt; use std::path::Path; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Condvar, Mutex, RwLock}; use std::thread; -use cgroups_rs::cgroup::get_cgroups_relative_paths_by_pid; -use cgroups_rs::hierarchies::{self}; -use cgroups_rs::{Cgroup, Subsystem}; use chrono::{DateTime, Utc}; use containerd_shim::error::Error as ShimError; use containerd_shim::event::Event; +#[cfg(unix)] use containerd_shim::mount::mount_rootfs; use containerd_shim::protos::events::task::{TaskCreate, TaskDelete, TaskExit, TaskIO, TaskStart}; use containerd_shim::protos::protobuf::well_known_types::timestamp::Timestamp; @@ -28,21 +27,16 @@ use containerd_shim::publisher::RemotePublisher; use containerd_shim::util::{timestamp as new_timestamp, write_address, IntoOption}; use containerd_shim::{self as shim, api, warn, ExitSignal, TtrpcContext, TtrpcResult}; use log::{debug, error}; +#[cfg(unix)] use nix::mount::{mount, MsFlags}; -use nix::sched::{setns, unshare, CloneFlags}; -use nix::sys::stat::Mode; -use nix::unistd::mkdir; use oci_spec::runtime; use shim::api::{StatsRequest, StatsResponse}; -use shim::protos::cgroups::metrics::{ - CPUStat, CPUUsage, MemoryEntry, MemoryStat, Metrics, PidsStat, Throttle, -}; -use shim::util::convert_to_any; use shim::Flags; use ttrpc::context::Context; use super::instance::{Instance, InstanceConfig, Nop, Wait}; use super::{oci, Error, SandboxService}; +use crate::sys::{get_metrics, setup_namespaces}; type InstanceDataStatus = (Mutex)>>, Condvar); @@ -852,14 +846,19 @@ impl Local { .as_ref() .ok_or_else(|| Error::InvalidArgument("rootfs is not set in runtime spec".to_string()))? .path(); - - if mkdir(rootfs, Mode::from_bits(0o755).unwrap()).is_ok() { /* ignore */ } + let mut mkdir = DirBuilder::new(); + mkdir.recursive(true); + #[cfg(unix)] + mkdir.mode(0o755); + if mkdir.create(rootfs).is_ok() { /* ignore */ } let rootfs_mounts = req.rootfs().to_vec(); if !rootfs_mounts.is_empty() { for m in rootfs_mounts { let mount_type = m.type_().none_if(|&x| x.is_empty()); let source = m.source.as_str().none_if(|&x| x.is_empty()); + + #[cfg(unix)] mount_rootfs(mount_type, source, &m.options.to_vec(), rootfs)?; } } @@ -926,6 +925,8 @@ impl Local { typ = None; newopts.push("rbind".to_string()); } + + #[cfg(unix)] mount_rootfs(typ, source, &newopts, &rootfs_target).map_err(|err| { ShimError::Other(format!( "error mounting {} to {} as {}: {}", @@ -1201,105 +1202,12 @@ impl Local { return Err(Error::InvalidArgument("task is not running".to_string())); } - let mut metrics = Metrics::new(); - let hier = hierarchies::auto(); - - let cgroup = if hier.v2() { - let path = format!("/proc/{}/cgroup", pid.unwrap()); - let content = fs::read_to_string(path)?; - let content = content.strip_suffix('\n').unwrap_or_default(); - - let parts: Vec<&str> = content.split("::").collect(); - let path_parts: Vec<&str> = parts[1].split('/').collect(); - let namespace = path_parts[1]; - let cgroup_name = path_parts[2]; - Cgroup::load( - hierarchies::auto(), - format!("/sys/fs/cgroup/{namespace}/{cgroup_name}"), - ) - } else { - let path = get_cgroups_relative_paths_by_pid(pid.unwrap()).unwrap(); - Cgroup::load_with_relative_paths(hierarchies::auto(), Path::new("."), path) - }; - - // from https://github.com/containerd/rust-extensions/blob/main/crates/shim/src/cgroup.rs#L97-L127 - for sub_system in Cgroup::subsystems(&cgroup) { - match sub_system { - Subsystem::Mem(mem_ctr) => { - let mem = mem_ctr.memory_stat(); - let mut mem_entry = MemoryEntry::new(); - mem_entry.set_usage(mem.usage_in_bytes); - let mut mem_stat = MemoryStat::new(); - mem_stat.set_usage(mem_entry); - mem_stat.set_total_inactive_file(mem.stat.total_inactive_file); - metrics.set_memory(mem_stat); - } - Subsystem::Cpu(cpu_ctr) => { - let mut cpu_usage = CPUUsage::new(); - let mut throttle = Throttle::new(); - let stat = cpu_ctr.cpu().stat; - for line in stat.lines() { - let parts = line.split(' ').collect::>(); - if parts.len() != 2 { - Err(Error::Others(format!("invalid cpu stat line: {}", line)))?; - } + let metrics = get_metrics(pid.unwrap())?; - // https://github.com/opencontainers/runc/blob/dbe8434359ca35af1c1e10df42b1f4391c1e1010/libcontainer/cgroups/fs2/cpu.go#L70 - match parts[0] { - "usage_usec" => { - cpu_usage.set_total(parts[1].parse::().unwrap()); - } - "user_usec" => { - cpu_usage.set_user(parts[1].parse::().unwrap()); - } - "system_usec" => { - cpu_usage.set_kernel(parts[1].parse::().unwrap()); - } - "nr_periods" => { - throttle.set_periods(parts[1].parse::().unwrap()); - } - "nr_throttled" => { - throttle.set_throttled_periods(parts[1].parse::().unwrap()); - } - "throttled_usec" => { - throttle.set_throttled_time(parts[1].parse::().unwrap()); - } - _ => {} - } - } - let mut cpu_stats = CPUStat::new(); - cpu_stats.set_throttling(throttle); - cpu_stats.set_usage(cpu_usage); - metrics.set_cpu(cpu_stats); - } - Subsystem::Pid(pid_ctr) => { - let mut pid_stats = PidsStat::new(); - pid_stats.set_current(pid_ctr.get_pid_current().map_err(|err| { - Error::Others(format!("failed to get current pid: {}", err)) - })?); - pid_stats.set_limit( - pid_ctr - .get_pid_max() - .map(|val| match val { - // See https://github.com/opencontainers/runc/blob/dbe8434359ca35af1c1e10df42b1f4391c1e1010/libcontainer/cgroups/fs/pids.go#L55 - cgroups_rs::MaxValue::Max => 0, - cgroups_rs::MaxValue::Value(val) => val as u64, - }) - .map_err(|err| { - Error::Others(format!("failed to get max pid: {}", err)) - })?, - ); - metrics.set_pids(pid_stats) - } - _ => { - // TODO: add other subsystems - } - } - } let mut stats = StatsResponse { ..Default::default() }; - stats.set_stats(convert_to_any(Box::new(metrics))?); + stats.set_stats(metrics); Ok(stats) } } @@ -1418,43 +1326,6 @@ impl Task for Local { } } -#[cfg(target_os = "linux")] -fn setup_namespaces(spec: &runtime::Spec) -> Result<()> { - let namespaces = spec - .linux() - .as_ref() - .unwrap() - .namespaces() - .as_ref() - .unwrap(); - for ns in namespaces { - if ns.typ() == runtime::LinuxNamespaceType::Network { - if let Some(p) = ns.path() { - let f = File::open(p).map_err(|err| { - ShimError::Other(format!( - "could not open network namespace {}: {}", - p.display(), - err - )) - })?; - setns(f.as_raw_fd(), CloneFlags::CLONE_NEWNET).map_err(|err| { - ShimError::Other(format!("could not set network namespace: {0}", err)) - })?; - } else { - unshare(CloneFlags::CLONE_NEWNET).map_err(|err| { - ShimError::Other(format!("could not unshare network namespace: {0}", err)) - })?; - } - } - } - - // Keep all mounts changes (such as for the rootfs) private to the shim - // This way mounts will automatically be cleaned up when the shim exits. - unshare(CloneFlags::CLONE_NEWNS) - .map_err(|err| shim::Error::Other(format!("failed to unshare mount namespace: {}", err)))?; - Ok(()) -} - /// Cli implements the containerd-shim cli interface using `Local` as the task service. pub struct Cli { pub engine: T::Engine, @@ -1502,8 +1373,7 @@ where setup_namespaces(&spec) .map_err(|e| shim::Error::Other(format!("failed to setup namespaces: {}", e)))?; - let envs = vec![] as Vec<(&str, &str)>; - + #[cfg(unix)] mount::( None, "/".as_ref(), @@ -1511,7 +1381,9 @@ where MsFlags::MS_REC | MsFlags::MS_SLAVE, None, ) - .map_err(|err| shim::Error::Other(format!("failed to remount rootfs as slave: {}", err)))?; + .map_err(|err| { + shim::Error::Other(format!("failed to remount rootfs as secondary: {}", err)) + })?; if let Some(mounts) = spec.mounts() { for m in mounts { @@ -1566,6 +1438,7 @@ where })?; } + #[cfg(unix)] mount::( Some(&src.to_string_lossy()), dest, @@ -1582,6 +1455,7 @@ where } } + let envs = vec![] as Vec<(&str, &str)>; let (_child, address) = shim::spawn(opts, &grouping, envs)?; write_address(&address)?; @@ -1632,6 +1506,7 @@ fn forward_events( .unwrap(); } +#[cfg(unix)] // This is a copy of the parse_mount function from the youki libcontainer crate. fn parse_mount(m: &runtime::Mount) -> MsFlags { let mut flags = MsFlags::empty(); diff --git a/crates/containerd-shim-wasm/src/sandbox/testutil.rs b/crates/containerd-shim-wasm/src/sandbox/testutil.rs index 0b009af2b..d58228f3c 100644 --- a/crates/containerd-shim-wasm/src/sandbox/testutil.rs +++ b/crates/containerd-shim-wasm/src/sandbox/testutil.rs @@ -74,11 +74,20 @@ macro_rules! function { }}; } +#[cfg(unix)] use caps::{CapSet, Capability}; pub use function; /// Determines if the current process has the CAP_SYS_ADMIN capability in its effective set. pub fn has_cap_sys_admin() -> bool { - let caps = caps::read(None, CapSet::Effective).unwrap(); - caps.contains(&Capability::CAP_SYS_ADMIN) + #[cfg(unix)] + { + let caps = caps::read(None, CapSet::Effective).unwrap(); + caps.contains(&Capability::CAP_SYS_ADMIN) + } + + #[cfg(windows)] + { + false + } } diff --git a/crates/containerd-shim-wasm/src/sys/unix.rs b/crates/containerd-shim-wasm/src/sys/unix.rs new file mode 100644 index 000000000..db3694a4a --- /dev/null +++ b/crates/containerd-shim-wasm/src/sys/unix.rs @@ -0,0 +1,156 @@ +use std::fs::{self, File}; +use std::os::unix::io::AsRawFd; +use std::path::Path; + +use anyhow::Result; +use cgroups_rs::cgroup::get_cgroups_relative_paths_by_pid; +use cgroups_rs::hierarchies::{self}; +use cgroups_rs::{Cgroup, Subsystem}; +use containerd_shim::error::Error as ShimError; +use containerd_shim::{self as shim}; +use nix::sched::{setns, unshare, CloneFlags}; +use oci_spec::runtime; +use protobuf::well_known_types::any::Any; +use shim::protos::cgroups::metrics::{ + CPUStat, CPUUsage, MemoryEntry, MemoryStat, Metrics, PidsStat, Throttle, +}; +use shim::util::convert_to_any; + +use crate::sandbox::Error; + +pub fn get_metrics(pid: u32) -> Result { + let mut metrics = Metrics::new(); + let hier = hierarchies::auto(); + + let cgroup = if hier.v2() { + let path = format!("/proc/{}/cgroup", pid); + let content = fs::read_to_string(path)?; + let content = content.strip_suffix('\n').unwrap_or_default(); + + let parts: Vec<&str> = content.split("::").collect(); + let path_parts: Vec<&str> = parts[1].split('/').collect(); + let namespace = path_parts[1]; + let cgroup_name = path_parts[2]; + Cgroup::load( + hierarchies::auto(), + format!("/sys/fs/cgroup/{namespace}/{cgroup_name}"), + ) + } else { + let path = get_cgroups_relative_paths_by_pid(pid).unwrap(); + Cgroup::load_with_relative_paths(hierarchies::auto(), Path::new("."), path) + }; + + // from https://github.com/containerd/rust-extensions/blob/main/crates/shim/src/cgroup.rs#L97-L127 + for sub_system in Cgroup::subsystems(&cgroup) { + match sub_system { + Subsystem::Mem(mem_ctr) => { + let mem = mem_ctr.memory_stat(); + let mut mem_entry = MemoryEntry::new(); + mem_entry.set_usage(mem.usage_in_bytes); + let mut mem_stat = MemoryStat::new(); + mem_stat.set_usage(mem_entry); + mem_stat.set_total_inactive_file(mem.stat.total_inactive_file); + metrics.set_memory(mem_stat); + } + Subsystem::Cpu(cpu_ctr) => { + let mut cpu_usage = CPUUsage::new(); + let mut throttle = Throttle::new(); + let stat = cpu_ctr.cpu().stat; + for line in stat.lines() { + let parts = line.split(' ').collect::>(); + if parts.len() != 2 { + Err(Error::Others(format!("invalid cpu stat line: {}", line)))?; + } + + // https://github.com/opencontainers/runc/blob/dbe8434359ca35af1c1e10df42b1f4391c1e1010/libcontainer/cgroups/fs2/cpu.go#L70 + match parts[0] { + "usage_usec" => { + cpu_usage.set_total(parts[1].parse::().unwrap()); + } + "user_usec" => { + cpu_usage.set_user(parts[1].parse::().unwrap()); + } + "system_usec" => { + cpu_usage.set_kernel(parts[1].parse::().unwrap()); + } + "nr_periods" => { + throttle.set_periods(parts[1].parse::().unwrap()); + } + "nr_throttled" => { + throttle.set_throttled_periods(parts[1].parse::().unwrap()); + } + "throttled_usec" => { + throttle.set_throttled_time(parts[1].parse::().unwrap()); + } + _ => {} + } + } + let mut cpu_stats = CPUStat::new(); + cpu_stats.set_throttling(throttle); + cpu_stats.set_usage(cpu_usage); + metrics.set_cpu(cpu_stats); + } + Subsystem::Pid(pid_ctr) => { + let mut pid_stats = PidsStat::new(); + pid_stats.set_current( + pid_ctr.get_pid_current().map_err(|err| { + Error::Others(format!("failed to get current pid: {}", err)) + })?, + ); + pid_stats.set_limit( + pid_ctr + .get_pid_max() + .map(|val| match val { + // See https://github.com/opencontainers/runc/blob/dbe8434359ca35af1c1e10df42b1f4391c1e1010/libcontainer/cgroups/fs/pids.go#L55 + cgroups_rs::MaxValue::Max => 0, + cgroups_rs::MaxValue::Value(val) => val as u64, + }) + .map_err(|err| Error::Others(format!("failed to get max pid: {}", err)))?, + ); + metrics.set_pids(pid_stats); + } + _ => { + // TODO: add other subsystems + } + } + } + + let metrics = convert_to_any(Box::new(metrics)).map_err(|e| Error::Others(e.to_string()))?; + Ok(metrics) +} + +pub fn setup_namespaces(spec: &runtime::Spec) -> Result<()> { + let namespaces = spec + .linux() + .as_ref() + .unwrap() + .namespaces() + .as_ref() + .unwrap(); + for ns in namespaces { + if ns.typ() == runtime::LinuxNamespaceType::Network { + if let Some(p) = ns.path() { + let f = File::open(p).map_err(|err| { + ShimError::Other(format!( + "could not open network namespace {}: {}", + p.display(), + err + )) + })?; + setns(f.as_raw_fd(), CloneFlags::CLONE_NEWNET).map_err(|err| { + ShimError::Other(format!("could not set network namespace: {0}", err)) + })?; + } else { + unshare(CloneFlags::CLONE_NEWNET).map_err(|err| { + ShimError::Other(format!("could not unshare network namespace: {0}", err)) + })?; + } + } + } + + // Keep all mounts changes (such as for the rootfs) private to the shim + // This way mounts will automatically be cleaned up when the shim exits. + unshare(CloneFlags::CLONE_NEWNS) + .map_err(|err| shim::Error::Other(format!("failed to unshare mount namespace: {}", err)))?; + Ok(()) +} diff --git a/crates/containerd-shim-wasm/src/sys/windows.rs b/crates/containerd-shim-wasm/src/sys/windows.rs new file mode 100644 index 000000000..b0fede726 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sys/windows.rs @@ -0,0 +1,21 @@ +use anyhow::Result; +use containerd_shim::{self as shim}; +use oci_spec::runtime; +use protobuf::well_known_types::any::Any; +use shim::util::convert_to_any; + +use crate::sandbox::Error; + +pub fn get_metrics(pid: u32) -> Result { + // Create empty message for now + // https://github.com/containerd/rust-extensions/pull/178 + let m = protobuf::well_known_types::any::Any::new(); + + let metrics = convert_to_any(Box::new(m)).map_err(|e| Error::Others(e.to_string()))?; + Ok(metrics) +} + +pub fn setup_namespaces(spec: &runtime::Spec) -> Result<()> { + // noop for now + Ok(()) +} diff --git a/crates/containerd-shim-wasmedge/Cargo.toml b/crates/containerd-shim-wasmedge/Cargo.toml index 70c8896a2..150eebcc3 100644 --- a/crates/containerd-shim-wasmedge/Cargo.toml +++ b/crates/containerd-shim-wasmedge/Cargo.toml @@ -17,10 +17,12 @@ oci-spec = { workspace = true, features = ["runtime"] } thiserror = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -nix = { workspace = true } libc = { workspace = true } + +[target.'cfg(unix)'.dependencies] libcontainer = { workspace = true } dbus = { version = "*", optional = true } +nix = { workspace = true } [dev-dependencies] tempfile = "3.7" diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance/instance_linux.rs similarity index 100% rename from crates/containerd-shim-wasmedge/src/instance.rs rename to crates/containerd-shim-wasmedge/src/instance/instance_linux.rs diff --git a/crates/containerd-shim-wasmedge/src/instance/instance_windows.rs b/crates/containerd-shim-wasmedge/src/instance/instance_windows.rs new file mode 100644 index 000000000..ce36b6146 --- /dev/null +++ b/crates/containerd-shim-wasmedge/src/instance/instance_windows.rs @@ -0,0 +1,44 @@ +use std::path::PathBuf; +use std::process::ExitCode; + +use containerd_shim_wasm::sandbox::error::Error; +use containerd_shim_wasm::sandbox::instance::Wait; +use containerd_shim_wasm::sandbox::{Instance, InstanceConfig}; +use wasmedge_sdk::Vm; + +pub struct Wasi { + id: String, + + exit_code: ExitCode, + + stdin: String, + stdout: String, + stderr: String, + bundle: String, + + rootdir: PathBuf, +} + +impl Instance for Wasi { + type Engine = (); + + fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { + todo!() + } + + fn start(&self) -> std::result::Result { + todo!() + } + + fn kill(&self, signal: u32) -> std::result::Result<(), Error> { + todo!() + } + + fn delete(&self) -> std::result::Result<(), Error> { + todo!() + } + + fn wait(&self, waiter: &Wait) -> std::result::Result<(), Error> { + todo!() + } +} diff --git a/crates/containerd-shim-wasmedge/src/lib.rs b/crates/containerd-shim-wasmedge/src/lib.rs index eead58232..5e92dcfb8 100644 --- a/crates/containerd-shim-wasmedge/src/lib.rs +++ b/crates/containerd-shim-wasmedge/src/lib.rs @@ -1,8 +1,14 @@ pub mod error; -pub mod executor; + +#[cfg_attr(unix, path = "instance/instance_linux.rs")] +#[cfg_attr(windows, path = "instance/instance_windows.rs")] pub mod instance; pub mod oci_utils; +#[cfg(unix)] +pub mod executor; + +#[cfg(unix)] #[cfg(test)] mod test { use std::os::unix::prelude::OsStrExt; diff --git a/crates/containerd-shim-wasmtime/Cargo.toml b/crates/containerd-shim-wasmtime/Cargo.toml index ed4749e6c..64c4023bc 100644 --- a/crates/containerd-shim-wasmtime/Cargo.toml +++ b/crates/containerd-shim-wasmtime/Cargo.toml @@ -32,11 +32,13 @@ cap-std = { workspace = true } oci-spec = { workspace = true, features = ["runtime"] } thiserror = { workspace = true } serde_json = { workspace = true } +serde = { workspace = true } +libc = { workspace = true } + +[target.'cfg(unix)'.dependencies] nix = { workspace = true } libcontainer = { workspace = true } dbus = { version = "*", optional = true } -serde = { workspace = true } -libc = { workspace = true } [dev-dependencies] tempfile = "3.7" diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance/instance_linux.rs similarity index 100% rename from crates/containerd-shim-wasmtime/src/instance.rs rename to crates/containerd-shim-wasmtime/src/instance/instance_linux.rs diff --git a/crates/containerd-shim-wasmtime/src/instance/instance_windows.rs b/crates/containerd-shim-wasmtime/src/instance/instance_windows.rs new file mode 100644 index 000000000..ea54312ba --- /dev/null +++ b/crates/containerd-shim-wasmtime/src/instance/instance_windows.rs @@ -0,0 +1,43 @@ +use std::path::PathBuf; + +use containerd_shim_wasm::sandbox::error::Error; +use containerd_shim_wasm::sandbox::instance::{ExitCode, Wait}; +use containerd_shim_wasm::sandbox::{Instance, InstanceConfig}; + +pub struct Wasi { + exit_code: ExitCode, + engine: wasmtime::Engine, + stdin: String, + stdout: String, + stderr: String, + bundle: String, + rootdir: PathBuf, + id: String, +} + +impl Instance for Wasi { + type Engine = wasmtime::Engine; + + fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { + todo!() + } + + fn start(&self) -> std::result::Result { + todo!() + } + + fn kill(&self, signal: u32) -> std::result::Result<(), Error> { + todo!() + } + + fn delete(&self) -> std::result::Result<(), Error> { + todo!() + } + + fn wait( + &self, + waiter: &containerd_shim_wasm::sandbox::instance::Wait, + ) -> std::result::Result<(), Error> { + todo!() + } +} diff --git a/crates/containerd-shim-wasmtime/src/lib.rs b/crates/containerd-shim-wasmtime/src/lib.rs index 312731e44..6e4ac2761 100644 --- a/crates/containerd-shim-wasmtime/src/lib.rs +++ b/crates/containerd-shim-wasmtime/src/lib.rs @@ -1,4 +1,8 @@ pub mod error; -pub mod executor; +#[cfg_attr(unix, path = "instance/instance_linux.rs")] +#[cfg_attr(windows, path = "instance/instance_windows.rs")] pub mod instance; pub mod oci_wasmtime; + +#[cfg(unix)] +pub mod executor; diff --git a/docs/windows-getting-started.md b/docs/windows-getting-started.md index be477b8d4..117a8fda7 100644 --- a/docs/windows-getting-started.md +++ b/docs/windows-getting-started.md @@ -23,3 +23,31 @@ After this, you can execute an example, like: `ctr run --rm --runtime=io.contain > To kill the process from the example, you can run: `ctr task kill -s SIGKILL testwasm`. +## Building and developing on Windows + +You need to install `wasmedge`, `llvm` and `make`. This can be done using `winget`, `choco` or manually. (note as of writing this `winget` doesn't have the latest package and will builds will fail). See `.github/scripts/build-windows.sh` for an example. + +Once you have those dependencies you will need to set env: + +``` +$env:WASMEDGE_LIB_DIR="C:\Program Files\WasmEdge\lib" +$env:WASMEDGE_INCLUDE_DIR="C:\Program Files\WasmEdge\include" +``` + +Then you can run: + +``` +make build +``` + +### Using VS code +If you are using VS Code for development you can use the following `settings.json` in the `.vscode` folder of the project: + +``` +{ + "rust-analyzer.cargo.noDefaultFeatures": true, + "rust-analyzer.cargo.extraEnv": { + "WASMEDGE_LIB_DIR": "C:\\Program Files\\WasmEdge\\lib", + "WASMEDGE_INCLUDE_DIR": "C:\\Program Files\\WasmEdge\\include" + } +} \ No newline at end of file diff --git a/rustfmt.toml b/rustfmt.toml index f2f2c7499..fdf2716ef 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,4 +1,4 @@ -newline_style = "Unix" +newline_style = "Native" unstable_features = true # Cargo fmt now needs to be called with `cargo +nightly fmt` group_imports = "StdExternalCrate" # create three groups for std, external and local crates # Merge imports from the same module diff --git a/scripts/setup.sh b/scripts/setup-linux.sh similarity index 100% rename from scripts/setup.sh rename to scripts/setup-linux.sh diff --git a/scripts/setup-windows.sh b/scripts/setup-windows.sh new file mode 100644 index 000000000..6c4a8cf1b --- /dev/null +++ b/scripts/setup-windows.sh @@ -0,0 +1,4 @@ +#!/bin/bash +choco install -y wasmedge --version 0.13.1 +# require clang for wasmedge for bindgen, which is used in the build script to generate the rust bindings to the c codebase +choco install -y llvm --version 16.0.6 diff --git a/scripts/validate-docs.sh b/scripts/validate-docs.sh index ec1bbd8fd..fa4533ebf 100755 --- a/scripts/validate-docs.sh +++ b/scripts/validate-docs.sh @@ -2,7 +2,12 @@ set -e -cargo build --all --verbose --features generate_doc +CARGO_FLAGS="" +if [[ $RUNNER_OS == "Windows" ]]; then + CARGO_FLAGS="--no-default-features" +fi + +cargo build --all --verbose --features generate_doc $CARGO_FLAGS git status --porcelain | grep README.md || exit 0 echo "README.md is not up to date. Please run 'cargo build --all --features generate_doc' and commit the changes." >&2