diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19a1a2cd3..7c3f5ed0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --all-features + args: --all-features -- --test-threads=1 doc: name: Documentation diff --git a/Cargo.lock b/Cargo.lock index 3bf8a8f5a..21202c91e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -422,9 +422,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" +checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" dependencies = [ "cfg-if", "lazy_static", @@ -1225,7 +1225,7 @@ dependencies = [ [[package]] name = "northstar" -version = "0.6.4" +version = "0.7.0-dev" dependencies = [ "anyhow", "async-stream", @@ -1246,6 +1246,7 @@ dependencies = [ "futures", "hex", "humanize-rs", + "humantime", "inotify", "itertools", "lazy_static", @@ -1287,6 +1288,7 @@ dependencies = [ "futures", "lazy_static", "log", + "nanoid", "nix 0.23.1", "northstar", "regex", @@ -2083,6 +2085,8 @@ dependencies = [ "clap", "env_logger 0.9.0", "futures", + "humantime", + "itertools", "log", "northstar", "rand 0.8.5", diff --git a/README.md b/README.md index d06ec4aca..324de824d 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ kernel configuration with the `CONFIG_` entries in the `check_conf.sh` script. ### Container launch sequence -**TODO**:
+
### Manifest Format diff --git a/doc/diagrams/container_startup.png b/doc/diagrams/container_startup.png deleted file mode 100644 index cc4157636..000000000 Binary files a/doc/diagrams/container_startup.png and /dev/null differ diff --git a/doc/diagrams/container_startup.puml b/doc/diagrams/container_startup.puml deleted file mode 100644 index 504cd0c31..000000000 --- a/doc/diagrams/container_startup.puml +++ /dev/null @@ -1,33 +0,0 @@ -@startuml container_startup - -activate Runtime -Runtime -> Runtime: Check and Mount container -create Trampoline -Runtime -> Trampoline: Fork -activate Trampoline -create Init -Trampoline -> Init: Fork -activate Init -Trampoline -> Runtime: Init PID -destroy Trampoline -Runtime -> Runtime: Wait for Trampoline exit (waitpid) -Init -> Init: Wait for run signal (Condition::wait) -Runtime -> Runtime: Configure cgroups -Runtime -> Init: Signal run (Condition::notify) -Runtime -> Runtime: Wait for execve (Condition::wait) -Init -> Init: Mount, Chroot, UID / GID,\ndrop privileges, file descriptors -create Container -Init -> Container: Fork -activate Container -Init -> Init: Wait for container to exit (waitpid) -Container -> Container: Set seccomp filter -Container -> : Execve(..) -Runtime -> Runtime: Condition pipe closed: Container is started -note left: Condition pipe is CLOEXEC -Container -> Init: Exit -destroy Container -Init -> Runtime: Exit -Runtime -> Runtime: Read exit status from pipe or waitpid on pid of init -destroy Init - -@enduml diff --git a/examples/console/manifest.yaml b/examples/console/manifest.yaml index 0592ddadc..e80a4dad8 100644 --- a/examples/console/manifest.yaml +++ b/examples/console/manifest.yaml @@ -5,10 +5,8 @@ console: true uid: 1000 gid: 1000 io: - stdout: - log: - level: DEBUG - tag: console + stdout: pipe + stderr: pipe mounts: /dev: type: dev diff --git a/examples/cpueater/manifest.yaml b/examples/cpueater/manifest.yaml index 07d7e716e..eceb048f6 100644 --- a/examples/cpueater/manifest.yaml +++ b/examples/cpueater/manifest.yaml @@ -24,7 +24,5 @@ mounts: type: bind host: /system io: - stdout: - log: - level: DEBUG - tag: cpueater + stdout: pipe + stderr: pipe diff --git a/examples/cpueater/src/main.rs b/examples/cpueater/src/main.rs index 3ed4fbd5e..40c0de44a 100644 --- a/examples/cpueater/src/main.rs +++ b/examples/cpueater/src/main.rs @@ -1,7 +1,7 @@ use std::env::var; fn main() { - let version = var("VERSION").expect("Failed to read VERSION"); + let version = var("NORTHSTAR_VERSION").expect("Failed to read NORTHSTAR_VERSION"); let threads = var("THREADS") .expect("Failed to read THREADS") .parse::() diff --git a/examples/crashing/manifest.yaml b/examples/crashing/manifest.yaml index f83430406..d6b5a6c65 100644 --- a/examples/crashing/manifest.yaml +++ b/examples/crashing/manifest.yaml @@ -5,6 +5,9 @@ uid: 1000 gid: 1000 env: RUST_BACKTRACE: 1 +io: + stdout: pipe + stderr: discard mounts: /dev: type: dev @@ -19,8 +22,3 @@ mounts: /system: type: bind host: /system -io: - stdout: - log: - level: DEBUG - tag: crashing diff --git a/examples/hello-ferris/manifest.yaml b/examples/hello-ferris/manifest.yaml index 6a249e20b..b8273d7a0 100644 --- a/examples/hello-ferris/manifest.yaml +++ b/examples/hello-ferris/manifest.yaml @@ -37,7 +37,5 @@ mounts: dir: / options: noexec,nodev,nosuid io: - stdout: - log: - level: DEBUG - tag: ferris + stdout: pipe + stderr: pipe diff --git a/examples/hello-resource/manifest.yaml b/examples/hello-resource/manifest.yaml index 045168037..e4533d3c7 100644 --- a/examples/hello-resource/manifest.yaml +++ b/examples/hello-resource/manifest.yaml @@ -23,7 +23,5 @@ mounts: type: bind host: /system io: - stdout: - log: - level: DEBUG - tag: hello + stdout: pipe + stderr: pipe diff --git a/examples/hello-world/manifest.yaml b/examples/hello-world/manifest.yaml index e8cfc98de..49ce0cca6 100644 --- a/examples/hello-world/manifest.yaml +++ b/examples/hello-world/manifest.yaml @@ -6,10 +6,8 @@ gid: 1000 env: HELLO: northstar io: - stdout: - log: - level: DEBUG - tag: hello + stdout: pipe + stderr: pipe mounts: /dev: type: dev diff --git a/examples/hello-world/src/main.rs b/examples/hello-world/src/main.rs index 4de022188..ae428f8ca 100644 --- a/examples/hello-world/src/main.rs +++ b/examples/hello-world/src/main.rs @@ -1,13 +1,9 @@ fn main() { - let hello = std::env::var("HELLO").unwrap_or_else(|_| "unknown".into()); - let version = std::env::var("VERSION").unwrap_or_else(|_| "unknown".into()); + let hello = std::env::var("NORTHSTAR_CONTAINER").unwrap_or_else(|_| "unknown".into()); - println!("Hello again {} from version {}!", hello, version); + println!("Hello again {}!", hello); for i in 0..u64::MAX { - println!( - "...and hello again #{} {} from version {}...", - i, hello, version - ); + println!("...and hello again #{} {} ...", i, hello); std::thread::sleep(std::time::Duration::from_secs(1)); } } diff --git a/examples/inspect/manifest.yaml b/examples/inspect/manifest.yaml index 726a2c8b9..3307b025f 100644 --- a/examples/inspect/manifest.yaml +++ b/examples/inspect/manifest.yaml @@ -1,17 +1,11 @@ -name: inspect +name: inspect version: 0.0.1 init: /inspect uid: 1000 gid: 1000 io: - stdout: - log: - level: DEBUG - tag: inspect - stderr: - log: - level: WARN - tag: inspect + stdout: pipe + stderr: discard mounts: /dev: type: dev diff --git a/examples/memeater/manifest.yaml b/examples/memeater/manifest.yaml index 5e9059ce4..7b2e27c0b 100644 --- a/examples/memeater/manifest.yaml +++ b/examples/memeater/manifest.yaml @@ -23,7 +23,5 @@ mounts: type: bind host: /system io: - stdout: - log: - level: DEBUG - tag: memeater + stdout: pipe + stderr: pipe diff --git a/examples/persistence/manifest.yaml b/examples/persistence/manifest.yaml index 2e5595de4..19dda8c4e 100644 --- a/examples/persistence/manifest.yaml +++ b/examples/persistence/manifest.yaml @@ -20,7 +20,5 @@ mounts: type: bind host: /system io: - stdout: - log: - level: DEBUG - tag: persistence + stdout: pipe + stderr: pipe diff --git a/examples/seccomp/manifest.yaml b/examples/seccomp/manifest.yaml index 3fb514724..415624770 100644 --- a/examples/seccomp/manifest.yaml +++ b/examples/seccomp/manifest.yaml @@ -18,10 +18,8 @@ mounts: type: bind host: /system io: - stdout: - log: - level: DEBUG - tag: seccomp + stdout: pipe + stderr: pipe seccomp: profile: - default \ No newline at end of file + default diff --git a/images/container-startup.png b/images/container-startup.png index cc4157636..e698a4616 100644 Binary files a/images/container-startup.png and b/images/container-startup.png differ diff --git a/images/container-startup.puml b/images/container-startup.puml index 504cd0c31..d34ec7a7c 100644 --- a/images/container-startup.puml +++ b/images/container-startup.puml @@ -1,33 +1,69 @@ @startuml container_startup +create Client +activate Client + +create Runtime activate Runtime -Runtime -> Runtime: Check and Mount container + +create Forker +Runtime -> Forker: Fork +activate Forker + +Client -> Runtime: Connect: Hello +Client <- Runtime: ConnectAck +Client -> Runtime: Start container +Runtime -> Runtime: Check and mount container(s) +Runtime -> Runtime: Open PTY + +Runtime -> Forker: Create container + create Trampoline -Runtime -> Trampoline: Fork +Forker -> Trampoline: Fork activate Trampoline +Trampoline -> Trampoline: Create PID namespace + create Init Trampoline -> Init: Fork activate Init -Trampoline -> Runtime: Init PID +Init -> Init: Mount, Chroot, UID / GID,\ndrop privileges, file descriptors + +Trampoline -> Forker: Forked init with PID destroy Trampoline -Runtime -> Runtime: Wait for Trampoline exit (waitpid) -Init -> Init: Wait for run signal (Condition::wait) + +Forker -> Forker: reap Trampoline + +Forker -> Runtime: Created init with PID + Runtime -> Runtime: Configure cgroups -Runtime -> Init: Signal run (Condition::notify) -Runtime -> Runtime: Wait for execve (Condition::wait) -Init -> Init: Mount, Chroot, UID / GID,\ndrop privileges, file descriptors +Runtime -> Runtime: Configure debug +Runtime -> Runtime: Configure PTY forward + +Runtime -> Forker: Exec container +Forker -> Init: Exec Container create Container Init -> Container: Fork activate Container +Forker <- Init: Exec +Runtime <- Forker: Exec +Client <- Runtime: Started +Client <- Runtime: Notification: Started + Init -> Init: Wait for container to exit (waitpid) +Container -> Container: Setup PTY Container -> Container: Set seccomp filter Container -> : Execve(..) -Runtime -> Runtime: Condition pipe closed: Container is started -note left: Condition pipe is CLOEXEC -Container -> Init: Exit +... +Container -> Init: SIGCHLD destroy Container -Init -> Runtime: Exit -Runtime -> Runtime: Read exit status from pipe or waitpid on pid of init + +Init -> Init: waitpid: Exit status of container +Init -> Forker: Container exit status destroy Init +Forker -> Runtime: Container exit status +Runtime -> Runtime: Stop PTY thread +Runtime -> Runtime: Destroy cgroups +Client <- Runtime: Notification: Exit + @enduml diff --git a/main/Cargo.toml b/main/Cargo.toml index f321b6b7a..0070c2cb8 100644 --- a/main/Cargo.toml +++ b/main/Cargo.toml @@ -16,7 +16,7 @@ clap = { version = "3.1.0", features = ["derive"] } log = "0.4.14" nix = "0.23.0" northstar = { path = "../northstar", features = ["runtime"] } -tokio = { version = "1.17.0", features = ["rt", "macros", "signal"] } +tokio = { version = "1.17.0", features = ["rt-multi-thread", "macros", "signal"] } toml = "0.5.8" [target.'cfg(not(target_os = "android"))'.dependencies] diff --git a/main/src/logger.rs b/main/src/logger.rs index a3a486779..b9ee80617 100644 --- a/main/src/logger.rs +++ b/main/src/logger.rs @@ -9,7 +9,7 @@ pub fn init() { } #[cfg(not(target_os = "android"))] -static TAG_SIZE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(20); +static TAG_SIZE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(28); /// Initialize the logger #[cfg(not(target_os = "android"))] @@ -17,51 +17,63 @@ pub fn init() { use env_logger::fmt::Color; use std::{io::Write, sync::atomic::Ordering}; + fn color(target: &str) -> Color { + // Some colors are hard to read on (at least) dark terminals + // and I consider some others as ugly ;-) + let hash = target.bytes().fold(42u8, |c, x| c ^ x); + Color::Ansi256(match hash { + c @ 0..=1 => c + 2, + c @ 16..=21 => c + 6, + c @ 52..=55 | c @ 126..=129 => c + 4, + c @ 163..=165 | c @ 200..=201 => c + 3, + c @ 207 => c + 1, + c @ 232..=240 => c + 9, + c => c, + }) + } + let mut builder = env_logger::Builder::new(); builder.parse_filters("northstar=debug"); builder.format(|buf, record| { - let mut style = buf.style(); + let timestamp = buf.timestamp_millis().to_string(); + let timestamp = timestamp.strip_suffix('Z').unwrap(); + + let mut level = buf.default_level_style(record.metadata().level()); + level.set_bold(true); + let level = level.value(record.metadata().level().as_str()); - let timestamp = buf.timestamp_millis(); - let level = buf.default_styled_level(record.metadata().level()); + let pid = std::process::id().to_string(); + let mut pid_style = buf.style(); + pid_style.set_color(color(&pid)); - if let Some(module_path) = record - .module_path() + if let Some(target) = Option::from(record.target().is_empty()) + .map(|_| record.target()) + .or_else(|| record.module_path()) .and_then(|module_path| module_path.find(&"::").map(|p| &module_path[p + 2..])) { - TAG_SIZE.fetch_max(module_path.len(), Ordering::SeqCst); + let mut tag_style = buf.style(); + TAG_SIZE.fetch_max(target.len(), Ordering::SeqCst); let tag_size = TAG_SIZE.load(Ordering::SeqCst); - fn hashed_color(i: &str) -> Color { - // Some colors are hard to read on (at least) dark terminals - // and I consider some others as ugly ;-) - Color::Ansi256(match i.bytes().fold(42u8, |c, x| c ^ x) { - c @ 0..=1 => c + 2, - c @ 16..=21 => c + 6, - c @ 52..=55 | c @ 126..=129 => c + 4, - c @ 163..=165 | c @ 200..=201 => c + 3, - c @ 207 => c + 1, - c @ 232..=240 => c + 9, - c => c, - }) - } - style.set_color(hashed_color(module_path)); + tag_style.set_color(color(target)); writeln!( buf, - "{}: {:>s$} {:<5}: {}", + "{} {:>s$} {} {:<5}: {}", timestamp, - style.value(module_path), + tag_style.value(target), + pid_style.value("⬤"), level, record.args(), - s = tag_size + s = tag_size, ) } else { writeln!( buf, - "{}: {} {:<5}: {}", + "{} {} {} {:<5}: {}", timestamp, " ".repeat(TAG_SIZE.load(Ordering::SeqCst)), + pid_style.value("⬤"), level, record.args(), ) diff --git a/main/src/main.rs b/main/src/main.rs index dba0c49f2..80547ae73 100644 --- a/main/src/main.rs +++ b/main/src/main.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, Context, Error}; use clap::Parser; use log::{debug, info, warn}; use nix::mount::MsFlags; -use northstar::runtime; +use northstar::{runtime, runtime::Runtime as Northstar}; use runtime::config::Config; use std::{ fs::{self, read_to_string}, @@ -32,16 +32,31 @@ struct Opt { pub disable_mount_namespace: bool, } -#[tokio::main(flavor = "current_thread")] -async fn main() -> Result<(), Error> { +fn main() -> Result<(), Error> { + // Initialize logging + logger::init(); + + // Parse command line arguments and prepare the environment + let config = init()?; + + // Create the runtime launcher. This must be done *before* spawning the tokio threadpool. + let northstar = Northstar::new(config)?; + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("northstar") + .build() + .context("Failed to create runtime")? + .block_on(run(northstar)) +} + +fn init() -> Result { let opt = Opt::parse(); let config = read_to_string(&opt.config) .with_context(|| format!("Failed to read configuration file {}", opt.config.display()))?; let config: Config = toml::from_str(&config) .with_context(|| format!("Failed to read configuration file {}", opt.config.display()))?; - logger::init(); - fs::create_dir_all(&config.run_dir).context("Failed to create run_dir")?; fs::create_dir_all(&config.data_dir).context("Failed to create data_dir")?; fs::create_dir_all(&config.log_dir).context("Failed to create log dir")?; @@ -64,9 +79,15 @@ async fn main() -> Result<(), Error> { debug!("Mount namespace is disabled"); } - let mut runtime = runtime::Runtime::start(config) + Ok(config) +} + +async fn run(northstar: Northstar) -> Result<(), Error> { + let mut runtime = northstar + .start() .await - .context("Failed to start runtime")?; + .context("Failed to start Northstar")?; + let mut sigint = tokio::signal::unix::signal(SignalKind::interrupt()) .context("Failed to install sigint handler")?; let mut sigterm = tokio::signal::unix::signal(SignalKind::terminate()) @@ -87,7 +108,7 @@ async fn main() -> Result<(), Error> { info!("Received SIGHUP. Stopping Northstar runtime"); runtime.shutdown().await } - status = &mut runtime => status, + status = runtime.stopped() => status, }; match status { diff --git a/northstar-tests/Cargo.toml b/northstar-tests/Cargo.toml index c73648d94..309c2e3ac 100644 --- a/northstar-tests/Cargo.toml +++ b/northstar-tests/Cargo.toml @@ -11,6 +11,7 @@ env_logger = "0.9.0" futures = "0.3.21" lazy_static = "1.4.0" log = "0.4.14" +nanoid = "0.4.0" nix = "0.23.0" northstar = { path = "../northstar", features = ["api", "runtime"] } regex = "1.5.4" diff --git a/northstar-tests/src/macros.rs b/northstar-tests/src/macros.rs index aab9a3d8d..e5e78bb5f 100644 --- a/northstar-tests/src/macros.rs +++ b/northstar-tests/src/macros.rs @@ -1,38 +1,3 @@ -use super::logger; -use nix::{mount, sched}; -use sched::{unshare, CloneFlags}; - -pub fn init() { - logger::init(); - log::set_max_level(log::LevelFilter::Debug); - - // Enter a mount namespace. This needs to be done before spawning - // the tokio threadpool. - unshare(CloneFlags::CLONE_NEWNS).unwrap(); - - // Set the mount propagation to private on root. This ensures that *all* - // mounts get cleaned up upon process termination. The approach to bind - // mount the run_dir only (this is where the mounts from northstar happen) - // doesn't work for the tests since the run_dir is a tempdir which is a - // random dir on every run. Checking at the beginning of the tests if - // run_dir is bind mounted - a leftover from a previous crash - obviously - // doesn't work. Technically, it is only necessary set the propagation of - // the parent mount of the run_dir, but this not easy to find and the change - // of mount propagation on root is fine for the tests which are development - // only. - mount::mount( - Some("/"), - "/", - Option::<&str>::None, - mount::MsFlags::MS_PRIVATE | mount::MsFlags::MS_REC, - Option::<&'static [u8]>::None, - ) - .expect( - "Failed to set mount propagation to private on - root", - ); -} - /// Northstar integration test #[macro_export] macro_rules! test { @@ -41,15 +6,52 @@ macro_rules! test { #![rusty_fork(timeout_ms = 300000)] #[test] fn $name() { - northstar_tests::macros::init(); - match tokio::runtime::Builder::new_current_thread() + crate::logger::init(); + log::set_max_level(log::LevelFilter::Debug); + + // Enter a mount namespace. This needs to be done before spawning + // the tokio threadpool. + nix::sched::unshare(nix::sched::CloneFlags::CLONE_NEWNS).unwrap(); + + // Set the mount propagation to private on root. This ensures that *all* + // mounts get cleaned up upon process termination. The approach to bind + // mount the run_dir only (this is where the mounts from northstar happen) + // doesn't work for the tests since the run_dir is a tempdir which is a + // random dir on every run. Checking at the beginning of the tests if + // run_dir is bind mounted - a leftover from a previous crash - obviously + // doesn't work. Technically, it is only necessary set the propagation of + // the parent mount of the run_dir, but this not easy to find and the change + // of mount propagation on root is fine for the tests which are development + // only. + nix::mount::mount( + Some("/"), + "/", + Option::<&str>::None, + nix::mount::MsFlags::MS_PRIVATE | nix::mount::MsFlags::MS_REC, + Option::<&'static [u8]>::None, + ) + .expect( + "Failed to set mount propagation to private on + root", + ); + let runtime = northstar_tests::runtime::Runtime::new().expect("Failed to start runtime"); + + match tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) .enable_all() .thread_name(stringify!($name)) .build() .expect("Failed to start runtime") - .block_on(async { $e }) { + .block_on(async { + let runtime = runtime.start().await?; + $e + northstar_tests::runtime::client().shutdown().await?; + drop(runtime); + tokio::fs::remove_file(northstar_tests::runtime::console().path()).await?; + Ok(()) + }) { Ok(_) => std::process::exit(0), - Err(e) => panic!("{}", e), + anyhow::Result::<()>::Err(e) => panic!("{}", e), } } } diff --git a/northstar-tests/src/runtime.rs b/northstar-tests/src/runtime.rs index 018a4cf4c..aba9d230d 100644 --- a/northstar-tests/src/runtime.rs +++ b/northstar-tests/src/runtime.rs @@ -3,15 +3,16 @@ use super::{containers::*, logger}; use anyhow::{anyhow, Context, Result}; use futures::StreamExt; +use nanoid::nanoid; use northstar::{ api::{ - client::Client, + client, model::{Container, ExitStatus, Notification}, }, common::non_null_string::NonNullString, runtime::{ - self, config::{self, Config, RepositoryType}, + Runtime as Northstar, }, }; use std::{ @@ -21,49 +22,35 @@ use std::{ use tempfile::{NamedTempFile, TempDir}; use tokio::{fs, net::UnixStream, pin, select, time}; -pub struct Northstar { - /// Runtime configuration - pub config: Config, - /// Runtime console address (Unix socket) - pub console: String, - /// Client instance - client: northstar::api::client::Client, - /// Runtime instance - runtime: runtime::Runtime, - /// Tmpdir for NPK dumps - tmpdir: TempDir, -} +pub static mut CLIENT: Option = None; -impl std::ops::Deref for Northstar { - type Target = Client; +pub fn client() -> &'static mut Client { + unsafe { CLIENT.as_mut().unwrap() } +} - fn deref(&self) -> &Self::Target { - &self.client - } +pub fn console() -> url::Url { + let console = std::env::temp_dir().join(format!("northstar-{}", std::process::id())); + url::Url::parse(&format!("unix://{}", console.display())).unwrap() } -impl std::ops::DerefMut for Northstar { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.client - } +pub enum Runtime { + Created(Northstar, TempDir), + Started(Northstar, TempDir), } -impl Northstar { - /// Launches an instance of Northstar - pub async fn launch() -> Result { - let pid = std::process::id(); +impl Runtime { + pub fn new() -> Result { let tmpdir = tempfile::Builder::new().prefix("northstar-").tempdir()?; - let run_dir = tmpdir.path().join("run"); - fs::create_dir(&run_dir).await?; + std::fs::create_dir(&run_dir)?; let data_dir = tmpdir.path().join("data"); - fs::create_dir(&data_dir).await?; + std::fs::create_dir(&data_dir)?; let log_dir = tmpdir.path().join("log"); - fs::create_dir(&log_dir).await?; + std::fs::create_dir(&log_dir)?; let test_repository = tmpdir.path().join("test"); - fs::create_dir(&test_repository).await?; + std::fs::create_dir(&test_repository)?; let example_key = tmpdir.path().join("key.pub"); - fs::write(&example_key, include_bytes!("../../examples/northstar.pub")).await?; + std::fs::write(&example_key, include_bytes!("../../examples/northstar.pub"))?; let mut repositories = HashMap::new(); repositories.insert( @@ -77,71 +64,84 @@ impl Northstar { "test-1".into(), config::Repository { r#type: RepositoryType::Memory, - key: Some(example_key.clone()), + key: Some(example_key), }, ); - let console = format!( - "{}/northstar-{}", - tmpdir.path().display(), - std::process::id() - ); - let console_url = url::Url::parse(&format!("unix://{}", console))?; - let config = Config { - console: Some(vec![console_url.clone()]), + console: Some(vec![console()]), run_dir, - data_dir: data_dir.clone(), + data_dir, log_dir, mount_parallel: 10, - cgroup: NonNullString::try_from(format!("northstar-{}", pid)).unwrap(), + cgroup: NonNullString::try_from(format!("northstar-{}", nanoid!())).unwrap(), repositories, debug: None, }; + let b = Northstar::new(config)?; - // Start the runtime - let runtime = runtime::Runtime::start(config.clone()) - .await - .context("Failed to start runtime")?; - // Wait until the console is up and running - super::logger::assume("Started console on", 5u64).await?; + Ok(Runtime::Created(b, tmpdir)) + } + + pub async fn start(self) -> Result { + if let Runtime::Created(launcher, tmpdir) = self { + let runtime = launcher.start().await?; + logger::assume("Runtime up and running", 10u64).await?; + + unsafe { + CLIENT = Some(Client::new().await?); + } + + Ok(Runtime::Started(runtime, tmpdir)) + } else { + anyhow::bail!("Runtime is already started") + } + } +} + +pub struct Client { + /// Client instance + client: northstar::api::client::Client, +} + +impl std::ops::Deref for Client { + type Target = client::Client; + fn deref(&self) -> &Self::Target { + &self.client + } +} + +impl std::ops::DerefMut for Client { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.client + } +} + +impl Client { + /// Launches an instance of Northstar + pub async fn new() -> Result { // Connect to the runtime - let io = UnixStream::connect(&console) + let io = UnixStream::connect(console().path()) .await .expect("Failed to connect to console"); - let client = Client::new(io, Some(1000), time::Duration::from_secs(30)).await?; + let client = client::Client::new(io, Some(1000), time::Duration::from_secs(30)).await?; // Wait until a successful connection logger::assume("Client .* connected", 5u64).await?; - Ok(Northstar { - config, - console, - client, - runtime, - tmpdir, - }) + Ok(Client { client }) } /// Connect a new client instance to the runtime - pub async fn client(&self) -> Result> { - let io = UnixStream::connect(&self.console) + pub async fn client(&self) -> Result> { + let io = UnixStream::connect(console().path()) .await .context("Failed to connect to console")?; - Client::new(io, Some(1000), time::Duration::from_secs(30)) + client::Client::new(io, Some(1000), time::Duration::from_secs(30)) .await .context("Failed to create client") } - /// Launches an instance of Northstar with the test container and - /// resource installed. - pub async fn launch_install_test_container() -> Result { - let mut runtime = Self::launch().await?; - runtime.install_test_resource().await?; - runtime.install_test_container().await?; - Ok(runtime) - } - pub async fn stop(&mut self, container: &str, timeout: u64) -> Result<()> { self.client.kill(container, 15).await?; let container: Container = container.try_into()?; @@ -158,26 +158,15 @@ impl Northstar { Ok(()) } - pub async fn shutdown(self) -> Result<()> { - // Dropping the client closes the connection to the runtime - drop(self.client); - - // Stop the runtime - self.runtime - .shutdown() - .await - .context("Failed to stop the runtime")?; - - logger::assume("Closed listener", 5u64).await?; - - // Remove the tmpdir - self.tmpdir.close().expect("Failed to remove tmpdir"); + pub async fn shutdown(&mut self) -> Result<()> { + drop(self.client.shutdown().await); + logger::assume("Shutdown complete", 5u64).await?; Ok(()) } // Install a npk from a buffer pub async fn install(&mut self, npk: &[u8], repository: &str) -> Result<()> { - let f = NamedTempFile::new_in(self.tmpdir.path())?; + let f = NamedTempFile::new()?; fs::write(&f, npk).await?; self.client.install(f.path(), repository).await?; Ok(()) diff --git a/northstar-tests/test-container/manifest.yaml b/northstar-tests/test-container/manifest.yaml index c28409031..2487a077d 100644 --- a/northstar-tests/test-container/manifest.yaml +++ b/northstar-tests/test-container/manifest.yaml @@ -3,6 +3,9 @@ version: 0.0.1 init: /test-container uid: 1000 gid: 1000 +io: + stdout: pipe + stderr: pipe # cgroups: # memory: # limit_in_bytes: 10000000 @@ -35,15 +38,6 @@ mounts: version: 0.0.1 dir: test options: nosuid,nodev,noexec -io: - stdout: - log: - level: DEBUG - tag: test-container - stderr: - log: - level: DEBUG - tag: test-container rlimits: nproc: soft: 10000 diff --git a/northstar-tests/test-container/src/main.rs b/northstar-tests/test-container/src/main.rs index 4d2e68b7b..def03d394 100644 --- a/northstar-tests/test-container/src/main.rs +++ b/northstar-tests/test-container/src/main.rs @@ -18,6 +18,22 @@ struct Opt { command: Option, } +#[derive(Debug)] +enum Io { + Stdout, + Stderr, +} + +impl From<&str> for Io { + fn from(s: &str) -> Io { + match s { + "stdout" => Io::Stdout, + "stderr" => Io::Stderr, + _ => panic!("Invalid io: {}", s), + } + } +} + #[derive(Debug, Parser)] enum Command { Cat { @@ -25,13 +41,15 @@ enum Command { path: PathBuf, }, Crash, - Echo { - message: Vec, - }, Exit { code: i32, }, Inspect, + Print { + message: String, + #[structopt(short, long, parse(from_str), default_value = "stdout")] + io: Io, + }, Touch { path: PathBuf, }, @@ -49,15 +67,15 @@ fn main() -> Result<()> { let command = Opt::parse().command.unwrap_or(Command::Sleep); println!("Executing \"{:?}\"", command); match command { + Command::CallDeleteModule { flags } => call_delete_module(flags)?, Command::Cat { path } => cat(&path)?, Command::Crash => crash(), - Command::Echo { message } => echo(&message), Command::Exit { code } => exit(code), Command::Inspect => inspect(), - Command::Touch { path } => touch(&path)?, + Command::Print { message, io } => print(&message, &io), Command::Sleep => (), + Command::Touch { path } => touch(&path)?, Command::Write { message, path } => write(&message, path.as_path())?, - Command::CallDeleteModule { flags } => call_delete_module(flags)?, }; sleep(); @@ -92,8 +110,11 @@ fn crash() { panic!("witness me!"); } -fn echo(message: &[String]) { - println!("{}", message.join(" ")); +fn print(message: &str, io: &Io) { + match io { + Io::Stdout => println!("{}", message), + Io::Stderr => eprintln!("{}", message), + } } fn exit(code: i32) { diff --git a/northstar-tests/tests/examples.rs b/northstar-tests/tests/examples.rs index 836445616..718917e2a 100644 --- a/northstar-tests/tests/examples.rs +++ b/northstar-tests/tests/examples.rs @@ -1,13 +1,12 @@ use logger::assume; use northstar::api::model::{ExitStatus, Notification}; -use northstar_tests::{containers::*, logger, runtime::Northstar, test}; +use northstar_tests::{containers::*, logger, runtime::client, test}; // Start crashing example test!(crashing, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_CRASHING_NPK, "test-0").await?; - runtime.start(EXAMPLE_CRASHING).await?; - runtime + client().install(EXAMPLE_CRASHING_NPK, "test-0").await?; + client().start(EXAMPLE_CRASHING).await?; + client() .assume_notification( |n| { matches!( @@ -21,43 +20,39 @@ test!(crashing, { 20, ) .await?; - runtime.shutdown().await }); // Start console example test!(console, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_CONSOLE_NPK, "test-0").await?; - runtime.start(EXAMPLE_CONSOLE).await?; + client().install(EXAMPLE_CONSOLE_NPK, "test-0").await?; + client().start(EXAMPLE_CONSOLE).await?; // The console example stop itself - so wait for it... assume("Client console:0.0.1 connected", 5).await?; assume("Killing console:0.0.1 with SIGTERM", 5).await?; - runtime.shutdown().await }); // Start cpueater example and assume log message test!(cpueater, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_CPUEATER_NPK, "test-0").await?; - runtime.start(EXAMPLE_CPUEATER).await?; + client().install(EXAMPLE_CPUEATER_NPK, "test-0").await?; + client().start(EXAMPLE_CPUEATER).await?; assume("Eating CPU", 5).await?; - runtime.stop(EXAMPLE_CPUEATER, 10).await?; - runtime.shutdown().await + client().stop(EXAMPLE_CPUEATER, 10).await?; }); // Start hello-ferris example test!(hello_ferris, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_FERRIS_NPK, "test-0").await?; - runtime.install(EXAMPLE_MESSAGE_0_0_1_NPK, "test-0").await?; - runtime.install(EXAMPLE_HELLO_FERRIS_NPK, "test-0").await?; - runtime.start(EXAMPLE_HELLO_FERRIS).await?; + client().install(EXAMPLE_FERRIS_NPK, "test-0").await?; + client() + .install(EXAMPLE_MESSAGE_0_0_1_NPK, "test-0") + .await?; + client().install(EXAMPLE_HELLO_FERRIS_NPK, "test-0").await?; + client().start(EXAMPLE_HELLO_FERRIS).await?; assume("Hello once more from 0.0.1!", 5).await?; // The hello-ferris example terminates after printing something. - // Wait for the notification that it stopped, otherwise the runtime + // Wait for the notification that it stopped, otherwise the client() // will try to shutdown the application which is already exited. - runtime + client() .assume_notification( |n| { matches!( @@ -71,18 +66,17 @@ test!(hello_ferris, { 15, ) .await?; - - runtime.shutdown().await }); // Start hello-resource example test!(hello_resource, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_MESSAGE_0_0_2_NPK, "test-0").await?; - runtime + client() + .install(EXAMPLE_MESSAGE_0_0_2_NPK, "test-0") + .await?; + client() .install(EXAMPLE_HELLO_RESOURCE_NPK, "test-0") .await?; - runtime.start(EXAMPLE_HELLO_RESOURCE).await?; + client().start(EXAMPLE_HELLO_RESOURCE).await?; assume( "0: Content of /message/hello: Hello once more from v0.0.2!", 5, @@ -93,42 +87,33 @@ test!(hello_resource, { 5, ) .await?; - runtime.shutdown().await }); // Start inspect example test!(inspect, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_INSPECT_NPK, "test-0").await?; - runtime.start(EXAMPLE_INSPECT).await?; - runtime.stop(EXAMPLE_INSPECT, 5).await?; - // TODO - runtime.shutdown().await + client().install(EXAMPLE_INSPECT_NPK, "test-0").await?; + client().start(EXAMPLE_INSPECT).await?; + client().stop(EXAMPLE_INSPECT, 5).await?; }); // Start memeater example // test!(memeater, { -// let mut runtime = Northstar::launch().await?; -// runtime.install(&EXAMPLE_MEMEATER_NPK, "test-0").await?; -// runtime.start(EXAMPLE_MEMEATER).await?; -// assume("Process memeater:0.0.1 is out of memory", 20).await?; -// runtime.shutdown().await +// let mut client() = Northstar::launch().await?; +// client().install(&EXAMPLE_MEMEATER_NPK, "test-0").await?; +// client().start(EXAMPLE_MEMEATER).await?; +// assume("Process memeater:0.0.1 is out of memory", 20).await // }); // Start persistence example and check output test!(persistence, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_PERSISTENCE_NPK, "test-0").await?; - runtime.start(EXAMPLE_PERSISTENCE).await?; + client().install(EXAMPLE_PERSISTENCE_NPK, "test-0").await?; + client().start(EXAMPLE_PERSISTENCE).await?; assume("Writing Hello! to /data/file", 5).await?; assume("Content of /data/file: Hello!", 5).await?; - runtime.shutdown().await }); // Start seccomp example test!(seccomp, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_SECCOMP_NPK, "test-0").await?; - runtime.start(EXAMPLE_SECCOMP).await?; - runtime.shutdown().await + client().install(EXAMPLE_SECCOMP_NPK, "test-0").await?; + client().start(EXAMPLE_SECCOMP).await?; }); diff --git a/northstar-tests/tests/tests.rs b/northstar-tests/tests/tests.rs index f47403866..505c67556 100644 --- a/northstar-tests/tests/tests.rs +++ b/northstar-tests/tests/tests.rs @@ -1,4 +1,5 @@ -use anyhow::Result; +use std::path::{Path, PathBuf}; + use futures::{SinkExt, StreamExt}; use log::debug; use logger::assume; @@ -6,8 +7,7 @@ use northstar::api::{ self, model::{self, ConnectNack, ExitStatus, Notification}, }; -use northstar_tests::{containers::*, logger, runtime::Northstar, test}; -use std::path::{Path, PathBuf}; +use northstar_tests::{containers::*, logger, runtime::client, test}; use tokio::{ io::{AsyncRead, AsyncWrite}, net::UnixStream, @@ -18,175 +18,198 @@ test!(logger_smoketest, { debug!("Yippie"); assume("Yippie", 3).await?; assert!(assume("Juhuuu!", 1).await.is_err()); - Result::<()>::Ok(()) -}); - -// Smoke test the runtime startup and shutdown -test!(runtime_launch, { - Northstar::launch().await?.shutdown().await }); // Install and uninstall is a loop. After a number of installation // try to start the test container test!(install_uninstall_test_container, { - let mut runtime = Northstar::launch().await?; for _ in 0u32..10 { - runtime.install_test_container().await?; - runtime.uninstall_test_container().await?; + client().install_test_container().await?; + client().uninstall_test_container().await?; } - runtime.shutdown().await }); // Install a container that already exists with the same name and version test!(install_duplicate, { - let mut runtime = Northstar::launch().await?; - runtime.install_test_container().await?; - assert!(runtime.install_test_container().await.is_err()); - runtime.shutdown().await + client().install_test_container().await?; + client().install_test_resource().await?; + assert!(client().install_test_container().await.is_err()); }); // Install a container that already exists in another repository test!(install_duplicate_other_repository, { - let mut runtime = Northstar::launch().await?; - runtime.install(TEST_CONTAINER_NPK, "test-0").await?; - assert!(runtime.install(TEST_CONTAINER_NPK, "test-1").await.is_err()); - runtime.shutdown().await + client().install(TEST_CONTAINER_NPK, "test-0").await?; + assert!(client() + .install(TEST_CONTAINER_NPK, "test-1") + .await + .is_err()); }); // Start and stop a container multiple times test!(start_stop, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; for _ in 0..10u32 { - runtime.start_with_args(TEST_CONTAINER, ["sleep"]).await?; + client().start_with_args(TEST_CONTAINER, ["sleep"]).await?; assume("Sleeping", 5u64).await?; - runtime.stop(TEST_CONTAINER, 5).await?; + client().stop(TEST_CONTAINER, 5).await?; assume("Process test-container:0.0.1 exited", 5).await?; } +}); - runtime.shutdown().await +// Install and uninsteall the example npks +test!(install_uninstall_examples, { + client().install(EXAMPLE_CPUEATER_NPK, "test-0").await?; + client().install(EXAMPLE_CONSOLE_NPK, "test-0").await?; + client().install(EXAMPLE_CRASHING_NPK, "test-0").await?; + client().install(EXAMPLE_FERRIS_NPK, "test-0").await?; + client().install(EXAMPLE_HELLO_FERRIS_NPK, "test-0").await?; + client() + .install(EXAMPLE_HELLO_RESOURCE_NPK, "test-0") + .await?; + client().install(EXAMPLE_INSPECT_NPK, "test-0").await?; + client().install(EXAMPLE_MEMEATER_NPK, "test-0").await?; + client() + .install(EXAMPLE_MESSAGE_0_0_1_NPK, "test-0") + .await?; + client() + .install(EXAMPLE_MESSAGE_0_0_2_NPK, "test-0") + .await?; + client().install(EXAMPLE_PERSISTENCE_NPK, "test-0").await?; + client().install(EXAMPLE_SECCOMP_NPK, "test-0").await?; + client().install(TEST_CONTAINER_NPK, "test-0").await?; + client().install(TEST_RESOURCE_NPK, "test-0").await?; + + client().uninstall(EXAMPLE_CPUEATER).await?; + client().uninstall(EXAMPLE_CONSOLE).await?; + client().uninstall(EXAMPLE_CRASHING).await?; + client().uninstall(EXAMPLE_FERRIS).await?; + client().uninstall(EXAMPLE_HELLO_FERRIS).await?; + client().uninstall(EXAMPLE_HELLO_RESOURCE).await?; + client().uninstall(EXAMPLE_INSPECT).await?; + client().uninstall(EXAMPLE_MEMEATER).await?; + client().uninstall(EXAMPLE_MESSAGE_0_0_1).await?; + client().uninstall(EXAMPLE_MESSAGE_0_0_2).await?; + client().uninstall(EXAMPLE_PERSISTENCE).await?; + client().uninstall(EXAMPLE_SECCOMP).await?; + client().uninstall(TEST_CONTAINER).await?; + client().uninstall(TEST_RESOURCE).await?; }); -// Mount and umount all containers known to the runtime +// Mount and umount all containers known to the client() test!(mount_umount, { - let mut runtime = Northstar::launch().await?; - runtime.install(EXAMPLE_CPUEATER_NPK, "test-0").await?; - runtime.install(EXAMPLE_CONSOLE_NPK, "test-0").await?; - runtime.install(EXAMPLE_CRASHING_NPK, "test-0").await?; - runtime.install(EXAMPLE_FERRIS_NPK, "test-0").await?; - runtime.install(EXAMPLE_HELLO_FERRIS_NPK, "test-0").await?; - runtime + client().install(EXAMPLE_CPUEATER_NPK, "test-0").await?; + client().install(EXAMPLE_CONSOLE_NPK, "test-0").await?; + client().install(EXAMPLE_CRASHING_NPK, "test-0").await?; + client().install(EXAMPLE_FERRIS_NPK, "test-0").await?; + client().install(EXAMPLE_HELLO_FERRIS_NPK, "test-0").await?; + client() .install(EXAMPLE_HELLO_RESOURCE_NPK, "test-0") .await?; - runtime.install(EXAMPLE_INSPECT_NPK, "test-0").await?; - runtime.install(EXAMPLE_MEMEATER_NPK, "test-0").await?; - runtime.install(EXAMPLE_MESSAGE_0_0_1_NPK, "test-0").await?; - runtime.install(EXAMPLE_MESSAGE_0_0_2_NPK, "test-0").await?; - runtime.install(EXAMPLE_PERSISTENCE_NPK, "test-0").await?; - runtime.install(EXAMPLE_SECCOMP_NPK, "test-0").await?; - runtime.install(TEST_CONTAINER_NPK, "test-0").await?; - runtime.install(TEST_RESOURCE_NPK, "test-0").await?; - - let mut containers = runtime.containers().await?; - runtime + client().install(EXAMPLE_INSPECT_NPK, "test-0").await?; + client().install(EXAMPLE_MEMEATER_NPK, "test-0").await?; + client() + .install(EXAMPLE_MESSAGE_0_0_1_NPK, "test-0") + .await?; + client() + .install(EXAMPLE_MESSAGE_0_0_2_NPK, "test-0") + .await?; + client().install(EXAMPLE_PERSISTENCE_NPK, "test-0").await?; + client().install(EXAMPLE_SECCOMP_NPK, "test-0").await?; + client().install(TEST_CONTAINER_NPK, "test-0").await?; + client().install(TEST_RESOURCE_NPK, "test-0").await?; + + let mut containers = client().containers().await?; + client() .mount(containers.drain(..).map(|c| c.container)) .await?; - let containers = &mut runtime.containers().await?; + let containers = &mut client().containers().await?; for c in containers.iter().filter(|c| c.mounted) { - runtime.umount(c.container.clone()).await?; + client().umount(c.container.clone()).await?; } - - runtime.shutdown().await }); // Try to stop a not started container and expect an Err test!(try_to_stop_unknown_container, { - let mut runtime = Northstar::launch().await?; let container = "foo:0.0.1:default"; - assert!(runtime.stop(container, 5).await.is_err()); - runtime.shutdown().await + assert!(client().stop(container, 5).await.is_err()); }); // Try to start a container which is not installed/known test!(try_to_start_unknown_container, { - let mut runtime = Northstar::launch().await?; let container = "unknown_application:0.0.12:asdf"; - assert!(runtime.start(container).await.is_err()); - runtime.shutdown().await + assert!(client().start(container).await.is_err()); }); // Try to start a container where a dependency is missing test!(try_to_start_containter_that_misses_a_resource, { - let mut runtime = Northstar::launch().await?; - runtime.install_test_container().await?; + client().install_test_container().await?; // The TEST_RESOURCE is not installed. - assert!(runtime.start(TEST_CONTAINER).await.is_err()); - runtime.shutdown().await + assert!(client().start(TEST_CONTAINER).await.is_err()); }); // Start a container that uses a resource test!(check_test_container_resource_usage, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; // Start the test_container process - runtime + client() .start_with_args(TEST_CONTAINER, ["cat", "/resource/hello"]) .await?; assume("hello from test resource", 5).await?; // The container might have finished at this point - runtime.stop(TEST_CONTAINER, 5).await?; - - runtime.uninstall_test_container().await?; - runtime.uninstall_test_resource().await?; + client().stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().uninstall_test_container().await?; + client().uninstall_test_resource().await?; }); // Try to uninstall a started container test!(try_to_uninstall_a_started_container, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; - runtime.start_with_args(TEST_CONTAINER, ["sleep"]).await?; - assume("test-container: Sleeping...", 5u64).await?; + client().start_with_args(TEST_CONTAINER, ["sleep"]).await?; + assume("Sleeping...", 5u64).await?; - let result = runtime.uninstall_test_container().await; + let result = client().uninstall_test_container().await; assert!(result.is_err()); - runtime.stop(TEST_CONTAINER, 5).await?; - - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); test!(start_mounted_container_with_not_mounted_resource, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; // Start a container that depends on a resource. - runtime.start_with_args(TEST_CONTAINER, ["sleep"]).await?; - assume("test-container: Sleeping...", 5u64).await?; - runtime.stop(TEST_CONTAINER, 5).await?; + client().start_with_args(TEST_CONTAINER, ["sleep"]).await?; + assume("Sleeping...", 5u64).await?; + client().stop(TEST_CONTAINER, 5).await?; // Umount the resource and start the container again. - runtime.umount(TEST_RESOURCE).await?; + client().umount(TEST_RESOURCE).await?; - runtime.start_with_args(TEST_CONTAINER, ["sleep"]).await?; - assume("test-container: Sleeping...", 5u64).await?; + client().start_with_args(TEST_CONTAINER, ["sleep"]).await?; + assume("Sleeping...", 5u64).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // The test is flaky and needs to listen for notifications // in order to be implemented correctly test!(container_crash_exit, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; for _ in 0..10 { - runtime.start_with_args(TEST_CONTAINER, ["crash"]).await?; - runtime + client().start_with_args(TEST_CONTAINER, ["crash"]).await?; + client() .assume_notification( |n| { matches!( @@ -202,150 +225,169 @@ test!(container_crash_exit, { .await?; } - runtime.uninstall_test_container().await?; - runtime.uninstall_test_resource().await?; - - runtime.shutdown().await + client().uninstall_test_container().await?; + client().uninstall_test_resource().await?; }); // Check uid. In the manifest of the test container the uid // is set to 1000 test!(container_uses_correct_uid, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("getuid: 1000", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Check gid. In the manifest of the test container the gid // is set to 1000 test!(container_uses_correct_gid, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("getgid: 1000", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Check parent pid. Northstar starts an init process which must have pid 1. test!(container_ppid_must_be_init, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("getppid: 1", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Check session id which needs to be pid of init test!(container_sid_must_be_init_or_none, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("getsid: 1", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // The test container only gets the cap_kill capability. See the manifest test!(container_shall_only_have_configured_capabilities, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("caps bounding: \\{\\}", 10).await?; assume("caps effective: \\{\\}", 10).await?; assume("caps permitted: \\{\\}", 10).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // The test container has a configured resource limit of tasks test!(container_rlimits, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume( "Max processes 10000 20000 processes", 10, ) .await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); -// Check whether after a runtime start, container start and shutdown +// Check whether after a client() start, container start and shutdown // any file descriptor is leaked -test!( - start_stop_runtime_and_containers_shall_not_leak_file_descriptors, - { - /// Collect a set of files in /proc/$$/fd - fn fds() -> Result, std::io::Error> { - let mut links = std::fs::read_dir("/proc/self/fd")? - .filter_map(Result::ok) - .flat_map(|entry| entry.path().read_link()) - .collect::>(); - links.sort(); - Ok(links) - } - // Collect list of fds - let before = fds()?; - - let mut runtime = Northstar::launch_install_test_container().await?; - - runtime.start_with_args(TEST_CONTAINER, ["sleep"]).await?; - assume("test-container: Sleeping", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - - let result = runtime.shutdown().await; - - // Compare the list of fds before and after the RT run. - assert_eq!(before, fds()?); - - result +test!(start_stop_and_container_shall_not_leak_file_descriptors, { + /// Collect a set of files in /proc/$$/fd + fn fds() -> Result, std::io::Error> { + let mut links = std::fs::read_dir("/proc/self/fd")? + .filter_map(Result::ok) + .flat_map(|entry| entry.path().read_link()) + .collect::>(); + links.sort(); + Ok(links) } -); + + let before = fds()?; + + client().install_test_container().await?; + client().install_test_resource().await?; + + client().start_with_args(TEST_CONTAINER, ["sleep"]).await?; + assume("Sleeping", 5).await?; + client().stop(TEST_CONTAINER, 5).await?; + + client().uninstall_test_container().await?; + client().uninstall_test_resource().await?; + + // Compare the list of fds before and after the RT run. + assert_eq!(before, fds()?); + + let result = client().shutdown().await; + + assert!(result.is_ok()); +}); // Check open file descriptors in the test container that should be // stdin: /dev/null // stdout: some pipe // stderr: /dev/null test!(container_shall_only_have_configured_fds, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("/proc/self/fd/0: /dev/null", 5).await?; - assume("/proc/self/fd/1: pipe:.*", 5).await?; - assume("/proc/self/fd/2: pipe:.*", 5).await?; + assume("/proc/self/fd/1: socket", 5).await?; + assume("/proc/self/fd/2: socket", 5).await?; assume("total: 3", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Check if /proc is mounted ro test!(proc_is_mounted_ro, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume("proc /proc proc ro,", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Check that mount flags nosuid,nodev,noexec are properly set for bind mounts // assumption: mount flags are always listed the same order (according mount.h) // note: MS_REC is not explicitly listed an cannot be checked with this test test!(mount_flags_are_set_for_bind_mounts, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; assume( "/.* /resource \\w+ ro,(\\w+,)*nosuid,(\\w+,)*nodev,(\\w+,)*noexec", 5, ) .await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // The test container only gets the cap_kill capability. See the manifest test!(selinux_mounted_squasfs_has_correct_context, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime.start_with_args(TEST_CONTAINER, ["inspect"]).await?; + client().install_test_container().await?; + client().install_test_resource().await?; + client() + .start_with_args(TEST_CONTAINER, ["inspect"]) + .await?; // Only expect selinux context if system supports it if Path::new("/sys/fs/selinux/enforce").exists() { assume( @@ -356,36 +398,36 @@ test!(selinux_mounted_squasfs_has_correct_context, { } else { assume("/.* squashfs (\\w+,)*", 5).await?; } - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Call syscall with specifically allowed argument test!(seccomp_allowed_syscall_with_allowed_arg, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime + client().install_test_container().await?; + client().install_test_resource().await?; + client() .start_with_args(TEST_CONTAINER, ["call-delete-module", "1"]) .await?; assume("delete_module syscall was successful", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Call syscall with argument allowed by bitmask test!(seccomp_allowed_syscall_with_masked_arg, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime + client().install_test_container().await?; + client().install_test_resource().await?; + client() .start_with_args(TEST_CONTAINER, ["call-delete-module", "4"]) .await?; assume("delete_module syscall was successful", 5).await?; - runtime.stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + client().stop(TEST_CONTAINER, 5).await?; }); // Call syscall with prohibited argument test!(seccomp_allowed_syscall_with_prohibited_arg, { - let mut runtime = Northstar::launch_install_test_container().await?; - runtime + client().install_test_container().await?; + client().install_test_resource().await?; + client() .start_with_args(TEST_CONTAINER, ["call-delete-module", "7"]) .await?; @@ -396,15 +438,15 @@ test!(seccomp_allowed_syscall_with_prohibited_arg, { .. } if signal == &31) }; - runtime.assume_notification(n, 5).await?; - runtime.shutdown().await + client().assume_notification(n, 5).await?; }); // Iterate all exit codes in the u8 range test!(exitcodes, { - let mut runtime = Northstar::launch_install_test_container().await?; + client().install_test_container().await?; + client().install_test_resource().await?; for c in &[0, 1, 10, 127, 128, 255] { - runtime + client() .start_with_args(TEST_CONTAINER, ["exit".to_string(), c.to_string()]) .await?; let n = |n: &Notification| { @@ -413,39 +455,18 @@ test!(exitcodes, { .. } if code == c) }; - runtime.assume_notification(n, 5).await?; + client().assume_notification(n, 5).await?; } - runtime.shutdown().await }); -// Open many connections to the runtime -test!(open_many_connections_to_the_runtime_and_shutdown, { - let runtime = Northstar::launch().await?; - - let mut clients = Vec::new(); - for _ in 0..500 { - let client = runtime.client().await?; - clients.push(client); - } - - let result = runtime.shutdown().await; - - for client in &mut clients { - assert!(client.containers().await.is_err()); - } - clients.clear(); - - result -}); - -// Verify that the runtime reject a version mismatch in Connect +// Verify that the client() reject a version mismatch in Connect test!(check_api_version_on_connect, { - let runtime = Northstar::launch().await?; - trait AsyncReadWrite: AsyncRead + AsyncWrite + Unpin + Send {} impl AsyncReadWrite for T {} - let mut connection = api::codec::framed(UnixStream::connect(&runtime.console).await?); + let mut connection = api::codec::Framed::new( + UnixStream::connect(&northstar_tests::runtime::console().path()).await?, + ); // Send a connect with an version unequal to the one defined in the model let mut version = api::model::version(); @@ -469,6 +490,20 @@ test!(check_api_version_on_connect, { let expected_message = model::Message::new_connect(model::Connect::Nack { error }); assert_eq!(connack, expected_message); +}); + +// Check printing on stdout and stderr +test!(stdout_stderr, { + client().install_test_container().await?; + client().install_test_resource().await?; + + let args = ["print", "--io", "stdout", "hello stdout"]; + client().start_with_args(TEST_CONTAINER, args).await?; + assume("hello stdout", 10).await?; + client().stop(TEST_CONTAINER, 5).await?; - runtime.shutdown().await + let args = ["print", "--io", "stderr", "hello stderr"]; + client().start_with_args(TEST_CONTAINER, args).await?; + assume("hello stderr", 10).await?; + client().stop(TEST_CONTAINER, 5).await?; }); diff --git a/northstar/Cargo.toml b/northstar/Cargo.toml index 2bf2659ac..a964ea1d0 100644 --- a/northstar/Cargo.toml +++ b/northstar/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "northstar" -version = "0.6.4" +version = "0.7.0-dev" authors = ["ESRLabs"] edition = "2021" build = "build.rs" @@ -24,6 +24,7 @@ ed25519-dalek = { version = "1.0.1", optional = true } futures = { version = "0.3.21", features = ["thread-pool"], optional = true } hex = { version = "0.4.3", optional = true } humanize-rs = { version = "0.1.5", optional = true } +humantime = { version = "2.1.0", optional = true } inotify = { version = "0.10.0", features = ["stream"], optional = true } itertools = { version = "0.10.1", optional = true } lazy_static = { version = "1.4.0", optional = true } @@ -44,7 +45,7 @@ serde_yaml = { version = "0.8.21", optional = true } sha2 = { version = "0.10.2", optional = true } tempfile = { version = "3.3.0", optional = true } thiserror = "1.0.30" -tokio = { version = "1.17.0", features = ["fs", "io-std", "io-util", "macros", "process", "rt", "sync", "time"], optional = true } +tokio = { version = "1.17.0", features = ["fs", "io-std", "io-util", "macros", "process", "rt-multi-thread", "sync", "time"], optional = true } tokio-eventfd = { version = "0.2.0", optional = true } tokio-util = { version = "0.7.0", features = ["codec", "io"], optional = true } url = { version = "2.2.2", features = ["serde"], optional = true } @@ -92,9 +93,11 @@ runtime = [ "caps", "cgroups-rs", "devicemapper", + "derive-new", "ed25519-dalek", "futures", "hex", + "humantime", "itertools", "inotify", "lazy_static", @@ -123,14 +126,9 @@ seccomp = [ [dev-dependencies] anyhow = "1.0.54" -nix = "0.23.0" proptest = "1.0.0" -tempfile = "3.3.0" -tokio = { version = "1.17.0", features = ["rt-multi-thread"] } +serde_json = "1.0.68" [build-dependencies] anyhow = { version = "1.0.54", optional = true } bindgen = { version = "0.59.1", default-features = false, features = ["runtime"], optional = true } -nix = { version = "0.23.0", optional = true } -tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread"], optional = true } - diff --git a/northstar/src/api/client.rs b/northstar/src/api/client.rs index 95b409951..4b5ca0188 100644 --- a/northstar/src/api/client.rs +++ b/northstar/src/api/client.rs @@ -1,5 +1,5 @@ use super::{ - codec::{self, framed}, + codec, model::{ self, Connect, Container, ContainerData, ContainerStats, Message, MountResult, Notification, RepositoryId, Request, Response, @@ -21,10 +21,13 @@ use std::{ use thiserror::Error; use tokio::{ fs, - io::{self, AsyncRead, AsyncWrite}, + io::{self, AsyncRead, AsyncWrite, BufWriter}, time, }; +/// Default buffer size for installation transfers +const BUFFER_SIZE: usize = 1024 * 1024; + /// API error #[allow(missing_docs)] #[derive(Error, Debug)] @@ -104,7 +107,7 @@ pub async fn connect( notifications: Option, timeout: time::Duration, ) -> Result, Error> { - let mut connection = framed(io); + let mut connection = codec::Framed::with_capacity(io, BUFFER_SIZE); // Send connect message let connect = Connect::Connect { version: model::version(), @@ -431,7 +434,7 @@ impl<'a, T: AsyncRead + AsyncWrite + Unpin> Client { /// ``` pub async fn install(&mut self, npk: &Path, repository: &str) -> Result<(), Error> { self.fused()?; - let mut file = fs::File::open(npk).await.map_err(Error::Io)?; + let file = fs::File::open(npk).await.map_err(Error::Io)?; let size = file.metadata().await.unwrap().len(); let request = Request::Install { repository: repository.into(), @@ -443,12 +446,15 @@ impl<'a, T: AsyncRead + AsyncWrite + Unpin> Client { Error::Stopped })?; - io::copy(&mut file, &mut self.connection) - .await - .map_err(|e| { - self.fuse(); - Error::Io(e) - })?; + self.connection.flush().await?; + debug_assert!(self.connection.write_buffer().is_empty()); + + let mut reader = io::BufReader::with_capacity(BUFFER_SIZE, file); + let mut writer = BufWriter::with_capacity(BUFFER_SIZE, self.connection.get_mut()); + io::copy_buf(&mut reader, &mut writer).await.map_err(|e| { + self.fuse(); + Error::Io(e) + })?; loop { match self.connection.next().await { diff --git a/northstar/src/api/codec.rs b/northstar/src/api/codec.rs index b7616e26b..b6a879cb0 100644 --- a/northstar/src/api/codec.rs +++ b/northstar/src/api/codec.rs @@ -1,43 +1,51 @@ use super::model; -use futures::Stream; -use std::{ - cmp::min, - io::ErrorKind, - pin::Pin, - task::{self, Poll}, -}; -use task::Context; +use std::io::ErrorKind; use tokio::io::{self, AsyncRead, AsyncWrite}; -use tokio_util::codec::{Decoder, Encoder, FramedParts}; +use tokio_util::codec::{Decoder, Encoder}; /// Newline delimited json codec for api::Message that on top implements AsyncRead and Write pub struct Framed { inner: tokio_util::codec::Framed, } -impl Framed { - /// Consumes the Framed, returning its underlying I/O stream, the buffer with unprocessed data, and the codec. - pub fn into_parts(self) -> FramedParts { - self.inner.into_parts() +impl Framed { + /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this + /// I/O object, using [`Decoder`] and [`Encoder`] to read and write the raw data. + pub fn new(inner: T) -> Framed { + Framed { + inner: tokio_util::codec::Framed::new(inner, Codec::default()), + } + } + + /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this + /// I/O object, using [`Decoder`] and [`Encoder`] to read and write the raw data, + /// with a specific read buffer initial capacity. + /// [`split`]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html#method.split + pub fn with_capacity(inner: T, capacity: usize) -> Framed { + Framed { + inner: tokio_util::codec::Framed::with_capacity(inner, Codec::default(), capacity), + } } +} + +impl std::ops::Deref for Framed { + type Target = tokio_util::codec::Framed; - /// Consumes the Framed, returning its underlying I/O stream. - pub fn into_inner(self) -> T { - self.inner.into_inner() + fn deref(&self) -> &Self::Target { + &self.inner } } -/// Constructs a new Framed with Codec from `io` -pub fn framed(io: T) -> Framed { - Framed { - inner: tokio_util::codec::Framed::new(io, Codec::default()), +impl std::ops::DerefMut for Framed { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner } } /// Newline delimited json #[derive(Default)] pub struct Codec { - lines: tokio_util::codec::LinesCodec, + inner: tokio_util::codec::LinesCodec, } impl Decoder for Codec { @@ -45,7 +53,7 @@ impl Decoder for Codec { type Error = io::Error; fn decode(&mut self, src: &mut bytes::BytesMut) -> Result, Self::Error> { - self.lines + self.inner .decode(src) .map_err(|e| io::Error::new(ErrorKind::Other, e))? // See LinesCodecError. .as_deref() @@ -63,83 +71,12 @@ impl Encoder for Codec { item: model::Message, dst: &mut bytes::BytesMut, ) -> Result<(), Self::Error> { - self.lines + self.inner .encode(serde_json::to_string(&item)?.as_str(), dst) .map_err(|e| io::Error::new(ErrorKind::Other, e)) } } -impl AsyncWrite for Framed { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - assert!(self.inner.write_buffer().is_empty()); - let t: &mut T = self.inner.get_mut(); - AsyncWrite::poll_write(Pin::new(t), cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let t: &mut T = self.inner.get_mut(); - AsyncWrite::poll_flush(Pin::new(t), cx) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let t: &mut T = self.inner.get_mut(); - AsyncWrite::poll_shutdown(Pin::new(t), cx) - } -} - -impl AsyncRead for Framed { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut io::ReadBuf<'_>, - ) -> Poll> { - if self.inner.read_buffer().is_empty() { - let t: &mut T = self.inner.get_mut(); - AsyncRead::poll_read(Pin::new(t), cx, buf) - } else { - let n = min(buf.remaining(), self.inner.read_buffer().len()); - buf.put_slice(&self.inner.read_buffer_mut().split_to(n)); - Poll::Ready(Ok(())) - } - } -} - -impl Stream for Framed { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let framed = Pin::new(&mut self.inner); - framed.poll_next(cx) - } -} - -impl futures::sink::Sink for Framed { - type Error = io::Error; - - fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_ready(cx) - } - - fn start_send(mut self: Pin<&mut Self>, item: model::Message) -> Result<(), Self::Error> { - Pin::new(&mut self.inner).start_send(item) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_flush(cx) - } - - fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_close(cx) - } -} - #[cfg(test)] mod tests { use std::convert::TryInto; diff --git a/northstar/src/common/container.rs b/northstar/src/common/container.rs index 7edacd6c5..cbe3431dd 100644 --- a/northstar/src/common/container.rs +++ b/northstar/src/common/container.rs @@ -12,9 +12,8 @@ use std::{ use thiserror::Error; /// Container identification -#[derive(Clone, Eq, PartialOrd, PartialEq, Debug, Hash, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Eq, PartialOrd, Ord, PartialEq, Debug, Hash, JsonSchema)] pub struct Container { - #[serde(flatten)] inner: Arc, } @@ -98,7 +97,26 @@ impl, N: TryInto, V: ToString> TryFrom<(N, V)> f } } -#[derive(Eq, PartialOrd, PartialEq, Debug, Hash, Serialize, Deserialize, JsonSchema)] +impl Serialize for Container { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&format!("{}:{}", self.inner.name, self.inner.version)) + } +} + +impl<'de> Deserialize<'de> for Container { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Container::try_from(value.as_str()).map_err(serde::de::Error::custom) + } +} + +#[derive(Eq, PartialOrd, PartialEq, Ord, Debug, Hash, Serialize, Deserialize, JsonSchema)] struct Inner { name: Name, version: Version, diff --git a/northstar/src/common/name.rs b/northstar/src/common/name.rs index be262580d..a5e1fca81 100644 --- a/northstar/src/common/name.rs +++ b/northstar/src/common/name.rs @@ -8,7 +8,9 @@ use std::{ use thiserror::Error; /// Name of a container -#[derive(Clone, Eq, PartialOrd, PartialEq, Debug, Hash, Serialize, Deserialize, JsonSchema)] +#[derive( + Clone, Eq, PartialOrd, Ord, PartialEq, Debug, Hash, Serialize, Deserialize, JsonSchema, +)] #[serde(try_from = "String")] pub struct Name(String); diff --git a/northstar/src/common/non_null_string.rs b/northstar/src/common/non_null_string.rs index d76fe8c5f..782034dd6 100644 --- a/northstar/src/common/non_null_string.rs +++ b/northstar/src/common/non_null_string.rs @@ -2,6 +2,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ convert::{TryFrom, TryInto}, + ffi::CString, fmt::{Display, Formatter}, ops::Deref, }; @@ -37,6 +38,12 @@ impl Deref for NonNullString { } } +impl From for CString { + fn from(s: NonNullString) -> Self { + CString::new(s.0.as_bytes()).unwrap() + } +} + impl TryFrom for NonNullString { type Error = InvalidNullChar; @@ -70,3 +77,10 @@ impl InvalidNullChar { self.0 } } + +#[test] +fn try_from() { + assert!(NonNullString::try_from("hel\0lo").is_err()); + assert!(NonNullString::try_from("hello").is_ok()); + assert!(NonNullString::try_from("hello\0").is_err()); +} diff --git a/northstar/src/common/version.rs b/northstar/src/common/version.rs index 171ba0449..83b128ab2 100644 --- a/northstar/src/common/version.rs +++ b/northstar/src/common/version.rs @@ -122,6 +122,12 @@ impl PartialOrd for Version { } } +impl Ord for Version { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.partial_cmp(other).unwrap() + } +} + impl JsonSchema for Version { fn schema_name() -> String { "Version".to_string() diff --git a/northstar/src/lib.rs b/northstar/src/lib.rs index 85774bdd6..7cdb5c031 100644 --- a/northstar/src/lib.rs +++ b/northstar/src/lib.rs @@ -22,7 +22,3 @@ pub mod runtime; #[cfg(feature = "seccomp")] /// Support for seccomp syscall filtering pub mod seccomp; - -/// Northstar internal utilities -#[cfg(feature = "runtime")] -mod util; diff --git a/northstar/src/npk/manifest.rs b/northstar/src/npk/manifest.rs index 329d8cb0d..083c2fe7b 100644 --- a/northstar/src/npk/manifest.rs +++ b/northstar/src/npk/manifest.rs @@ -1,5 +1,5 @@ use crate::{ - common::{name::Name, non_null_string::NonNullString, version::Version}, + common::{container::Container, name::Name, non_null_string::NonNullString, version::Version}, seccomp::{Seccomp, Selinux, SyscallRule}, }; use derive_more::Deref; @@ -60,7 +60,8 @@ pub struct Manifest { /// Resource limits pub rlimits: Option>, /// IO configuration - pub io: Option, + #[serde(default, skip_serializing_if = "is_default")] + pub io: Io, /// Optional custom data. The runtime doesnt use this. pub custom: Option, } @@ -69,6 +70,11 @@ impl Manifest { /// Manifest version supported by the runtime pub const VERSION: Version = Version::new(0, 1, 0); + /// Container that is specified in the manifest + pub fn container(&self) -> Container { + Container::new(self.name.clone(), self.version.clone()) + } + /// Read a manifest from `reader` pub fn from_reader(reader: R) -> Result { let manifest: Self = serde_yaml::from_reader(reader).map_err(Error::SerdeYaml)?; @@ -83,19 +89,24 @@ impl Manifest { fn verify(&self) -> Result<(), Error> { // Most optionals in the manifest are not valid for a resource container - if self.init.is_none() - && (self.args.is_some() - || self.env.is_some() - || self.autostart.is_some() - || self.cgroups.is_some() - || self.seccomp.is_some() - || self.capabilities.is_some() - || self.suppl_groups.is_some() - || self.io.is_some()) + + if let Some(init) = &self.init { + if NonNullString::try_from(init.display().to_string()).is_err() { + return Err(Error::Invalid( + "Init path must be a string without zero bytes".to_string(), + )); + } + } else if self.args.is_some() + || self.env.is_some() + || self.autostart.is_some() + || self.cgroups.is_some() + || self.seccomp.is_some() + || self.capabilities.is_some() + || self.suppl_groups.is_some() { return Err(Error::Invalid( "Resource containers must not define any of the following manifest entries:\ - args, env, autostart, cgroups, seccomp, capabilities, suppl_groups, io" + args, env, autostart, cgroups, seccomp, capabilities, suppl_groups, io" .to_string(), )); } @@ -108,6 +119,18 @@ impl Manifest { return Err(Error::Invalid("Invalid gid of 0".to_string())); } + // Check for reserved env variable names + if let Some(env) = &self.env { + for name in ["NAME", "VERSION", "NORTHSTAR_CONSOLE"] { + if env.keys().any(|k| name == k.as_str()) { + return Err(Error::Invalid(format!( + "Invalid env: resevered variable {}", + name + ))); + } + } + } + // Check for relative and overlapping bind mounts let mut prev_comps = vec![RootDir]; self.mounts @@ -426,31 +449,30 @@ pub enum Mount { } /// IO configuration for stdin, stdout, stderr -#[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize, JsonSchema)] +#[derive(Default, Clone, Eq, PartialEq, Debug, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Io { /// stdout configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub stdout: Option, + pub stdout: Output, /// stderr configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub stderr: Option, + pub stderr: Output, } /// Io redirection for stdout/stderr #[derive(Clone, Eq, PartialEq, Debug, Serialize, Deserialize, JsonSchema)] pub enum Output { - /// Inherit the runtimes stdout/stderr + /// Discard output + #[serde(rename = "discard")] + Discard, + /// Forward output to the logging system with level and optional tag #[serde(rename = "pipe")] Pipe, - /// Forward output to the logging system with level and optional tag - #[serde(rename = "log")] - Log { - /// Level - level: Level, - /// Tag - tag: String, - }, +} + +impl Default for Output { + fn default() -> Output { + Output::Discard + } } /// Log level @@ -822,7 +844,7 @@ mounts: options: noexec autostart: critical seccomp: - allow: + allow: fork: any waitpid: any cgroups: @@ -1081,10 +1103,7 @@ seccomp: capabilities: - CAP_NET_ADMIN io: - stdout: - log: - level: DEBUG - tag: test + stdout: pipe stderr: pipe cgroups: memory: diff --git a/northstar/src/runtime/cgroups.rs b/northstar/src/runtime/cgroups.rs index 905e7afc5..bd4bc4151 100644 --- a/northstar/src/runtime/cgroups.rs +++ b/northstar/src/runtime/cgroups.rs @@ -116,6 +116,7 @@ impl Hierarchy for RuntimeHierarchy { #[derive(Debug)] pub struct CGroups { + container: Container, cgroup: cgroups_rs::Cgroup, memory_monitor: MemoryMonitor, } @@ -166,14 +167,18 @@ impl CGroups { }; Ok(CGroups { + container: container.clone(), cgroup, memory_monitor, }) } pub async fn destroy(self) { + debug!("Stopping oom monitor of {}", self.container); self.memory_monitor.stop().await; - info!("Destroying cgroup"); + + info!("Destroying cgroup of {}", self.container); + assert!(self.cgroup.tasks().is_empty()); self.cgroup.delete().expect("Failed to remove cgroups"); } @@ -253,10 +258,7 @@ impl MemoryMonitor { 'outer: loop { select! { - _ = stop.cancelled() => { - debug!("Stopping oom monitor of {}", container); - break 'outer; - } + _ = stop.cancelled() => break 'outer, _ = tx.closed() => break 'outer, _ = event_fd.read(&mut buffer) => { 'inner: loop { @@ -274,6 +276,9 @@ impl MemoryMonitor { } } } + drop(event_control); + drop(oom_control); + drop(event_fd); }) }; @@ -306,10 +311,7 @@ impl MemoryMonitor { 'outer: loop { select! { - _ = stop.cancelled() => { - debug!("Stopping oom monitor of {}", container); - break 'outer; - } + _ = stop.cancelled() => break 'outer, _ = tx.closed() => break 'outer, _ = stream.next() => { let events = fs::read_to_string(&path).await.expect("Failed to read memory events"); diff --git a/northstar/src/runtime/config.rs b/northstar/src/runtime/config.rs index 4cf2abc38..facf1c4fa 100644 --- a/northstar/src/runtime/config.rs +++ b/northstar/src/runtime/config.rs @@ -1,7 +1,13 @@ use super::{Error, RepositoryId}; -use crate::{common::non_null_string::NonNullString, util::is_rw}; +use crate::common::non_null_string::NonNullString; +use nix::{sys::stat, unistd}; use serde::Deserialize; -use std::{collections::HashMap, path::PathBuf}; +use std::{ + collections::HashMap, + os::unix::prelude::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, +}; +use tokio::fs; use url::Url; /// Runtime configuration @@ -141,3 +147,24 @@ impl Config { Ok(()) } } + +/// Return true if path is read and writeable +async fn is_rw(path: &Path) -> bool { + match fs::metadata(path).await { + Ok(stat) => { + let same_uid = stat.uid() == unistd::getuid().as_raw(); + let same_gid = stat.gid() == unistd::getgid().as_raw(); + let mode = stat::Mode::from_bits_truncate(stat.permissions().mode()); + + let is_readable = (same_uid && mode.contains(stat::Mode::S_IRUSR)) + || (same_gid && mode.contains(stat::Mode::S_IRGRP)) + || mode.contains(stat::Mode::S_IROTH); + let is_writable = (same_uid && mode.contains(stat::Mode::S_IWUSR)) + || (same_gid && mode.contains(stat::Mode::S_IWGRP)) + || mode.contains(stat::Mode::S_IWOTH); + + is_readable && is_writable + } + Err(_) => false, + } +} diff --git a/northstar/src/runtime/console.rs b/northstar/src/runtime/console.rs index 416ebd51a..2ca30db4c 100644 --- a/northstar/src/runtime/console.rs +++ b/northstar/src/runtime/console.rs @@ -1,6 +1,6 @@ use super::{ContainerEvent, Event, NotificationTx, RepositoryId}; use crate::{ - api, + api::{self, codec::Framed}, common::container::Container, runtime::{EventTx, ExitStatus}, }; @@ -18,7 +18,7 @@ use std::{fmt, path::PathBuf, unreachable}; use thiserror::Error; use tokio::{ fs, - io::{self, AsyncRead, AsyncReadExt, AsyncWrite, BufReader}, + io::{self, AsyncRead, AsyncReadExt, AsyncWrite}, net::{TcpListener, UnixListener}, pin, select, sync::{broadcast, mpsc, oneshot}, @@ -28,6 +28,8 @@ use tokio::{ use tokio_util::{either::Either, io::ReaderStream, sync::CancellationToken}; use url::Url; +const BUFFER_SIZE: usize = 1024 * 1024; + // Request from the main loop to the console #[derive(Debug)] pub(crate) enum Request { @@ -114,7 +116,7 @@ impl Console { debug!("Client {} connected", peer); // Get a framed stream and sink interface. - let mut network_stream = api::codec::framed(stream); + let mut network_stream = api::codec::Framed::with_capacity(stream, BUFFER_SIZE); // Wait for a connect message within timeout let connect = network_stream.next(); @@ -253,7 +255,7 @@ impl Console { /// async fn process_request( client_id: &Peer, - stream: &mut S, + stream: &mut Framed, stop: &CancellationToken, event_loop: &EventTx, message: model::Message, @@ -263,7 +265,10 @@ where { let (reply_tx, reply_rx) = oneshot::channel(); if let model::Message::Request { - request: model::Request::Install { repository, size }, + request: model::Request::Install { + repository, + mut size, + }, } = message { debug!( @@ -281,8 +286,21 @@ where let event = Event::Console(request, reply_tx); event_loop.send(event).map_err(|_| Error::Shutdown).await?; + // The codec might have pulled bytes in the the read buffer of the connection. + if !stream.read_buffer().is_empty() { + let read_buffer = stream.read_buffer_mut().split(); + + // TODO: handle this case. The connected entity pushed the install file + // and a subsequenc request. If the codec pullen in the *full* install blob + // and some bytes from the following command the logic is screwed up. + assert!(read_buffer.len() as u64 <= size); + + size -= read_buffer.len() as u64; + tx.send(read_buffer.freeze()).await.ok(); + } + // If the connections breaks: just break. If the receiver is dropped: just break. - let mut take = ReaderStream::new(BufReader::new(stream.take(size))); + let mut take = ReaderStream::with_capacity(stream.get_mut().take(size), 1024 * 1024); while let Some(Ok(buf)) = take.next().await { if tx.send(buf).await.is_err() { break; @@ -409,6 +427,12 @@ impl From<&str> for Peer { } } +impl From for Peer { + fn from(s: String) -> Self { + Peer(s) + } +} + impl From for Peer { fn from(socket: std::net::SocketAddr) -> Self { Peer(socket.to_string()) diff --git a/northstar/src/runtime/fork/forker/impl.rs b/northstar/src/runtime/fork/forker/impl.rs new file mode 100644 index 000000000..3a9b0e41e --- /dev/null +++ b/northstar/src/runtime/fork/forker/impl.rs @@ -0,0 +1,291 @@ +use super::{ + init, + init::Init, + messages::{Message, Notification}, + util::fork, +}; +use crate::{ + common::container::Container, + debug, + runtime::{ + fork::util::{self, set_log_target}, + ipc::{self, owned_fd::OwnedFd, socket_pair, AsyncMessage, Message as IpcMessage}, + ExitStatus, Pid, + }, +}; +use futures::{ + stream::{FuturesUnordered, StreamExt}, + Future, +}; +use itertools::Itertools; +use nix::{ + errno::Errno, + sys::{signal::Signal, wait::waitpid}, + unistd, +}; +use std::{ + collections::HashMap, + os::unix::{ + io::FromRawFd, + net::UnixStream as StdUnixStream, + prelude::{IntoRawFd, RawFd}, + }, + path::PathBuf, +}; +use tokio::{net::UnixStream, select, sync::mpsc, task}; + +type Inits = HashMap; + +/// Handle the communication between the forker and the init process. +struct InitProcess { + pid: Pid, + /// Used to send messages to the init process. + stream: AsyncMessage, +} + +/// Entry point of the forker process +pub async fn run(stream: StdUnixStream, notifications: StdUnixStream) -> ! { + let mut notifications: AsyncMessage = notifications + .try_into() + .expect("Failed to create async message"); + let mut stream: AsyncMessage = + stream.try_into().expect("Failed to create async message"); + let mut inits = Inits::new(); + let mut exits = FuturesUnordered::new(); + + debug!("Entering main loop"); + + let (tx, mut rx) = mpsc::channel(1); + + // Separate tasks for notifications and messages + + task::spawn(async move { + loop { + select! { + exit = rx.recv() => { + match exit { + Some(exit) => exits.push(exit), + None => break, + } + } + exit = exits.next(), if !exits.is_empty() => { + let (container, exit_status) = exit.expect("Invalid exit status"); + debug!("Forwarding exit status notification of {}: {}", container, exit_status); + notifications.send(Notification::Exit { container, exit_status }).await.expect("Failed to send exit notification"); + } + } + } + }); + + loop { + select! { + request = recv(&mut stream) => { + match request { + Some(Message::CreateRequest { init, console }) => { + let container = init.container.clone(); + + if inits.contains_key(&container) { + let error = format!("Container {} already created", container); + log::warn!("{}", error); + stream.send(Message::Failure(error)).await.expect("Failed to send response"); + continue; + } + + debug!("Creating init process for {}", init.container); + let (pid, init_process) = create(init, console).await; + inits.insert(container, init_process); + stream.send(Message::CreateResult { init: pid }).await.expect("Failed to send response"); + } + Some(Message::ExecRequest { container, path, args, env, io }) => { + let io = io.unwrap(); + if let Some(init) = inits.remove(&container) { + let (response, exit) = exec(init, container, path, args, env, io).await; + tx.send(exit).await.ok(); + stream.send(response).await.expect("Failed to send response"); + } else { + let error = format!("Container {} not created", container); + log::warn!("{}", error); + stream.send(Message::Failure(error)).await.expect("Failed to send response"); + } + } + Some(_) => unreachable!("Unexpected message"), + None => { + debug!("Forker request channel closed. Exiting "); + std::process::exit(0); + } + } + } + } + } +} + +/// Create a new init process ("container") +async fn create(init: Init, console: Option) -> (Pid, InitProcess) { + let container = init.container.clone(); + debug!("Creating container {}", container); + let mut stream = socket_pair().expect("Failed to create socket pair"); + + let trampoline_pid = fork(|| { + set_log_target("northstar::forker-trampoline".into()); + util::set_parent_death_signal(Signal::SIGKILL); + + // Create pid namespace + debug!("Creating pid namespace"); + nix::sched::unshare(nix::sched::CloneFlags::CLONE_NEWPID) + .expect("Failed to create pid namespace"); + + // Work around the borrow checker and fork + let stream = stream.second().into_raw_fd(); + + // Fork the init process + debug!("Forking init of {}", container); + let init_pid = fork(|| { + let stream = unsafe { StdUnixStream::from_raw_fd(stream) }; + // Dive into init and never return + let stream = IpcMessage::from(stream); + init.run(stream, console); + }) + .expect("Failed to fork init"); + + let stream = unsafe { StdUnixStream::from_raw_fd(stream) }; + + // Send the pid of init to the forker process + let mut stream = ipc::Message::from(stream); + stream.send(init_pid).expect("Failed to send init pid"); + + debug!("Exiting trampoline"); + Ok(()) + }) + .expect("Failed to fork trampoline process"); + + let mut stream: AsyncMessage = stream + .first_async() + .map(Into::into) + .expect("Failed to turn socket into async UnixStream"); + + debug!("Waiting for init pid of container {}", container); + let pid = stream + .recv() + .await + .expect("Failed to receive init pid") + .unwrap(); + + // Reap the trampoline process + debug!("Waiting for trampoline process {} to exit", trampoline_pid); + let trampoline_pid = unistd::Pid::from_raw(trampoline_pid as i32); + match waitpid(Some(trampoline_pid), None) { + Ok(_) | Err(Errno::ECHILD) => (), // Ok - or reaped by the reaper thread + Err(e) => panic!("Failed to wait for the trampoline process: {}", e), + } + + debug!("Created container {} with pid {}", container, pid); + + (pid, InitProcess { pid, stream }) +} + +/// Send a exec request to a container +async fn exec( + mut init: InitProcess, + container: Container, + path: PathBuf, + args: Vec, + env: Vec, + io: [OwnedFd; 3], +) -> (Message, impl Future) { + debug_assert!(io.len() == 3); + + debug!( + "Forwarding exec request for container {}: {}", + container, + args.iter().map(ToString::to_string).join(" ") + ); + + // Send the exec request to the init process + let message = init::Message::new_exec(path, args, env); + init.stream + .send(message) + .await + .expect("Failed to send exec to init"); + + // Send io file descriptors + init.stream.send_fds(&io).await.expect("Failed to send fd"); + drop(io); + + match init.stream.recv().await.expect("Failed to receive") { + Some(init::Message::Forked { .. }) => (), + _ => panic!("Unexpected init message"), + } + + // Construct a future that waits to the init to signal a exit of it's child + // Afterwards reap the init process which should have exitted already + let exit = async move { + match init.stream.recv().await { + Ok(Some(init::Message::Exit { + pid: _, + exit_status, + })) => { + // Reap init process + debug!("Reaping init process of {} ({})", container, init.pid); + waitpid(unistd::Pid::from_raw(init.pid as i32), None) + .expect("Failed to reap init process"); + (container, exit_status) + } + Ok(None) | Err(_) => { + // Reap init process + debug!("Reaping init process of {} ({})", container, init.pid); + waitpid(unistd::Pid::from_raw(init.pid as i32), None) + .expect("Failed to reap init process"); + (container, ExitStatus::Exit(-1)) + } + Ok(_) => panic!("Unexpected message from init"), + } + }; + + (Message::ExecResult, exit) +} + +async fn recv(stream: &mut AsyncMessage) -> Option { + let request = match stream.recv().await { + Ok(request) => request, + Err(e) => { + debug!("Forker request error: {}. Breaking", e); + std::process::exit(0); + } + }; + match request { + Some(Message::CreateRequest { init, console: _ }) => { + let console = if init.console { + debug!("Console is enabled. Waiting for console stream"); + let console = stream + .recv_fds::() + .await + .expect("Failed to receive console fd"); + let console = unsafe { OwnedFd::from_raw_fd(console[0]) }; + Some(console) + } else { + None + }; + Some(Message::CreateRequest { init, console }) + } + Some(Message::ExecRequest { + container, + path, + args, + env, + .. + }) => { + let io = stream + .recv_fds::() + .await + .expect("Failed to receive io"); + Some(Message::ExecRequest { + container, + path, + args, + env, + io: Some(io), + }) + } + m => m, + } +} diff --git a/northstar/src/runtime/fork/forker/messages.rs b/northstar/src/runtime/fork/forker/messages.rs new file mode 100644 index 000000000..8bbf415a4 --- /dev/null +++ b/northstar/src/runtime/fork/forker/messages.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; + +use super::init::Init; +use crate::{ + common::container::Container, + runtime::{ipc::owned_fd::OwnedFd, ExitStatus, Pid}, +}; +use derive_new::new; +use serde::{Deserialize, Serialize}; + +/// Request from the runtime to the forker +#[non_exhaustive] +#[derive(new, Debug, Serialize, Deserialize)] +pub enum Message { + CreateRequest { + init: Init, + #[serde(skip)] + console: Option, + }, + CreateResult { + init: Pid, + }, + ExecRequest { + container: Container, + path: PathBuf, + args: Vec, + env: Vec, + #[serde(skip)] + io: Option<[OwnedFd; 3]>, + }, + ExecResult, + Failure(String), +} + +/// Notification from the forker to the runtime +#[derive(Debug, Serialize, Deserialize)] +pub enum Notification { + Exit { + container: Container, + exit_status: ExitStatus, + }, +} diff --git a/northstar/src/runtime/fork/forker/mod.rs b/northstar/src/runtime/fork/forker/mod.rs new file mode 100644 index 000000000..6be3f2cd6 --- /dev/null +++ b/northstar/src/runtime/fork/forker/mod.rs @@ -0,0 +1,164 @@ +use super::{ + super::{error::Error, Pid}, + init, + util::{self}, +}; +use crate::{ + common::container::Container, + debug, + npk::manifest::Manifest, + runtime::{ + config::Config, + error::Context, + fork::util::set_log_target, + ipc::{owned_fd::OwnedFd, socket_pair, AsyncMessage}, + }, +}; +use futures::FutureExt; +pub use messages::{Message, Notification}; +use nix::sys::signal::{signal, SigHandler, Signal}; +use std::{os::unix::net::UnixStream as StdUnixStream, path::PathBuf}; +use tokio::{net::UnixStream, runtime}; + +mod r#impl; +mod messages; + +pub struct ForkerChannels { + pub stream: StdUnixStream, + pub notifications: StdUnixStream, +} + +/// Fork the forker process +pub fn start() -> Result<(Pid, ForkerChannels), Error> { + let mut stream_pair = socket_pair().expect("Failed to open socket pair"); + let mut notifications = socket_pair().expect("Failed to open socket pair"); + + let pid = util::fork(|| { + set_log_target("northstar::forker".into()); + util::set_child_subreaper(true); + util::set_parent_death_signal(Signal::SIGKILL); + util::set_process_name("northstar-fork"); + + let stream = stream_pair.second(); + let notifications = notifications.second(); + + debug!("Setting signal handlers for SIGINT and SIGHUP"); + unsafe { + signal(Signal::SIGINT, SigHandler::SigIgn).context("Setting SIGINT handler failed")?; + signal(Signal::SIGHUP, SigHandler::SigIgn).context("Setting SIGHUP handler failed")?; + } + + debug!("Starting async runtime"); + runtime::Builder::new_current_thread() + .thread_name("northstar-fork-runtime") + .enable_time() + .enable_io() + .build() + .expect("Failed to start runtime") + .block_on(async { + r#impl::run(stream, notifications).await; + }); + Ok(()) + }) + .expect("Failed to start Forker process"); + + let forker = ForkerChannels { + stream: stream_pair.first(), + notifications: notifications.first(), + }; + + Ok((pid, forker)) +} + +/// Handle to the forker process +#[derive(Debug)] +pub struct Forker { + /// Framed stream/sink for sending messages to the forker process + stream: AsyncMessage, +} + +impl Forker { + /// Create a new forker handle + pub fn new(stream: StdUnixStream) -> Self { + let stream = stream.try_into().expect("Failed to create AsyncMessage"); + Self { stream } + } + + /// Send a request to the forker process to create a new container + pub async fn create( + &mut self, + config: &Config, + manifest: &Manifest, + console: Option, + ) -> Result { + debug_assert_eq!(manifest.console, console.is_some()); + + let init = init::build(config, manifest).await?; + let console = console.map(Into::into); + let message = Message::new_create_request(init, console); + + match self + .request_response(message) + .await + .expect("Failed to send request") + { + Message::CreateResult { init } => Ok(init), + Message::Failure(error) => { + Err(Error::StartContainerFailed(manifest.container(), error)) + } + _ => panic!("Unexpected forker response"), + } + } + + /// Start container process in a previously created container + pub async fn exec( + &mut self, + container: Container, + path: PathBuf, + args: Vec, + env: Vec, + io: [OwnedFd; 3], + ) -> Result<(), Error> { + let message = Message::new_exec_request(container, path, args, env, Some(io)); + self.request_response(message).await.map(drop) + } + + /// Send a request to the forker process + async fn request_response(&mut self, request: Message) -> Result { + let mut request = request; + + // Remove fds from message + let fds = match &mut request { + Message::CreateRequest { init: _, console } => { + console.take().map(|console| Vec::from([console])) + } + Message::ExecRequest { io, .. } => io.take().map(Vec::from), + _ => None, + }; + + // Send it + self.stream + .send(request) + .await + .context("Failed to send request")?; + + // Send fds if any + if let Some(fds) = fds { + self.stream + .send_fds(&fds) + .await + .context("Failed to send fd")?; + drop(fds); + } + + // Receive reply + let reply = self + .stream + .recv() + .map(|s| s.map(|s| s.unwrap())) + .await + .context("Failed to receive response from forker")?; + + Ok(reply) + } +} diff --git a/northstar/src/runtime/process/fs.rs b/northstar/src/runtime/fork/init/builder.rs similarity index 67% rename from northstar/src/runtime/process/fs.rs rename to northstar/src/runtime/fork/init/builder.rs index e82c9e17f..702dc686b 100644 --- a/northstar/src/runtime/process/fs.rs +++ b/northstar/src/runtime/fork/init/builder.rs @@ -1,74 +1,95 @@ -use super::Error; +use super::{Init, Mount}; use crate::{ common::container::Container, npk::{ manifest, manifest::{Manifest, MountOption, MountOptions, Resource, Tmpfs}, }, - runtime::{config::Config, error::Context}, - util::PathExt, + runtime::{ + config::Config, + error::{Context, Error}, + }, + seccomp, }; -use log::debug; use nix::{mount::MsFlags, unistd}; -use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; +use std::{ + ffi::{c_void, CString}, + path::{Path, PathBuf}, + ptr::null, +}; use tokio::fs; -/// Instructions for mount system call done in init -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct Mount { - pub source: Option, - pub target: PathBuf, - pub fstype: Option, - pub flags: u64, - pub data: Option, - pub error_msg: String, +trait PathExt { + fn join_strip>(&self, w: T) -> PathBuf; } -impl Mount { - pub fn new( - source: Option, - target: PathBuf, - fstype: Option<&'static str>, - flags: MsFlags, - data: Option, - ) -> Mount { - let error_msg = format!( - "Failed to mount '{}' of type '{}' on '{}' with flags '{:?}' and data '{}'", - source.clone().unwrap_or_default().display(), - fstype.unwrap_or_default(), - target.display(), - flags, - data.clone().unwrap_or_default() - ); - Mount { - source, - target, - fstype: fstype.map(|s| s.to_string()), - flags: flags.bits(), - data, - error_msg, +pub async fn build(config: &Config, manifest: &Manifest) -> Result { + let container = manifest.container(); + let root = config.run_dir.join(container.to_string()); + + let capabilities = manifest.capabilities.clone(); + let console = manifest.console; + let gid = manifest.gid; + let groups = groups(manifest); + let mounts = prepare_mounts(config, &root, manifest).await?; + let rlimits = manifest.rlimits.clone(); + let seccomp = seccomp_filter(manifest); + let uid = manifest.uid; + + Ok(Init { + container, + root, + uid, + gid, + mounts, + groups, + capabilities, + rlimits, + seccomp, + console, + }) +} + +/// Generate a list of supplementary gids if the groups info can be retrieved. This +/// must happen before the init `clone` because the group information cannot be gathered +/// without `/etc` etc... +fn groups(manifest: &Manifest) -> Vec { + if let Some(groups) = manifest.suppl_groups.as_ref() { + let mut result = Vec::with_capacity(groups.len()); + for group in groups { + let cgroup = CString::new(group.as_str()).unwrap(); // Check during manifest parsing + let group_info = + unsafe { nix::libc::getgrnam(cgroup.as_ptr() as *const nix::libc::c_char) }; + if group_info == (null::() as *mut nix::libc::group) { + log::warn!("Skipping invalid supplementary group {}", group); + } else { + let gid = unsafe { (*group_info).gr_gid }; + // TODO: Are there gids cannot use? + result.push(gid) + } } + result + } else { + Vec::with_capacity(0) } +} - /// Execute this mount call - pub(super) fn mount(&self) { - nix::mount::mount( - self.source.as_ref(), - &self.target, - self.fstype.as_deref(), - // Safe because flags is private and only set in Mount::new via MsFlags::bits - unsafe { MsFlags::from_bits_unchecked(self.flags) }, - self.data.as_deref(), - ) - .expect(&self.error_msg); +/// Generate seccomp filter applied in init +fn seccomp_filter(manifest: &Manifest) -> Option { + if let Some(seccomp) = manifest.seccomp.as_ref() { + return Some(seccomp::seccomp_filter( + seccomp.profile.as_ref(), + seccomp.allow.as_ref(), + manifest.capabilities.as_ref(), + )); } + None } /// Iterate the mounts of a container and assemble a list of `mount` calls to be /// performed by init. Prepare an options persist dir. This fn fails if a resource /// is referenced that does not exist. -pub(super) async fn prepare_mounts( +async fn prepare_mounts( config: &Config, root: &Path, manifest: &Manifest, @@ -103,7 +124,7 @@ pub(super) async fn prepare_mounts( } fn proc(root: &Path, target: &Path) -> Mount { - debug!("Mounting proc on {}", target.display()); + log::debug!("Mounting proc on {}", target.display()); let source = PathBuf::from("proc"); let target = root.join_strip(target); let fstype = "proc"; @@ -115,7 +136,7 @@ fn bind(root: &Path, target: &Path, host: &Path, options: &MountOptions) -> Vec< if host.exists() { let rw = options.contains(&MountOption::Rw); let mut mounts = Vec::with_capacity(if rw { 2 } else { 1 }); - debug!( + log::debug!( "Mounting {} on {} with flags {}", host.display(), target.display(), @@ -140,7 +161,7 @@ fn bind(root: &Path, target: &Path, host: &Path, options: &MountOptions) -> Vec< } mounts } else { - debug!( + log::debug!( "Skipping bind mount of nonexistent source {} to {}", host.display(), target.display() @@ -157,13 +178,13 @@ async fn persist( gid: u16, ) -> Result { if !source.exists() { - debug!("Creating {}", source.display()); + log::debug!("Creating {}", source.display()); fs::create_dir_all(&source) .await .context(format!("Failed to create {}", source.display()))?; } - debug!("Chowning {} to {}:{}", source.display(), uid, gid); + log::debug!("Chowning {} to {}:{}", source.display(), uid, gid); unistd::chown( source.as_os_str(), Some(unistd::Uid::from_raw(uid.into())), @@ -176,7 +197,7 @@ async fn persist( gid ))?; - debug!("Mounting {} on {}", source.display(), target.display(),); + log::debug!("Mounting {} on {}", source.display(), target.display(),); let target = root.join_strip(target); let flags = MsFlags::MS_BIND | MsFlags::MS_NODEV | MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC; @@ -217,7 +238,7 @@ fn resource( dir }; - debug!( + log::debug!( "Mounting {} on {} with {}", src.display(), target.display(), @@ -236,7 +257,7 @@ fn resource( } fn tmpfs(root: &Path, target: &Path, size: u64) -> Mount { - debug!( + log::debug!( "Mounting tmpfs with size {} on {}", bytesize::ByteSize::b(size), target.display() @@ -261,3 +282,12 @@ fn options_to_flags(opt: &MountOptions) -> MsFlags { } flags } + +impl PathExt for Path { + fn join_strip>(&self, w: T) -> PathBuf { + self.join(match w.as_ref().strip_prefix("/") { + Ok(stripped) => stripped, + Err(_) => w.as_ref(), + }) + } +} diff --git a/northstar/src/runtime/process/init.rs b/northstar/src/runtime/fork/init/mod.rs similarity index 52% rename from northstar/src/runtime/process/init.rs rename to northstar/src/runtime/fork/init/mod.rs index c3a122765..c826c5775 100644 --- a/northstar/src/runtime/process/init.rs +++ b/northstar/src/runtime/fork/init/mod.rs @@ -1,68 +1,101 @@ -use super::{fs::Mount, io::Fd}; use crate::{ + common::container::Container, + debug, info, npk::manifest::{Capability, RLimitResource, RLimitValue}, runtime::{ - ipc::{channel::Channel, condition::ConditionNotify}, - ExitStatus, + fork::util::{self, fork, set_child_subreaper, set_log_target, set_process_name}, + ipc::{owned_fd::OwnedFd, Message as IpcMessage}, + ExitStatus, Pid, }, seccomp::AllowList, }; +pub use builder::build; +use derive_new::new; +use itertools::Itertools; use nix::{ errno::Errno, libc::{self, c_ulong}, + mount::MsFlags, sched::unshare, - sys::wait::{waitpid, WaitStatus}, - unistd::{self, Uid}, + sys::{ + signal::Signal, + wait::{waitpid, WaitStatus}, + }, + unistd, + unistd::Uid, }; use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, env, ffi::CString, - os::unix::prelude::RawFd, + os::unix::{ + net::UnixStream, + prelude::{AsRawFd, RawFd}, + }, path::PathBuf, process::exit, }; +mod builder; + +// Message from the forker to init and response +#[derive(new, Debug, Serialize, Deserialize)] +pub enum Message { + /// The init process forked a new child with `pid` + Forked { pid: Pid }, + /// A child of init exited with `exit_status` + Exit { pid: Pid, exit_status: ExitStatus }, + /// Exec a new process + Exec { + path: PathBuf, + args: Vec, + env: Vec, + }, +} + #[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct Init { +pub struct Init { + pub container: Container, pub root: PathBuf, - pub init: CString, - pub argv: Vec, - pub env: Vec, pub uid: u16, pub gid: u16, pub mounts: Vec, - pub fds: Vec<(RawFd, Fd)>, pub groups: Vec, pub capabilities: Option>, pub rlimits: Option>, pub seccomp: Option, + pub console: bool, } impl Init { - pub(super) fn run( - self, - condition_notify: ConditionNotify, - mut exit_status_channel: Channel, - ) -> ! { + pub fn run(self, mut stream: IpcMessage, console: Option) -> ! { + set_log_target(format!("northstar::init::{}", self.container)); + + // Become a subreaper + set_child_subreaper(true); + // Set the process name to init. This process inherited the process name // from the runtime - set_process_name(); + set_process_name(&format!("init-{}", self.container)); // Become a session group leader + debug!("Setting session id"); unistd::setsid().expect("Failed to call setsid"); // Enter mount namespace + debug!("Entering mount namespace"); unshare(nix::sched::CloneFlags::CLONE_NEWNS).expect("Failed to unshare NEWNS"); // Perform all mounts passed in mounts - mount(&self.mounts); + self.mount(); // Set the chroot to the containers root mount point + debug!("Chrooting to {}", self.root.display()); unistd::chroot(&self.root).expect("Failed to chroot"); // Set current working directory to root + debug!("Setting cwd to /"); env::set_current_dir("/").expect("Failed to set cwd to /"); // UID / GID @@ -75,65 +108,129 @@ impl Init { self.set_rlimits(); // No new privileges - set_no_new_privs(true); + Self::set_no_new_privs(true); // Capabilities self.drop_privileges(); - // Close and dup fds - self.file_descriptors(); + loop { + match stream.recv() { + Ok(Some(Message::Exec { + path, + args, + mut env, + })) => { + debug!("Execing {} {}", path.display(), args.iter().join(" ")); + + // The init process got adopted by the forker after the trampoline exited. It is + // safe to set the parent death signal now. + util::set_parent_death_signal(Signal::SIGKILL); + + if let Some(fd) = console.as_ref().map(AsRawFd::as_raw_fd) { + // Add the fd number to the environment of the application + env.push(format!("NORTHSTAR_CONSOLE={}", fd)); + } + + let io = stream.recv_fds::().expect("Failed to receive io"); + debug_assert!(io.len() == 3); + let stdin = io[0]; + let stdout = io[1]; + let stderr = io[2]; + + // Start new process inside the container + let pid = fork(|| { + set_log_target(format!("northstar::{}", self.container)); + util::set_parent_death_signal(Signal::SIGKILL); + + unistd::dup2(stdin, nix::libc::STDIN_FILENO).expect("Failed to dup2"); + unistd::dup2(stdout, nix::libc::STDOUT_FILENO).expect("Failed to dup2"); + unistd::dup2(stderr, nix::libc::STDERR_FILENO).expect("Failed to dup2"); - // Clone - match unsafe { unistd::fork() } { - Ok(result) => match result { - unistd::ForkResult::Parent { child } => { - // Drop checkpoint. The fds are cloned into the child and are closed upon execve. - drop(condition_notify); + unistd::close(stdin).expect("Failed to close stdout after dup2"); + unistd::close(stdout).expect("Failed to close stdout after dup2"); + unistd::close(stderr).expect("Failed to close stderr after dup2"); + + // Set seccomp filter + if let Some(ref filter) = self.seccomp { + filter.apply().expect("Failed to apply seccomp filter."); + } + + let path = CString::new(path.to_str().unwrap()).unwrap(); + let args = args + .iter() + .map(|s| CString::new(s.as_str()).unwrap()) + .collect::>(); + let env = env + .iter() + .map(|s| CString::new(s.as_str()).unwrap()) + .collect::>(); + + panic!( + "Execve: {:?} {:?}: {:?}", + &path, + &args, + unistd::execve(&path, &args, &env) + ) + }) + .expect("Failed to spawn child process"); + + // close fds + drop(console); + unistd::close(stdin).expect("Failed to close stdout"); + unistd::close(stdout).expect("Failed to close stdout"); + unistd::close(stderr).expect("Failed to close stderr"); + + let message = Message::Forked { pid }; + stream.send(&message).expect("Failed to send fork result"); // Wait for the child to exit loop { - match waitpid(Some(child), None) { + debug!("Waiting for child process {} to exit", pid); + match waitpid(Some(unistd::Pid::from_raw(pid as i32)), None) { Ok(WaitStatus::Exited(_pid, status)) => { + debug!("Child process {} exited with status code {}", pid, status); let exit_status = ExitStatus::Exit(status); - exit_status_channel - .send(&exit_status) - .expect("Failed to send exit status"); + stream + .send(Message::Exit { pid, exit_status }) + .expect("Channel error"); + + assert_eq!( + waitpid(Some(unistd::Pid::from_raw(pid as i32)), None), + Err(nix::Error::ECHILD) + ); + exit(0); } Ok(WaitStatus::Signaled(_pid, status, _)) => { + debug!("Child process {} exited with signal {}", pid, status); let exit_status = ExitStatus::Signalled(status as u8); - exit_status_channel - .send(&exit_status) - .expect("Failed to send exit status"); + stream + .send(Message::Exit { pid, exit_status }) + .expect("Channel error"); + + assert_eq!( + waitpid(Some(unistd::Pid::from_raw(pid as i32)), None), + Err(nix::Error::ECHILD) + ); + exit(0); } Ok(WaitStatus::Continued(_)) | Ok(WaitStatus::Stopped(_, _)) => { - continue + log::error!("Child process continued or stopped"); + continue; } Err(nix::Error::EINTR) => continue, - e => panic!("Failed to waitpid on {}: {:?}", child, e), + e => panic!("Failed to waitpid on {}: {:?}", pid, e), } } } - unistd::ForkResult::Child => { - drop(exit_status_channel); - - // Set seccomp filter - if let Some(ref filter) = self.seccomp { - filter.apply().expect("Failed to apply seccomp filter."); - } - - // Checkpoint fds are FD_CLOEXEC and act as a signal for the launcher that this child is started. - // Therefore no explicit drop (close) of _checkpoint_notify is needed here. - panic!( - "Execve: {:?} {:?}: {:?}", - &self.init, - &self.argv, - unistd::execve(&self.init, &self.argv, &self.env) - ) + Ok(None) => { + info!("Channel closed. Exiting..."); + std::process::exit(0); } - }, - Err(e) => panic!("Clone error: {}", e), + Ok(_) => unimplemented!("Unimplemented message"), + Err(e) => panic!("Failed to receive message: {}", e), + } } } @@ -149,10 +246,12 @@ impl Init { caps::securebits::set_keepcaps(true).expect("Failed to set keep caps"); } + debug!("Setting resgid {}", gid); let gid = unistd::Gid::from_raw(gid.into()); unistd::setresgid(gid, gid, gid).expect("Failed to set resgid"); let uid = unistd::Uid::from_raw(uid.into()); + debug!("Setting resuid {}", uid); unistd::setresuid(uid, uid, uid).expect("Failed to set resuid"); if rt_privileged { @@ -162,6 +261,7 @@ impl Init { } fn set_groups(&self) { + debug!("Setting groups {:?}", self.groups); let result = unsafe { nix::libc::setgroups(self.groups.len(), self.groups.as_ptr()) }; Errno::result(result) @@ -171,6 +271,7 @@ impl Init { fn set_rlimits(&self) { if let Some(limits) = self.rlimits.as_ref() { + debug!("Applying rlimits"); for (resource, limit) in limits { let resource = match resource { RLimitResource::AS => rlimit::Resource::AS, @@ -203,6 +304,7 @@ impl Init { /// Drop capabilities fn drop_privileges(&self) { + debug!("Dropping priviledges"); let mut bounded = caps::read(None, caps::CapSet::Bounding).expect("Failed to read bounding caps"); // Convert the set from the manifest to a set of caps::Capability @@ -231,54 +333,26 @@ impl Init { caps::set(None, caps::CapSet::Effective, &all).expect("Failed to reset effective caps"); } - /// Apply file descriptor configuration - fn file_descriptors(&self) { - for (fd, value) in &self.fds { - match value { - Fd::Close => { - // Ignore close errors because the fd list contains the ReadDir fd and fds from other tasks. - unistd::close(*fd).ok(); - } - Fd::Dup(n) => { - unistd::dup2(*n, *fd).expect("Failed to dup2"); - unistd::close(*n).expect("Failed to close"); - } - } + /// Execute list of mount calls + fn mount(&self) { + for mount in &self.mounts { + debug!("Mounting {:?} on {}", mount.source, mount.target.display()); + mount.mount(); } } -} -/// Execute list of mount calls -fn mount(mounts: &[Mount]) { - for mount in mounts { - mount.mount(); - } -} - -fn set_no_new_privs(value: bool) { - #[cfg(target_os = "android")] - pub const PR_SET_NO_NEW_PRIVS: libc::c_int = 38; - #[cfg(not(target_os = "android"))] - use libc::PR_SET_NO_NEW_PRIVS; + fn set_no_new_privs(value: bool) { + #[cfg(target_os = "android")] + pub const PR_SET_NO_NEW_PRIVS: libc::c_int = 38; + #[cfg(not(target_os = "android"))] + use libc::PR_SET_NO_NEW_PRIVS; - let result = unsafe { nix::libc::prctl(PR_SET_NO_NEW_PRIVS, value as c_ulong, 0, 0, 0) }; - Errno::result(result) - .map(drop) - .expect("Failed to set PR_SET_NO_NEW_PRIVS") -} - -#[cfg(target_os = "android")] -pub const PR_SET_NAME: libc::c_int = 15; -#[cfg(not(target_os = "android"))] -use libc::PR_SET_NAME; - -/// Set the name of the current process to "init" -fn set_process_name() { - let cname = "init\0"; - let result = unsafe { libc::prctl(PR_SET_NAME, cname.as_ptr() as c_ulong, 0, 0, 0) }; - Errno::result(result) - .map(drop) - .expect("Failed to set PR_SET_NAME"); + debug!("Setting no new privs"); + let result = unsafe { nix::libc::prctl(PR_SET_NO_NEW_PRIVS, value as c_ulong, 0, 0, 0) }; + Errno::result(result) + .map(drop) + .expect("Failed to set PR_SET_NO_NEW_PRIVS") + } } impl From for caps::Capability { @@ -328,3 +402,54 @@ impl From for caps::Capability { } } } + +/// Instructions for mount system call done in init +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Mount { + pub source: Option, + pub target: PathBuf, + pub fstype: Option, + pub flags: u64, + pub data: Option, + pub error_msg: String, +} + +impl Mount { + pub fn new( + source: Option, + target: PathBuf, + fstype: Option<&'static str>, + flags: MsFlags, + data: Option, + ) -> Mount { + let error_msg = format!( + "Failed to mount '{}' of type '{}' on '{}' with flags '{:?}' and data '{}'", + source.clone().unwrap_or_default().display(), + fstype.unwrap_or_default(), + target.display(), + flags, + data.clone().unwrap_or_default() + ); + Mount { + source, + target, + fstype: fstype.map(|s| s.to_string()), + flags: flags.bits(), + data, + error_msg, + } + } + + /// Execute this mount call + pub(super) fn mount(&self) { + nix::mount::mount( + self.source.as_ref(), + &self.target, + self.fstype.as_deref(), + // Safe because flags is private and only set in Mount::new via MsFlags::bits + unsafe { MsFlags::from_bits_unchecked(self.flags) }, + self.data.as_deref(), + ) + .expect(&self.error_msg); + } +} diff --git a/northstar/src/runtime/fork/mod.rs b/northstar/src/runtime/fork/mod.rs new file mode 100644 index 000000000..4f48b2c2b --- /dev/null +++ b/northstar/src/runtime/fork/mod.rs @@ -0,0 +1,5 @@ +mod forker; +mod init; +mod util; + +pub use forker::{start, Forker, ForkerChannels, Notification}; diff --git a/northstar/src/runtime/fork/util.rs b/northstar/src/runtime/fork/util.rs new file mode 100644 index 000000000..83f4d4e85 --- /dev/null +++ b/northstar/src/runtime/fork/util.rs @@ -0,0 +1,143 @@ +use crate::{ + debug, error, + runtime::{error::Error, Pid}, +}; +use nix::{ + errno::Errno, + libc::{self, c_ulong}, + sys::signal::Signal, + unistd::{self}, +}; +use std::process::exit; + +/// Set the parent death signal of the calling process +pub fn set_parent_death_signal(signal: Signal) { + #[cfg(target_os = "android")] + const PR_SET_PDEATHSIG: libc::c_int = 1; + #[cfg(not(target_os = "android"))] + use libc::PR_SET_PDEATHSIG; + + debug!("Setting parent death signal to {}", signal); + + let result = unsafe { nix::libc::prctl(PR_SET_PDEATHSIG, signal, 0, 0, 0) }; + Errno::result(result) + .map(drop) + .expect("Failed to set PR_SET_PDEATHSIG"); +} + +/// Set the name of the current process +pub fn set_process_name(name: &str) { + #[cfg(target_os = "android")] + const PR_SET_NAME: libc::c_int = 15; + #[cfg(not(target_os = "android"))] + use libc::PR_SET_NAME; + + debug!("Setting process name to {}", name); + + // PR_SET_NAME (since Linux 2.6.9) + // Set the name of the calling thread, using the value in the + // location pointed to by (char *) arg2. The name can be up + // to 16 bytes long, including the terminating null byte. + // (If the length of the string, including the terminating + // null byte, exceeds 16 bytes, the string is silently + // truncated.) This is the same attribute that can be set + // via pthread_setname_np(3) and retrieved using + // pthread_getname_np(3). The attribute is likewise + // accessible via /proc/self/task/[tid]/comm (see proc(5)), + // where [tid] is the thread ID of the calling thread, as + // returned by gettid(2). + let mut name = name.as_bytes().to_vec(); + name.truncate(15); + name.push(b'\0'); + + let result = unsafe { libc::prctl(PR_SET_NAME, name.as_ptr() as c_ulong, 0, 0, 0) }; + Errno::result(result) + .map(drop) + .expect("Failed to set PR_SET_NAME"); +} + +// Set the child subreaper flag of the calling thread +pub fn set_child_subreaper(value: bool) { + #[cfg(target_os = "android")] + const PR_SET_CHILD_SUBREAPER: nix::libc::c_int = 36; + #[cfg(not(target_os = "android"))] + use nix::libc::PR_SET_CHILD_SUBREAPER; + + debug!("Setting child subreaper flag to {}", value); + + let value = if value { 1u64 } else { 0u64 }; + let result = unsafe { nix::libc::prctl(PR_SET_CHILD_SUBREAPER, value, 0, 0, 0) }; + Errno::result(result) + .map(drop) + .expect("Failed to set child subreaper flag") +} + +/// Fork a new process. +/// +/// # Arguments: +/// +/// * `f` - The closure to run in the child process. +/// +pub fn fork(f: F) -> nix::Result +where + F: FnOnce() -> Result<(), Error>, +{ + match unsafe { unistd::fork()? } { + unistd::ForkResult::Parent { child } => Ok(child.as_raw() as Pid), + unistd::ForkResult::Child => match f() { + Ok(_) => exit(0), + Err(e) => { + error!("Failed after fork: {:?}", e); + exit(-1); + } + }, + } +} + +pub(crate) static mut LOG_TARGET: String = String::new(); + +pub(crate) fn set_log_target(tag: String) { + unsafe { + LOG_TARGET = tag; + } +} + +/// Log to debug +#[allow(unused)] +#[macro_export] +macro_rules! debug { + ($($arg:tt)+) => ( + assert!(unsafe { !crate::runtime::fork::util::LOG_TARGET.is_empty() }); + log::debug!(target: unsafe { crate::runtime::fork::util::LOG_TARGET.as_str() }, $($arg)+) + ) +} + +/// Log to info +#[allow(unused)] +#[macro_export] +macro_rules! info { + ($($arg:tt)+) => ( + assert!(unsafe { !crate::runtime::fork::util::LOG_TARGET.is_empty() }); + log::info!(target: unsafe { crate::runtime::fork::util::LOG_TARGET.as_str() }, $($arg)+) + ) +} + +/// Log to warn +#[allow(unused)] +#[macro_export] +macro_rules! warn { + ($($arg:tt)+) => ( + assert!(unsafe { !crate::runtime::fork::util::LOG_TARGET.is_empty() }); + log::warn!(target: unsafe { crate::runtime::fork::util::LOG_TARGET.as_str() }, $($arg)+) + ) +} + +/// Log to error +#[allow(unused)] +#[macro_export] +macro_rules! error { + ($($arg:tt)+) => ( + assert!(unsafe { !crate::runtime::fork::util::LOG_TARGET.is_empty() }); + log::error!(target: unsafe { crate::runtime::fork::util::LOG_TARGET.as_str() }, $($arg)+) + ) +} diff --git a/northstar/src/runtime/io.rs b/northstar/src/runtime/io.rs new file mode 100644 index 000000000..9876716a2 --- /dev/null +++ b/northstar/src/runtime/io.rs @@ -0,0 +1,133 @@ +use std::{ + os::unix::prelude::{AsRawFd, FromRawFd, IntoRawFd}, + path::{Path, PathBuf}, +}; + +use crate::{ + common::container::Container, + npk::manifest::{self, Output}, +}; +use log::debug; +use nix::{ + fcntl::OFlag, + pty, + sys::{stat::Mode, termios::SetArg}, +}; +use tokio::{ + io::{self, AsyncBufReadExt, AsyncRead}, + task::{self, JoinHandle}, +}; + +use super::ipc::owned_fd::{OwnedFd, OwnedFdRw}; + +pub struct ContainerIo { + pub io: [OwnedFd; 3], + /// A handle to the io forwarding task if stdout or stderr is set to `Output::Pipe` + pub log_task: Option>>, +} + +/// Create a new pty handle if configured in the manifest or open /dev/null instead. +pub async fn open(container: &Container, io: &manifest::Io) -> io::Result { + // Open dev null - needed in any case for stdin + let dev_null = openrw("/dev/null")?; + + // Don't start the output task if it is configured to be discarded + if io.stdout == Output::Discard && io.stderr == Output::Discard { + return Ok(ContainerIo { + io: [dev_null.clone()?, dev_null.clone()?, dev_null], + log_task: None, + }); + } + + debug!("Spawning output logging task for {}", container); + let (write, read) = output_device(OutputDevice::Socket)?; + + let log_target = format!("northstar::{}", container); + let log_task = task::spawn(log_lines(log_target, read)); + + let (stdout, stderr) = match (&io.stdout, &io.stderr) { + (Output::Discard, Output::Pipe) => (dev_null.clone()?, write), + (Output::Pipe, Output::Discard) => (write, dev_null.clone()?), + (Output::Pipe, Output::Pipe) => (write.clone()?, write), + _ => unreachable!(), + }; + + let io = [dev_null, stdout, stderr]; + + Ok(ContainerIo { + io, + log_task: Some(log_task), + }) +} + +/// Type of output device +enum OutputDevice { + Socket, + #[allow(dead_code)] + Pty, +} + +/// Open a device used to collect the container output and forward it to Northstar's log +fn output_device( + dev: OutputDevice, +) -> io::Result<(OwnedFd, Box)> { + match dev { + OutputDevice::Socket => { + let (msock, csock) = std::os::unix::net::UnixStream::pair()?; + let msock = { + msock.set_nonblocking(true)?; + tokio::net::UnixStream::from_std(msock)? + }; + Ok((csock.into(), Box::new(msock))) + } + OutputDevice::Pty => { + let (main, sec_path) = openpty(); + Ok((openrw(sec_path)?, Box::new(OwnedFdRw::new(main)?))) + } + } +} + +/// Open a path for reading and writing. +fn openrw>(f: T) -> io::Result { + nix::fcntl::open(f.as_ref(), OFlag::O_RDWR, Mode::empty()) + .map_err(|err| io::Error::from_raw_os_error(err as i32)) + .map(|fd| unsafe { OwnedFd::from_raw_fd(fd) }) +} + +/// Create a new pty and return the main fd along with the sub name. +fn openpty() -> (OwnedFd, PathBuf) { + let main = pty::posix_openpt(OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_NONBLOCK) + .expect("Failed to open pty"); + + nix::sys::termios::tcgetattr(main.as_raw_fd()) + .map(|mut termios| { + nix::sys::termios::cfmakeraw(&mut termios); + termios + }) + .and_then(|termios| { + nix::sys::termios::tcsetattr(main.as_raw_fd(), SetArg::TCSANOW, &termios) + }) + .and_then(|_| pty::grantpt(&main)) + .and_then(|_| pty::unlockpt(&main)) + .expect("Failed to configure pty"); + + // Get the name of the sub + let sub = pty::ptsname_r(&main) + .map(PathBuf::from) + .expect("Failed to get PTY sub name"); + + debug!("Created PTY {}", sub.display()); + let main = unsafe { OwnedFd::from_raw_fd(main.into_raw_fd()) }; + + (main, sub) +} + +/// Pipe task: Read pty until stop is cancelled. Write linewist to `log`. +async fn log_lines(target: String, output: R) -> io::Result<()> { + let mut lines = io::BufReader::new(output).lines(); + while let Ok(Some(line)) = lines.next_line().await { + log::debug!(target: &target, "{}", line); + } + + Ok(()) +} diff --git a/northstar/src/runtime/ipc/channel.rs b/northstar/src/runtime/ipc/channel.rs deleted file mode 100644 index f4656e85a..000000000 --- a/northstar/src/runtime/ipc/channel.rs +++ /dev/null @@ -1,201 +0,0 @@ -use std::{ - io::{ErrorKind, Read, Write}, - os::unix::prelude::{AsRawFd, RawFd}, -}; - -use bincode::Options; -use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use serde::{de::DeserializeOwned, Serialize}; -use tokio::io::AsyncReadExt; - -use super::pipe::{pipe, AsyncPipeRead, PipeRead, PipeWrite}; - -/// Wrap a pipe to transfer bincoded structs that implement `Serialize` and `Deserialize` -pub struct Channel { - tx: PipeWrite, - rx: std::io::BufReader, -} - -impl Channel { - /// Create a new pipe channel - pub fn new() -> Channel { - let (rx, tx) = pipe().expect("Failed to create pipe"); - Channel { - tx, - rx: std::io::BufReader::new(rx), - } - } - - /// Raw fds for the rx and tx pipe - pub fn as_raw_fd(&self) -> (RawFd, RawFd) { - (self.tx.as_raw_fd(), self.rx.get_ref().as_raw_fd()) - } -} - -impl Channel { - /// Drops the tx part and returns a AsyncChannelRead - pub fn into_async_read(self) -> AsyncChannelRead { - let rx = self - .rx - .into_inner() - .try_into() - .expect("Failed to convert pipe read"); - AsyncChannelRead { - rx: tokio::io::BufReader::new(rx), - } - } - - /// Send v bincode serialized - pub fn send(&mut self, v: &T) -> std::io::Result<()> { - let buffer = bincode::DefaultOptions::new() - .with_fixint_encoding() - .serialize(v) - .map_err(|e| std::io::Error::new(ErrorKind::Other, e))?; - self.tx.write_u32::(buffer.len() as u32)?; - self.tx.write_all(&buffer) - } - - /// Receive bincode serialized - #[allow(unused)] - pub fn recv(&mut self) -> std::io::Result - where - T: DeserializeOwned, - { - let size = self.rx.read_u32::()?; - let mut buffer = vec![0; size as usize]; - self.rx.read_exact(&mut buffer)?; - bincode::DefaultOptions::new() - .with_fixint_encoding() - .deserialize(&buffer) - .map_err(|e| std::io::Error::new(ErrorKind::Other, e)) - } -} - -/// Async version of Channel -pub struct AsyncChannelRead { - rx: tokio::io::BufReader, -} - -impl AsyncChannelRead { - pub async fn recv<'de, T>(&mut self) -> std::io::Result - where - T: DeserializeOwned, - { - let size = self.rx.read_u32().await?; - let mut buffer = vec![0; size as usize]; - self.rx.read_exact(&mut buffer).await?; - bincode::DefaultOptions::new() - .with_fixint_encoding() - .deserialize(&buffer) - .map_err(|e| std::io::Error::new(ErrorKind::Other, e)) - } -} - -#[cfg(test)] -mod tests { - use nix::{sys::wait, unistd}; - - use super::*; - use crate::runtime::ExitStatus; - - #[tokio::test(flavor = "current_thread")] - async fn channel_async() { - let mut channel = Channel::new(); - - for i in 0..=255 { - channel.send(&std::time::Duration::from_secs(1)).unwrap(); - channel.send(&ExitStatus::Exit(i)).unwrap(); - channel.send(&ExitStatus::Signalled(10)).unwrap(); - } - - for i in 0..=255 { - assert_eq!( - channel.recv::().unwrap(), - std::time::Duration::from_secs(1) - ); - assert_eq!(channel.recv::().unwrap(), ExitStatus::Exit(i)); - assert_eq!( - channel.recv::().unwrap(), - ExitStatus::Signalled(10) - ); - } - - for i in 0..=255 { - channel.send(&std::time::Duration::from_secs(1)).unwrap(); - channel.send(&ExitStatus::Exit(i)).unwrap(); - channel.send(&ExitStatus::Signalled(10)).unwrap(); - } - - let mut channel = channel.into_async_read(); - for i in 0..=255 { - assert_eq!( - channel.recv::().await.unwrap(), - std::time::Duration::from_secs(1) - ); - assert_eq!( - channel.recv::().await.unwrap(), - ExitStatus::Exit(i) - ); - assert_eq!( - channel.recv::().await.unwrap(), - ExitStatus::Signalled(10) - ); - } - } - - #[test] - fn channel_fork() { - let mut channel = Channel::new(); - - match unsafe { unistd::fork().expect("Failed to fork") } { - unistd::ForkResult::Parent { child } => { - wait::waitpid(Some(child), None).expect("Failed to waitpid"); - for i in 0..=255i32 { - assert_eq!( - channel.recv::().unwrap(), - std::time::Duration::from_secs(i as u64) - ); - assert_eq!(channel.recv::().unwrap(), ExitStatus::Exit(i)); - assert_eq!( - channel.recv::().unwrap(), - ExitStatus::Signalled(10) - ); - } - } - unistd::ForkResult::Child => { - for i in 0..=255i32 { - channel - .send(&std::time::Duration::from_secs(i as u64)) - .unwrap(); - channel.send(&ExitStatus::Exit(i)).unwrap(); - channel.send(&ExitStatus::Signalled(10)).unwrap(); - } - std::process::exit(0); - } - } - } - - #[tokio::test(flavor = "current_thread")] - async fn channel_fork_close() { - let channel = Channel::new(); - match unsafe { unistd::fork().expect("Failed to fork") } { - unistd::ForkResult::Parent { child } => { - let mut channel = channel.into_async_read(); - wait::waitpid(Some(child), None).expect("Failed to waitpid"); - assert!(channel.recv::().await.is_err()); - } - unistd::ForkResult::Child => { - drop(channel); - std::process::exit(0); - } - } - } - - #[tokio::test(flavor = "current_thread")] - async fn channel_close() { - let channel = Channel::new(); - // Converting into a AsyncChannelRead closes the sending part - let mut channel = channel.into_async_read(); - assert!(channel.recv::().await.is_err()); - } -} diff --git a/northstar/src/runtime/ipc/condition.rs b/northstar/src/runtime/ipc/condition.rs deleted file mode 100644 index 2bd481711..000000000 --- a/northstar/src/runtime/ipc/condition.rs +++ /dev/null @@ -1,154 +0,0 @@ -use std::{ - io::Result, - os::unix::prelude::{AsRawFd, IntoRawFd, RawFd}, -}; - -use tokio::io::AsyncReadExt; - -use super::{ - pipe::{pipe, AsyncPipeRead, PipeRead, PipeWrite}, - raw_fd_ext::RawFdExt, -}; - -#[allow(unused)] -#[derive(Debug)] -pub struct Condition { - read: PipeRead, - write: PipeWrite, -} - -#[allow(unused)] -impl Condition { - pub fn new() -> Result { - let (rfd, wfd) = pipe()?; - - Ok(Condition { - read: rfd, - write: wfd, - }) - } - - pub fn set_cloexec(&self) { - self.read.set_cloexec(true); - self.write.set_cloexec(true); - } - - pub fn wait(mut self) { - drop(self.write); - let buf: &mut [u8] = &mut [0u8; 1]; - use std::io::Read; - loop { - match self.read.read(buf) { - Ok(n) if n == 0 => break, - Ok(_) => continue, - Err(e) => break, - } - } - } - - pub fn notify(self) {} - - pub fn split(self) -> (ConditionWait, ConditionNotify) { - ( - ConditionWait { read: self.read }, - ConditionNotify { write: self.write }, - ) - } -} - -#[derive(Debug)] -pub struct ConditionWait { - read: PipeRead, -} - -impl ConditionWait { - #[allow(unused)] - pub fn wait(mut self) { - use std::io::Read; - loop { - match self.read.read(&mut [0u8; 1]) { - Ok(n) if n == 0 => break, - Ok(_) => continue, - Err(_) => break, - } - } - } - - pub async fn async_wait(self) { - let mut read: AsyncPipeRead = self.read.try_into().unwrap(); - loop { - match read.read(&mut [0u8; 1]).await { - Ok(n) if n == 0 => break, - Ok(_) => continue, - Err(_) => break, - } - } - } -} - -impl AsRawFd for ConditionWait { - fn as_raw_fd(&self) -> RawFd { - self.read.as_raw_fd() - } -} - -impl IntoRawFd for ConditionWait { - fn into_raw_fd(self) -> RawFd { - self.read.into_raw_fd() - } -} - -#[derive(Debug)] -pub struct ConditionNotify { - write: PipeWrite, -} - -impl ConditionNotify { - #[allow(unused)] - pub fn notify(self) { - drop(self.write) - } -} - -impl AsRawFd for ConditionNotify { - fn as_raw_fd(&self) -> RawFd { - self.write.as_raw_fd() - } -} - -impl IntoRawFd for ConditionNotify { - fn into_raw_fd(self) -> RawFd { - self.write.into_raw_fd() - } -} - -#[cfg(test)] -mod tests { - use nix::unistd; - - use super::*; - - #[test] - fn condition() { - let (w0, n0) = Condition::new().unwrap().split(); - let (w1, n1) = Condition::new().unwrap().split(); - - match unsafe { unistd::fork().unwrap() } { - unistd::ForkResult::Parent { .. } => { - drop(w0); - drop(n1); - - n0.notify(); - w1.wait(); - } - unistd::ForkResult::Child => { - drop(n0); - drop(w1); - - w0.wait(); - n1.notify(); - std::process::exit(0); - } - } - } -} diff --git a/northstar/src/runtime/ipc/message.rs b/northstar/src/runtime/ipc/message.rs new file mode 100644 index 000000000..c1ee5000e --- /dev/null +++ b/northstar/src/runtime/ipc/message.rs @@ -0,0 +1,423 @@ +use bincode::Options; +use byteorder::{BigEndian, WriteBytesExt}; +use bytes::{BufMut, BytesMut}; +use lazy_static::lazy_static; +use nix::{ + cmsg_space, + sys::{ + socket::{self, ControlMessageOwned}, + uio, + }, +}; +use serde::{de::DeserializeOwned, Serialize}; +use std::{ + io::{self, ErrorKind, Read, Write}, + mem::MaybeUninit, + os::unix::prelude::{AsRawFd, FromRawFd, RawFd}, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, Interest}, + net::UnixStream, +}; + +lazy_static! { + static ref OPTIONS: bincode::DefaultOptions = bincode::DefaultOptions::new(); +} + +/// Bincode encoded and length delimited message stream via Read/Write +pub struct Message { + inner: T, +} + +impl Message { + /// Send bincode encoded message with a length field + pub fn send(&mut self, v: M) -> io::Result<()> { + let size = OPTIONS + .serialized_size(&v) + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + self.inner.write_u32::(size as u32)?; + OPTIONS + .serialize_into(&mut self.inner, &v) + .map_err(|e| io::Error::new(ErrorKind::Other, e)) + } + + /// Receive a bincode encoded message with a length field + pub fn recv(&mut self) -> io::Result> { + let mut buffer = [0u8; 4]; + let mut read = 0; + while read < 4 { + match self.inner.read(&mut buffer[read..])? { + 0 => return Ok(None), + n => read += n, + } + } + let size = u32::from_be_bytes(buffer); + let mut buffer = vec![0; size as usize]; + self.inner.read_exact(&mut buffer)?; + OPTIONS + .deserialize(&buffer) + .map(Some) + .map_err(|e| io::Error::new(ErrorKind::Other, e)) + } +} + +impl Message { + /// Send a file descriptor over the socket + #[allow(unused)] + pub fn send_fds(&self, fds: &[T]) -> io::Result<()> { + let buf = &[0u8]; + let iov = &[uio::IoVec::from_slice(buf)]; + let fds = fds.iter().map(AsRawFd::as_raw_fd).collect::>(); + let cmsg = [socket::ControlMessage::ScmRights(&fds)]; + const FLAGS: socket::MsgFlags = socket::MsgFlags::empty(); + + socket::sendmsg(self.inner.as_raw_fd(), iov, &cmsg, FLAGS, None) + .map_err(os_err) + .map(drop) + } + + /// Receive a file descriptor via the socket + pub fn recv_fds(&self) -> io::Result<[T; N]> { + let mut buf = [0u8]; + let iov = &[uio::IoVec::from_mut_slice(&mut buf)]; + let mut cmsg_buffer = cmsg_space!([RawFd; N]); + const FLAGS: socket::MsgFlags = socket::MsgFlags::empty(); + + let message = socket::recvmsg(self.inner.as_raw_fd(), iov, Some(&mut cmsg_buffer), FLAGS) + .map_err(os_err)?; + + recv_control_msg::(message.cmsgs().next()) + } +} + +impl From for Message +where + T: Read + Write, +{ + fn from(inner: T) -> Self { + Message { inner } + } +} + +#[derive(Debug)] +pub struct AsyncMessage { + inner: T, + read_buffer: BytesMut, + write_buffer: BytesMut, +} + +impl AsyncMessage { + // Cancel safe send + pub async fn send(&mut self, v: M) -> io::Result<()> { + if self.write_buffer.is_empty() { + // Calculate the serialized message size + let size = OPTIONS + .serialized_size(&v) + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + self.write_buffer.reserve(4 + size as usize); + self.write_buffer.put_u32(size as u32); + + // Serialize the message + let buffer = OPTIONS + .serialize(&v) + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + self.write_buffer.extend_from_slice(&buffer); + } + + while !self.write_buffer.is_empty() { + let n = self.inner.write(&self.write_buffer).await?; + drop(self.write_buffer.split_to(n)); + } + Ok(()) + } + + // Cancel safe recv + pub async fn recv<'de, M: DeserializeOwned>(&mut self) -> io::Result> { + while self.read_buffer.len() < 4 { + let remaining = 4 - self.read_buffer.len(); + let mut buffer = BytesMut::with_capacity(remaining); + match self.inner().read_buf(&mut buffer).await? { + 0 => return Ok(None), + _ => self.read_buffer.extend_from_slice(&buffer), + } + } + + // Parse the message size + let msg_len = u32::from_be_bytes(self.read_buffer[..4].try_into().unwrap()) as usize; + + // Read until the read buffer has this length + let target_buffer_len = msg_len as usize + 4; + + while self.read_buffer.len() < target_buffer_len { + // Calculate how may bytes are missing to read the message + let remaining = target_buffer_len - self.read_buffer.len(); + let mut buffer = BytesMut::with_capacity(remaining); + match self.inner().read_buf(&mut buffer).await? { + 0 => return Ok(None), + _ => self.read_buffer.extend_from_slice(&buffer), + } + } + + let message = OPTIONS + .deserialize(&self.read_buffer[4..]) + .map(Some) + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + + self.read_buffer.clear(); + + Ok(message) + } + + pub fn inner(&mut self) -> &mut T { + &mut self.inner + } +} + +impl AsyncMessage { + /// Send a file descriptor via the stream. Ensure that fd is open until this fn returns. + pub async fn send_fds(&self, fds: &[T]) -> io::Result<()> { + assert!(self.write_buffer.is_empty()); + + loop { + self.inner.writable().await?; + + match self.inner.try_io(Interest::WRITABLE, || { + let buf = [0u8]; + let iov = &[uio::IoVec::from_slice(&buf)]; + + let fds = fds.iter().map(AsRawFd::as_raw_fd).collect::>(); + let cmsg = [socket::ControlMessage::ScmRights(&fds)]; + + let flags = socket::MsgFlags::MSG_DONTWAIT; + + socket::sendmsg(self.inner.as_raw_fd(), iov, &cmsg, flags, None).map_err(os_err) + }) { + Ok(_) => break Ok(()), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => break Err(e), + } + } + } + + /// Receive a file descriptor via the stream and convert it to T + pub async fn recv_fds(&self) -> io::Result<[T; N]> { + assert!(self.read_buffer.is_empty()); + + loop { + self.inner.readable().await?; + + let mut buf = [0u8]; + let iov = &[uio::IoVec::from_mut_slice(&mut buf)]; + let mut cmsg_buffer = cmsg_space!([RawFd; N]); + let flags = socket::MsgFlags::MSG_DONTWAIT; + + match self.inner.try_io(Interest::READABLE, || { + socket::recvmsg(self.inner.as_raw_fd(), iov, Some(&mut cmsg_buffer), flags) + .map_err(os_err) + }) { + Ok(message) => break recv_control_msg::(message.cmsgs().next()), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => break Err(e), + } + } + } +} + +#[inline] +fn os_err(err: nix::Error) -> io::Error { + io::Error::from_raw_os_error(err as i32) +} + +impl From for AsyncMessage { + fn from(inner: UnixStream) -> Self { + Self { + inner, + write_buffer: BytesMut::new(), + read_buffer: BytesMut::new(), + } + } +} + +impl TryFrom for AsyncMessage { + type Error = io::Error; + + fn try_from(inner: std::os::unix::net::UnixStream) -> io::Result { + inner.set_nonblocking(true)?; + let inner = UnixStream::from_std(inner)?; + Ok(AsyncMessage { + inner, + write_buffer: BytesMut::new(), + read_buffer: BytesMut::new(), + }) + } +} + +fn recv_control_msg( + message: Option, +) -> io::Result<[T; N]> { + match message { + Some(socket::ControlMessageOwned::ScmRights(fds)) => { + let mut result: [MaybeUninit; N] = unsafe { MaybeUninit::uninit().assume_init() }; + + for (fd, result) in fds.iter().zip(&mut result) { + result.write(unsafe { T::from_raw_fd(*fd) }); + } + + let ptr = &mut result as *mut _ as *mut [T; N]; + let res = unsafe { ptr.read() }; + core::mem::forget(result); + Ok(res) + } + Some(message) => Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Failed to receive fd: unexpected control message: {:?}", + message + ), + )), + None => Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Failed to receive fd: missing control message: {:?}", + message + ), + )), + } +} + +#[cfg(test)] +mod test { + use std::{io::Seek, process::exit}; + + use nix::unistd::close; + use tokio::{io::AsyncSeekExt, runtime::Builder}; + + use super::*; + + #[test] + fn send_recv_fd_async() { + let mut fd0 = nix::sys::memfd::memfd_create( + &std::ffi::CString::new("hello").unwrap(), + nix::sys::memfd::MemFdCreateFlag::empty(), + ) + .unwrap(); + let mut fd1 = nix::sys::memfd::memfd_create( + &std::ffi::CString::new("again").unwrap(), + nix::sys::memfd::MemFdCreateFlag::empty(), + ) + .unwrap(); + + let mut pair = super::super::socket_pair().unwrap(); + + const ITERATONS: usize = 100_000; + + match unsafe { nix::unistd::fork() }.unwrap() { + nix::unistd::ForkResult::Parent { child: _ } => { + let parent = pair.first(); + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + rt.block_on(async move { + let stream = AsyncMessage::try_from(parent).unwrap(); + + // Send and receive the fds a couple of times + for _ in 0..ITERATONS { + stream.send_fds(&[fd0, fd1]).await.unwrap(); + close(fd0).unwrap(); + close(fd1).unwrap(); + + let fds = stream.recv_fds::().await.unwrap(); + fd0 = fds[0]; + fd1 = fds[1]; + } + + // Done - check fd content + + let mut buf = String::new(); + let mut file0 = unsafe { tokio::fs::File::from_raw_fd(fd0) }; + file0.seek(io::SeekFrom::Start(0)).await.unwrap(); + file0.read_to_string(&mut buf).await.unwrap(); + assert_eq!(buf, "hello"); + + let mut buf = String::new(); + let mut file1 = unsafe { tokio::fs::File::from_raw_fd(fd1) }; + file1.seek(io::SeekFrom::Start(0)).await.unwrap(); + file1.read_to_string(&mut buf).await.unwrap(); + assert_eq!(buf, "again"); + }); + } + nix::unistd::ForkResult::Child => { + let child = pair.second(); + let rt = Builder::new_current_thread().enable_all().build().unwrap(); + rt.block_on(async move { + let stream = AsyncMessage::try_from(child).unwrap(); + + // Send and receive the fds a couple of times + for _ in 0..ITERATONS { + let mut files = stream.recv_fds::().await.unwrap(); + + files[0].seek(io::SeekFrom::Start(0)).await.unwrap(); + files[0].write_all(b"hello").await.unwrap(); + files[0].flush().await.unwrap(); + + files[1].seek(io::SeekFrom::Start(0)).await.unwrap(); + files[1].write_all(b"again").await.unwrap(); + files[1].flush().await.unwrap(); + + // Send it back + stream.send_fds(&files).await.unwrap(); + } + }); + exit(0); + } + } + } + + #[test] + fn send_recv_fd_blocking() { + let mut fd = nix::sys::memfd::memfd_create( + &std::ffi::CString::new("foo").unwrap(), + nix::sys::memfd::MemFdCreateFlag::empty(), + ) + .unwrap(); + + let mut pair = super::super::socket_pair().unwrap(); + + const ITERATONS: usize = 100_000; + + match unsafe { nix::unistd::fork() }.unwrap() { + nix::unistd::ForkResult::Parent { child: _ } => { + let parent = pair.first(); + let stream = Message::try_from(parent).unwrap(); + for _ in 0..ITERATONS { + stream.send_fds(&[fd]).unwrap(); + close(fd).unwrap(); + fd = stream.recv_fds::().unwrap()[0]; + } + + // Done - check fd content + let mut file = unsafe { std::fs::File::from_raw_fd(fd) }; + file.seek(io::SeekFrom::Start(0)).unwrap(); + let mut buf = String::new(); + file.read_to_string(&mut buf).unwrap(); + assert_eq!(buf, "hello"); + } + nix::unistd::ForkResult::Child => { + let child = pair.second(); + let mut stream = Message::try_from(child).unwrap(); + for _ in 0..ITERATONS { + let mut file = stream.recv_fds::().unwrap(); + + // Write some bytes in to the fd + file[0].seek(io::SeekFrom::Start(0)).unwrap(); + file[0].write_all(b"hello").unwrap(); + file[0].flush().unwrap(); + + // Send it back + stream.send_fds(&[file[0].as_raw_fd()]).unwrap(); + drop(file); + } + stream.recv::().ok(); + exit(0); + } + } + } +} diff --git a/northstar/src/runtime/ipc/mod.rs b/northstar/src/runtime/ipc/mod.rs index 8916f9b12..9da2e7e94 100644 --- a/northstar/src/runtime/ipc/mod.rs +++ b/northstar/src/runtime/ipc/mod.rs @@ -1,4 +1,8 @@ -pub mod channel; -pub mod condition; -pub mod pipe; -pub mod raw_fd_ext; +mod message; +pub mod owned_fd; +mod raw_fd_ext; +mod socket_pair; + +pub use message::{AsyncMessage, Message}; +pub use raw_fd_ext::RawFdExt; +pub use socket_pair::socket_pair; diff --git a/northstar/src/runtime/ipc/owned_fd.rs b/northstar/src/runtime/ipc/owned_fd.rs new file mode 100644 index 000000000..57f50c3fa --- /dev/null +++ b/northstar/src/runtime/ipc/owned_fd.rs @@ -0,0 +1,177 @@ +//! Owned Unix-like file descriptors. + +use std::{ + fmt, + io::ErrorKind, + os::unix::prelude::{AsRawFd, FromRawFd, IntoRawFd, RawFd}, + pin::Pin, + task::{Context, Poll}, +}; + +use nix::{libc, unistd::dup}; +use std::io; +use tokio::io::{unix::AsyncFd, AsyncRead, AsyncWrite, ReadBuf}; + +use super::RawFdExt; + +/// Owned raw fd that closes on drop. +pub struct OwnedFd { + inner: RawFd, +} + +impl OwnedFd { + #[inline] + pub fn clone(&self) -> io::Result { + dup(self.inner) + .map(|inner| Self { inner }) + .map_err(|err| io::Error::from_raw_os_error(err as i32)) + } +} + +impl AsRawFd for OwnedFd { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.inner + } +} + +impl IntoRawFd for OwnedFd { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.inner + } +} + +impl FromRawFd for OwnedFd { + /// Constructs a new instance of `Self` from the given raw file descriptor. + /// + /// # Safety + /// + /// The resource pointed to by `fd` must be open and suitable for assuming + /// ownership. The resource must not require any cleanup other than `close`. + #[inline] + unsafe fn from_raw_fd(inner: RawFd) -> Self { + assert_ne!(inner, u32::MAX as RawFd); + Self { inner } + } +} + +impl Drop for OwnedFd { + #[inline] + fn drop(&mut self) { + unsafe { + // Note that errors are ignored when closing a file descriptor. The + // reason for this is that if an error occurs we don't actually know if + // the file descriptor was closed or not, and if we retried (for + // something like EINTR), we might close another valid file descriptor + // opened after we closed ours. + let _ = libc::close(self.inner); + } + } +} + +impl fmt::Debug for OwnedFd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnedFd").field("fd", &self.inner).finish() + } +} + +impl From for OwnedFd { + #[inline] + fn from(stream: std::os::unix::net::UnixStream) -> Self { + unsafe { Self::from_raw_fd(stream.into_raw_fd()) } + } +} + +pub struct OwnedFdRw { + inner: AsyncFd, +} + +impl OwnedFdRw { + pub fn new(inner: OwnedFd) -> io::Result { + inner.set_nonblocking(true)?; + AsyncFd::new(inner).map(|inner| Self { inner }) + } +} + +impl AsRawFd for OwnedFdRw { + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.inner.as_raw_fd() + } +} + +impl AsyncRead for OwnedFdRw { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + loop { + let mut ready = match self.inner.poll_read_ready(cx) { + Poll::Ready(x) => x?, + Poll::Pending => return Poll::Pending, + }; + + let ret = unsafe { + nix::libc::read( + self.as_raw_fd(), + buf.unfilled_mut() as *mut _ as _, + buf.remaining(), + ) + }; + + return if ret < 0 { + let e = io::Error::last_os_error(); + if e.kind() == ErrorKind::WouldBlock { + ready.clear_ready(); + continue; + } else { + Poll::Ready(Err(e)) + } + } else { + let n = ret as usize; + unsafe { buf.assume_init(n) }; + buf.advance(n); + Poll::Ready(Ok(())) + }; + } + } +} + +impl AsyncWrite for OwnedFdRw { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + loop { + let mut ready = match self.inner.poll_write_ready(cx) { + Poll::Ready(x) => x?, + Poll::Pending => return Poll::Pending, + }; + + let ret = unsafe { nix::libc::write(self.as_raw_fd(), buf.as_ptr() as _, buf.len()) }; + + return if ret < 0 { + let e = io::Error::last_os_error(); + if e.kind() == ErrorKind::WouldBlock { + ready.clear_ready(); + continue; + } else { + Poll::Ready(Err(e)) + } + } else { + Poll::Ready(Ok(ret as usize)) + }; + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} diff --git a/northstar/src/runtime/ipc/pipe.rs b/northstar/src/runtime/ipc/pipe.rs deleted file mode 100644 index 4d6042bb1..000000000 --- a/northstar/src/runtime/ipc/pipe.rs +++ /dev/null @@ -1,345 +0,0 @@ -use futures::ready; -use nix::unistd; -use std::{ - convert::TryFrom, - io, - io::Result, - mem, - os::unix::io::{AsRawFd, IntoRawFd, RawFd}, - pin::Pin, - task::{Context, Poll}, -}; -use tokio::io::{unix::AsyncFd, AsyncRead, AsyncWrite, ReadBuf}; - -use super::raw_fd_ext::RawFdExt; - -#[derive(Debug)] -struct Inner { - fd: RawFd, -} - -impl Drop for Inner { - fn drop(&mut self) { - unistd::close(self.fd).ok(); - } -} - -impl From for Inner { - fn from(fd: RawFd) -> Self { - Inner { fd } - } -} - -/// Opens a pipe(2) with both ends blocking -pub fn pipe() -> Result<(PipeRead, PipeWrite)> { - unistd::pipe().map_err(from_nix).map(|(read, write)| { - ( - PipeRead { inner: read.into() }, - PipeWrite { - inner: write.into(), - }, - ) - }) -} - -/// Read end of a pipe(2). Last dropped clone closes the pipe -#[derive(Debug)] -pub struct PipeRead { - inner: Inner, -} - -impl io::Read for PipeRead { - fn read(&mut self, buf: &mut [u8]) -> Result { - unistd::read(self.as_raw_fd(), buf).map_err(from_nix) - } -} - -impl AsRawFd for PipeRead { - fn as_raw_fd(&self) -> RawFd { - self.inner.fd - } -} - -impl IntoRawFd for PipeRead { - fn into_raw_fd(self) -> RawFd { - let fd = self.inner.fd; - mem::forget(self); - fd - } -} - -/// Write end of a pipe(2). Last dropped clone closes the pipe -#[derive(Debug)] -pub struct PipeWrite { - inner: Inner, -} - -impl io::Write for PipeWrite { - fn write(&mut self, buf: &[u8]) -> Result { - unistd::write(self.as_raw_fd(), buf).map_err(from_nix) - } - - fn flush(&mut self) -> Result<()> { - unistd::fsync(self.as_raw_fd()).map_err(from_nix) - } -} - -impl AsRawFd for PipeWrite { - fn as_raw_fd(&self) -> RawFd { - self.inner.fd - } -} - -impl IntoRawFd for PipeWrite { - fn into_raw_fd(self) -> RawFd { - let fd = self.inner.fd; - mem::forget(self); - fd - } -} - -/// Pipe's synchronous reading end -#[derive(Debug)] -pub struct AsyncPipeRead { - inner: AsyncFd, -} - -impl TryFrom for AsyncPipeRead { - type Error = io::Error; - - fn try_from(reader: PipeRead) -> Result { - reader.set_nonblocking(); - Ok(AsyncPipeRead { - inner: AsyncFd::new(reader)?, - }) - } -} - -impl AsyncRead for AsyncPipeRead { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - loop { - let mut guard = ready!(self.inner.poll_read_ready(cx))?; - match guard.try_io(|inner| { - let fd = inner.get_ref().as_raw_fd(); - // map nix::Error to io::Error - match unistd::read(fd, buf.initialized_mut()) { - Ok(n) => Ok(n), - // read(2) on a nonblocking file (O_NONBLOCK) returns EAGAIN or EWOULDBLOCK in - // case that the read would block. That case is handled by `try_io`. - Err(e) => Err(from_nix(e)), - } - }) { - Ok(Ok(n)) => { - buf.advance(n); - return Poll::Ready(Ok(())); - } - Ok(Err(e)) if e.kind() == io::ErrorKind::WouldBlock => { - return Poll::Pending; - } - Ok(Err(e)) => { - return Poll::Ready(Err(e)); - } - Err(_would_block) => continue, - } - } - } -} - -/// Pipe's asynchronous writing end -#[derive(Debug)] -pub struct AsyncPipeWrite { - inner: AsyncFd, -} - -impl TryFrom for AsyncPipeWrite { - type Error = io::Error; - - fn try_from(write: PipeWrite) -> Result { - write.set_nonblocking(); - Ok(AsyncPipeWrite { - inner: AsyncFd::new(write)?, - }) - } -} - -impl AsyncWrite for AsyncPipeWrite { - fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - loop { - let mut guard = ready!(self.inner.poll_write_ready(cx))?; - match guard.try_io(|inner| unistd::write(inner.as_raw_fd(), buf).map_err(from_nix)) { - Ok(result) => return Poll::Ready(result), - Err(_would_block) => continue, - } - } - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - unistd::fsync(self.inner.as_raw_fd()).map_err(from_nix)?; - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} - -/// Maps an nix::Error to a io::Error -fn from_nix(error: nix::Error) -> io::Error { - match error { - nix::Error::EAGAIN => io::Error::from(io::ErrorKind::WouldBlock), - _ => io::Error::new(io::ErrorKind::Other, error), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::{ - convert::TryInto, - io::{Read, Write}, - process, thread, - }; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - #[test] - /// Smoke test - fn smoke() { - let (mut read, mut write) = pipe().unwrap(); - - write.write_all(b"Hello").unwrap(); - - let mut buf = [0u8; 5]; - read.read_exact(&mut buf).unwrap(); - - assert_eq!(&buf, b"Hello"); - } - - #[test] - /// Closing the write end must produce EOF on the read end - fn close() { - let (mut read, mut write) = pipe().unwrap(); - - write.write_all(b"Hello").unwrap(); - drop(write); - - let mut buf = String::new(); - // Read::read_to_string reads until EOF - read.read_to_string(&mut buf).unwrap(); - - assert_eq!(&buf, "Hello"); - } - - #[test] - #[should_panic] - /// Dropping the write end must reault in an EOF - fn drop_writer() { - let (mut read, write) = pipe().unwrap(); - drop(write); - assert!(matches!(read.read_exact(&mut [0u8; 1]), Ok(()))); - } - - #[test] - #[should_panic] - /// Dropping the read end must reault in an error on write - fn drop_reader() { - let (read, mut write) = pipe().unwrap(); - drop(read); - loop { - write.write_all(b"test").expect("Failed to send"); - } - } - - #[test] - /// Read and write bytes - fn read_write() { - let (mut read, mut write) = pipe().unwrap(); - - let writer = thread::spawn(move || { - for n in 0..=65535u32 { - write.write_all(&n.to_be_bytes()).unwrap(); - } - }); - - let mut buf = [0u8; 4]; - for n in 0..=65535u32 { - read.read_exact(&mut buf).unwrap(); - assert_eq!(buf, n.to_be_bytes()); - } - - writer.join().unwrap(); - } - - #[tokio::test] - /// Test async version of read and write - async fn r#async() { - let (read, write) = pipe().unwrap(); - - let mut read: AsyncPipeRead = read.try_into().unwrap(); - let mut write: AsyncPipeWrite = write.try_into().unwrap(); - - let write = tokio::spawn(async move { - for n in 0..=65535u32 { - write.write_all(&n.to_be_bytes()).await.unwrap(); - } - }); - - let mut buf = [0u8; 4]; - for n in 0..=65535u32 { - read.read_exact(&mut buf).await.unwrap(); - assert_eq!(buf, n.to_be_bytes()); - } - - write.await.unwrap() - } - - #[test] - /// Fork test - fn fork() { - let (mut read, mut write) = pipe().unwrap(); - - match unsafe { unistd::fork().unwrap() } { - unistd::ForkResult::Parent { child } => { - drop(read); - for n in 0..=65535u32 { - write.write_all(&n.to_be_bytes()).unwrap(); - } - nix::sys::wait::waitpid(child, None).ok(); - } - unistd::ForkResult::Child => { - drop(write); - let mut buf = [0u8; 4]; - for n in 0..=65535u32 { - read.read_exact(&mut buf).unwrap(); - assert_eq!(buf, n.to_be_bytes()); - } - process::exit(0); - } - } - - // And the other way round... - let (mut read, mut write) = pipe().unwrap(); - - match unsafe { unistd::fork().unwrap() } { - unistd::ForkResult::Parent { child } => { - drop(write); - let mut buf = [0u8; 4]; - for n in 0..=65535u32 { - read.read_exact(&mut buf).unwrap(); - assert_eq!(buf, n.to_be_bytes()); - } - nix::sys::wait::waitpid(child, None).ok(); - } - unistd::ForkResult::Child => { - drop(read); - for n in 0..=65535u32 { - write.write_all(&n.to_be_bytes()).unwrap(); - } - process::exit(0); - } - } - } -} diff --git a/northstar/src/runtime/ipc/raw_fd_ext.rs b/northstar/src/runtime/ipc/raw_fd_ext.rs index 2bd482da4..86792360f 100644 --- a/northstar/src/runtime/ipc/raw_fd_ext.rs +++ b/northstar/src/runtime/ipc/raw_fd_ext.rs @@ -1,35 +1,33 @@ -use std::{io, io::Result, os::unix::prelude::AsRawFd}; - use nix::fcntl; +use std::{io, io::Result, os::unix::prelude::AsRawFd}; -/// Sets O_NONBLOCK flag on self pub trait RawFdExt: AsRawFd { - fn set_nonblocking(&self); - fn set_blocking(&self); + /// Returns true of self is set to non-blocking. + fn is_nonblocking(&self) -> Result; + + /// Set non-blocking mode. + fn set_nonblocking(&self, value: bool) -> Result<()>; + + /// Set close-on-exec flag. fn set_cloexec(&self, value: bool) -> Result<()>; } impl RawFdExt for T { - fn set_nonblocking(&self) { - unsafe { - let opt = nix::libc::fcntl(self.as_raw_fd(), nix::libc::F_GETFL); - nix::libc::fcntl( - self.as_raw_fd(), - nix::libc::F_SETFL, - opt | nix::libc::O_NONBLOCK, - ); - } + fn is_nonblocking(&self) -> Result { + let flags = fcntl::fcntl(self.as_raw_fd(), fcntl::FcntlArg::F_GETFL) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + Ok(flags & fcntl::OFlag::O_NONBLOCK.bits() != 0) } - fn set_blocking(&self) { - unsafe { - let opt = nix::libc::fcntl(self.as_raw_fd(), nix::libc::F_GETFL); - nix::libc::fcntl( - self.as_raw_fd(), - nix::libc::F_SETFL, - opt & !nix::libc::O_NONBLOCK, - ); - } + fn set_nonblocking(&self, nonblocking: bool) -> Result<()> { + let flags = fcntl::fcntl(self.as_raw_fd(), fcntl::FcntlArg::F_GETFL) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let mut flags = fcntl::OFlag::from_bits_truncate(flags); + flags.set(fcntl::OFlag::O_NONBLOCK, nonblocking); + + fcntl::fcntl(self.as_raw_fd(), fcntl::FcntlArg::F_SETFL(flags)) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .map(drop) } fn set_cloexec(&self, value: bool) -> Result<()> { @@ -44,3 +42,25 @@ impl RawFdExt for T { .map(drop) } } + +#[test] +fn non_blocking() { + let (a, b) = nix::unistd::pipe().unwrap(); + nix::unistd::close(b).unwrap(); + + let opt = unsafe { nix::libc::fcntl(a, nix::libc::F_GETFL) }; + assert!((dbg!(opt) & nix::libc::O_NONBLOCK) == 0); + assert!(!a.is_nonblocking().unwrap()); + + a.set_nonblocking(true).unwrap(); + let opt = unsafe { nix::libc::fcntl(a, nix::libc::F_GETFL) }; + assert!((dbg!(opt) & nix::libc::O_NONBLOCK) != 0); + assert!(a.is_nonblocking().unwrap()); + + a.set_nonblocking(false).unwrap(); + let opt = unsafe { nix::libc::fcntl(a, nix::libc::F_GETFL) }; + assert!((dbg!(opt) & nix::libc::O_NONBLOCK) == 0); + assert!(!a.is_nonblocking().unwrap()); + + nix::unistd::close(a).unwrap(); +} diff --git a/northstar/src/runtime/ipc/socket_pair.rs b/northstar/src/runtime/ipc/socket_pair.rs new file mode 100644 index 000000000..6bd65d680 --- /dev/null +++ b/northstar/src/runtime/ipc/socket_pair.rs @@ -0,0 +1,43 @@ +use std::os::unix::net::UnixStream; + +use tokio::net::UnixStream as TokioUnixStream; + +pub fn socket_pair() -> std::io::Result { + let (first, second) = UnixStream::pair()?; + + Ok(SocketPair { + first: Some(first), + second: Some(second), + }) +} + +#[derive(Debug)] +pub struct SocketPair { + first: Option, + second: Option, +} + +impl SocketPair { + pub fn first(&mut self) -> UnixStream { + self.second.take().unwrap(); + self.first.take().unwrap() + } + + pub fn second(&mut self) -> UnixStream { + self.first.take().unwrap(); + self.second.take().unwrap() + } + + pub fn first_async(&mut self) -> std::io::Result { + let socket = self.first(); + socket.set_nonblocking(true)?; + TokioUnixStream::from_std(socket) + } + + #[allow(dead_code)] + pub fn second_async(&mut self) -> std::io::Result { + let socket = self.second(); + socket.set_nonblocking(true)?; + TokioUnixStream::from_std(socket) + } +} diff --git a/northstar/src/runtime/mod.rs b/northstar/src/runtime/mod.rs index 92a1894f6..02d1617d1 100644 --- a/northstar/src/runtime/mod.rs +++ b/northstar/src/runtime/mod.rs @@ -1,12 +1,21 @@ -use crate::{api, api::model::Container}; +use crate::{api, api::model::Container, runtime::ipc::AsyncMessage}; +use async_stream::stream; use config::Config; use error::Error; use fmt::Debug; -use futures::{future::ready, FutureExt}; -use log::debug; +use futures::{ + future::{ready, Either}, + FutureExt, StreamExt, +}; +use log::{debug, info}; use nix::{ libc::{EXIT_FAILURE, EXIT_SUCCESS}, - sys, + sys::{ + self, + signal::Signal, + wait::{waitpid, WaitStatus}, + }, + unistd, }; use serde::{Deserialize, Serialize}; use state::State; @@ -15,30 +24,33 @@ use std::{ fmt::{self}, future::Future, path::Path, - pin::Pin, - task::{Context, Poll}, }; use sync::mpsc; use tokio::{ + pin, select, sync::{self, broadcast, oneshot}, - task, + task::{self, JoinHandle}, }; -use tokio_util::sync::CancellationToken; +use tokio_util::sync::{CancellationToken, DropGuard}; + +use self::fork::ForkerChannels; mod cgroups; -/// Runtime configuration -pub mod config; mod console; mod debug; mod error; +mod fork; +mod io; mod ipc; mod key; mod mount; -pub(crate) mod process; mod repository; mod state; pub(crate) mod stats; +/// Runtime configuration +pub mod config; + type EventTx = mpsc::Sender; type NotificationTx = broadcast::Sender<(Container, ContainerEvent)>; type RepositoryId = String; @@ -51,9 +63,11 @@ const EVENT_BUFFER_SIZE: usize = 1000; const NOTIFICATION_BUFFER_SIZE: usize = 1000; /// Environment variable name passed to the container with the containers name -const ENV_NAME: &str = "NAME"; +const ENV_NAME: &str = "NORTHSTAR_NAME"; /// Environment variable name passed to the container with the containers version -const ENV_VERSION: &str = "VERSION"; +const ENV_VERSION: &str = "NORTHSTAR_VERSION"; +/// Environment variable name passed to the container with the containers id +const ENV_CONTAINER: &str = "NORTHSTAR_CONTAINER"; #[derive(Debug)] enum Event { @@ -126,6 +140,18 @@ pub enum ExitStatus { Signalled(u8), } +impl From for ExitStatus { + fn from(signal: Signal) -> Self { + ExitStatus::Signalled(signal as u8) + } +} + +impl From for ExitStatus { + fn from(code: ExitCode) -> Self { + ExitStatus::Exit(code) + } +} + impl ExitStatus { /// Exit success pub const SUCCESS: ExitCode = EXIT_SUCCESS; @@ -150,94 +176,128 @@ impl fmt::Display for ExitStatus { } } -/// Result of a Runtime action -pub type RuntimeResult = Result<(), Error>; - -/// Handle to the Northstar runtime -pub struct Runtime { - /// Channel receive a stop signal for the runtime - /// Drop the tx part to gracefully shutdown the mail loop. - stop: CancellationToken, - // Channel to signal the runtime exit status to the caller of `start` - // When the runtime is shut down the result of shutdown is sent to this - // channel. If a error happens during normal operation the error is also - // sent to this channel. - stopped: oneshot::Receiver, - // Runtime task - task: task::JoinHandle<()>, +/// Runtime handle +#[allow(clippy::large_enum_variant)] +pub enum Runtime { + /// The runtime is created but not yet started. + Created { + /// Runtime configuration + config: Config, + /// Forker pid + forker_pid: Pid, + /// Forker channles + forker_channels: ForkerChannels, + }, + /// The runtime is started. + Running { + /// Drop guard to stop the runtime + guard: DropGuard, + /// Runtime task + task: JoinHandle>, + }, } impl Runtime { + /// Create new runtime instance + pub fn new(config: Config) -> Result { + let (forker_pid, forker_channels) = fork::start()?; + Ok(Runtime::Created { + config, + forker_pid, + forker_channels, + }) + } + /// Start runtime with configuration `config` - pub async fn start(config: Config) -> Result { + pub async fn start(self) -> Result { + let (config, forker_pid, forker_channels) = if let Runtime::Created { + config, + forker_pid, + forker_channels, + } = self + { + (config, forker_pid, forker_channels) + } else { + panic!("Runtime::start called on a running runtime"); + }; + config.check().await?; - let stop = CancellationToken::new(); - let (stopped_tx, stopped) = oneshot::channel(); + let token = CancellationToken::new(); + let guard = token.clone().drop_guard(); // Start a task that drives the main loop and wait for shutdown results - let stop_task = stop.clone(); - let task = task::spawn(async move { - match runtime_task(&config, stop_task).await { - Err(e) => { - log::error!("Runtime error: {}", e); - stopped_tx.send(Err(e)).ok(); - } - Ok(_) => drop(stopped_tx.send(Ok(()))), - }; - }); - - Ok(Runtime { - stop, - stopped, - task, - }) + let task = task::spawn(run(config, token, forker_pid, forker_channels)); + + Ok(Runtime::Running { guard, task }) } /// Stop the runtime and wait for the termination - pub fn shutdown(self) -> impl Future { - self.stop.cancel(); - let stopped = self.stopped; - self.task.then(|_| { - stopped.then(|n| match n { - Ok(n) => ready(n), - Err(_) => ready(Ok(())), + pub fn shutdown(self) -> impl Future> { + if let Runtime::Running { guard, task } = self { + drop(guard); + Either::Left({ + task.then(|n| match n { + Ok(n) => ready(n), + Err(_) => ready(Ok(())), + }) }) - }) + } else { + Either::Right(futures::future::ready(Ok(()))) + } } -} - -impl Future for Runtime { - type Output = RuntimeResult; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match Pin::new(&mut self.stopped).poll(cx) { - Poll::Ready(r) => match r { - Ok(r) => Poll::Ready(r), - // Channel error -> tx side dropped - Err(_) => Poll::Ready(Ok(())), + /// Wait for the runtime to stop + pub async fn stopped(&mut self) -> Result<(), Error> { + match self { + Runtime::Running { ref mut task, .. } => match task.await { + Ok(r) => r, + Err(_) => Ok(()), }, - Poll::Pending => Poll::Pending, + Runtime::Created { .. } => panic!("Stopped called on a stopped runtime"), } } } -async fn runtime_task(config: &'_ Config, stop: CancellationToken) -> Result<(), Error> { - let cgroup = Path::new(&*config.cgroup.as_str()); - cgroups::init(cgroup).await?; +/// Main loop +async fn run( + config: Config, + token: CancellationToken, + forker_pid: Pid, + forker_channels: ForkerChannels, +) -> Result<(), Error> { + // Setup root cgroup(s) + let cgroup = Path::new(config.cgroup.as_str()).to_owned(); + cgroups::init(&cgroup).await?; + + // Join forker + let mut join_forker = task::spawn_blocking(move || { + let pid = unistd::Pid::from_raw(forker_pid as i32); + loop { + match waitpid(Some(pid), None) { + Ok(WaitStatus::Exited(_pid, status)) => { + break ExitStatus::Exit(status); + } + Ok(WaitStatus::Signaled(_pid, status, _)) => { + break ExitStatus::Signalled(status as u8); + } + Ok(WaitStatus::Continued(_)) | Ok(WaitStatus::Stopped(_, _)) => (), + Err(nix::Error::EINTR) => (), + e => panic!("Failed to waitpid on {}: {:?}", pid, e), + } + } + }); // Northstar runs in a event loop let (event_tx, mut event_rx) = mpsc::channel::(EVENT_BUFFER_SIZE); let (notification_tx, _) = sync::broadcast::channel(NOTIFICATION_BUFFER_SIZE); - let mut state = State::new(config, event_tx.clone(), notification_tx.clone()).await?; // Initialize the console if configured - let console = if let Some(consoles) = config.console.as_ref() { if consoles.is_empty() { None } else { - let mut console = console::Console::new(event_tx.clone(), notification_tx); + let mut console = console::Console::new(event_tx.clone(), notification_tx.clone()); for url in consoles { console.listen(url).await.map_err(Error::Console)?; } @@ -247,34 +307,75 @@ async fn runtime_task(config: &'_ Config, stop: CancellationToken) -> Result<(), None }; - // Wait for a external shutdown request - task::spawn(async move { - stop.cancelled().await; - event_tx.send(Event::Shutdown).await.ok(); - }); + // Convert stream and stream_fd into Tokio UnixStream + let (forker, mut exit_notifications) = { + let ForkerChannels { + stream, + notifications, + } = forker_channels; + + let forker = fork::Forker::new(stream); + let exit_notifications: AsyncMessage<_> = notifications + .try_into() + .expect("Failed to convert exit notification handle"); + (forker, exit_notifications) + }; + + // Merge the exit notification from the forker process with other events into the main loop channel + let event_rx = stream! { + loop { + select! { + Some(event) = event_rx.recv() => yield event, + Ok(Some(fork::Notification::Exit { container, exit_status })) = exit_notifications.recv() => { + let event = ContainerEvent::Exit(exit_status); + yield Event::Container(container, event); + } + else => unimplemented!(), + } + } + }; + pin!(event_rx); + + let mut state = State::new(config, event_tx.clone(), notification_tx, forker).await?; + + info!("Runtime up and running"); // Enter main loop loop { - if let Err(e) = match event_rx.recv().await.unwrap() { - // Process console events enqueued by console::Console - Event::Console(mut msg, txr) => state.on_request(&mut msg, txr).await, - // The runtime os commanded to shut down and exit. - Event::Shutdown => { - debug!("Shutting down Northstar runtime"); - if let Some(console) = console { - debug!("Shutting down console"); - console.shutdown().await.map_err(Error::Console)?; + tokio::select! { + // External shutdown event via the token + _ = token.cancelled() => event_tx.send(Event::Shutdown).await.expect("Failed to send shutdown event"), + // Process events + event = event_rx.next() => { + if let Err(e) = match event.unwrap() { + // Process console events enqueued by console::Console + Event::Console(mut msg, response) => state.on_request(&mut msg, response).await, + // The runtime os commanded to shut down and exit. + Event::Shutdown => { + debug!("Shutting down Northstar runtime"); + if let Some(console) = console { + debug!("Shutting down console"); + console.shutdown().await.map_err(Error::Console)?; + } + break state.shutdown(event_rx).await; + } + // Container event + Event::Container(container, event) => state.on_event(&container, &event, false).await, + } { + break Err(e); } - break state.shutdown().await; } - // Container event - Event::Container(container, event) => state.on_event(&container, &event).await, - } { - break Err(e); + exit_status = &mut join_forker => panic!("Forker exited with {:?}", exit_status), } }?; - cgroups::shutdown(cgroup).await?; + // Terminate forker process + debug!("Joining forker with pid {}", forker_pid); + // signal::kill(forker_pid, Some(SIGTERM)).ok(); + join_forker.await.expect("Failed to join forker"); + + // Shutdown cgroups + cgroups::shutdown(&cgroup).await?; debug!("Shutdown complete"); diff --git a/northstar/src/runtime/mount.rs b/northstar/src/runtime/mount.rs index 4ee4af1fc..89d56dbc0 100644 --- a/northstar/src/runtime/mount.rs +++ b/northstar/src/runtime/mount.rs @@ -4,7 +4,7 @@ use crate::{ npk::{dm_verity::VerityHeader, npk::Hashes}, }; use devicemapper::{DevId, DmError, DmName, DmOptions}; -use futures::Future; +use futures::{Future, FutureExt}; use log::{debug, info, warn}; use loopdev::LoopControl; use std::{ @@ -13,10 +13,9 @@ use std::{ path::{Path, PathBuf}, str::Utf8Error, sync::Arc, - thread, }; use thiserror::Error; -use tokio::{fs, time}; +use tokio::{fs, task, time}; use crate::seccomp::Selinux; pub use nix::mount::MsFlags as MountFlags; @@ -102,7 +101,7 @@ impl MountControl { let selinux = npk.manifest().selinux.clone(); let hashes = npk.hashes().cloned(); - async move { + task::spawn_blocking(move || { let start = time::Instant::now(); debug!("Mounting {}:{}", name, version); @@ -118,8 +117,7 @@ impl MountControl { hashes, &target, key.is_some(), - ) - .await?; + )?; let duration = start.elapsed(); info!( @@ -130,7 +128,11 @@ impl MountControl { ); Ok(device) - } + }) + .map(|r| match r { + Ok(r) => r, + Err(e) => panic!("Task error: {}", e), + }) } pub(super) async fn umount(&self, target: &Path) -> Result<(), Error> { @@ -149,7 +151,7 @@ impl MountControl { } #[allow(clippy::too_many_arguments)] -async fn mount( +fn mount( dm: Arc, lc: Arc, fd: RawFd, @@ -338,9 +340,8 @@ fn dmsetup( debug!("Waiting for verity device {}", device.display(),); while !device.exists() { - // Use a std::thread::sleep because this is run on a futures - // executor and not a tokio runtime - thread::sleep(time::Duration::from_millis(1)); + // This code runs on a dedicated blocking thread + std::thread::sleep(time::Duration::from_millis(1)); if start.elapsed() > DM_DEVICE_TIMEOUT { return Err(Error::Timeout(format!( diff --git a/northstar/src/runtime/process/io.rs b/northstar/src/runtime/process/io.rs deleted file mode 100644 index d9251c10e..000000000 --- a/northstar/src/runtime/process/io.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::{ - super::ipc::pipe::{pipe, AsyncPipeRead}, - Error, -}; -use crate::{ - npk, - npk::manifest::{Level, Manifest, Output}, - runtime::{error::Context as ErrorContext, ipc::pipe::PipeWrite}, -}; -use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, info, trace, warn}; -use nix::libc; -use serde::{Deserialize, Serialize}; -use std::{ - collections::HashMap, - convert::TryInto, - os::unix::prelude::{AsRawFd, RawFd}, - pin::Pin, - task::{Context, Poll}, -}; -use tokio::{ - io::{self, AsyncWrite, BufReader}, - task, -}; - -/// Implement AsyncWrite and forwards lines to Rust log -struct LogSink { - buffer: BytesMut, - level: Level, - tag: String, -} - -impl LogSink { - fn new(level: Level, tag: &str) -> LogSink { - LogSink { - level, - tag: tag.to_string(), - buffer: BytesMut::new(), - } - } -} - -impl LogSink { - fn log(&mut self) { - while let Some(p) = self.buffer.iter().position(|b| *b == b'\n') { - let line = self.buffer.split_to(p); - // Discard the newline - self.buffer.advance(1); - let line = String::from_utf8_lossy(&line); - match self.level { - Level::Trace => trace!("{}: {}", self.tag, line), - Level::Debug => debug!("{}: {}", self.tag, line), - Level::Info => info!("{}: {}", self.tag, line), - Level::Warn => warn!("{}: {}", self.tag, line), - Level::Error => error!("{}: {}", self.tag, line), - } - } - } -} - -impl AsyncWrite for LogSink { - fn poll_write( - mut self: Pin<&mut Self>, - _cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.buffer.extend(buf); - self.log(); - Poll::Ready(Ok(buf.len())) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll> { - // Log even unfinished lines received until now by adding a newline and print - self.buffer.reserve(1); - self.buffer.put_u8(b'\n'); - self.log(); - Poll::Ready(Ok(())) - } -} - -// Writing ends for stdout/stderr -pub(super) struct Io { - _stdout: Option, - _stderr: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) enum Fd { - // Close the fd - Close, - // Dup2 the the fd to fd - Dup(i32), -} - -pub(super) async fn from_manifest( - manifest: &Manifest, -) -> Result<(Option, HashMap), Error> { - let mut fds = HashMap::new(); - - // The default of all fds inherited from the parent is to close it - let mut proc_self_fd = tokio::fs::read_dir("/proc/self/fd") - .await - .context("Readdir")?; - while let Ok(Some(e)) = proc_self_fd.next_entry().await { - let file = e.file_name(); - let fd: i32 = file.to_str().unwrap().parse().unwrap(); // fds are always numeric - fds.insert(fd as RawFd, Fd::Close); - } - drop(proc_self_fd); - - let mut stdout_stderr = |c: Option<&Output>, fd| { - match c { - Some(npk::manifest::Output::Pipe) => { - // Do nothing with the stdout fd - just prevent remove it from the list of fds that - // has been gathered above and instructs the init to close those fds. - fds.remove(&fd); - Result::<_, Error>::Ok(None) - } - Some(npk::manifest::Output::Log { level, ref tag }) => { - // Create a pipe: the writing end is used in the child as stdout/stderr. The reading end is used in a LogSink - let (reader, writer) = pipe().context("Failed to open pipe")?; - let reader_fd = reader.as_raw_fd(); - let reader: AsyncPipeRead = reader - .try_into() - .context("Failed to get async handler from pipe reader")?; - - let mut reader = BufReader::new(reader); - let tag = tag.to_string(); - let mut log_sink = LogSink::new(level.clone(), &tag); - task::spawn(async move { - drop(io::copy_buf(&mut reader, &mut log_sink).await); - }); - - // The read fd shall be closed in the child. It's used in the runtime only - fds.insert(reader_fd, Fd::Close); - - // Remove fd that is set to be Fd::Close by default. fd is closed by dup2 - fds.remove(&writer.as_raw_fd()); - // The writing fd shall be dupped to 2 - fds.insert(fd, Fd::Dup(writer.as_raw_fd())); - - // Return the writer: Drop (that closes) it in the parent. Forget in the child. - Ok(Some(writer)) - } - None => Ok(None), - } - }; - - if let Some(io) = manifest.io.as_ref() { - let io = Some(Io { - _stdout: stdout_stderr(io.stdout.as_ref(), libc::STDOUT_FILENO)?, - _stderr: stdout_stderr(io.stdout.as_ref(), libc::STDERR_FILENO)?, - }); - Ok((io, fds)) - } else { - Ok((None, fds)) - } -} diff --git a/northstar/src/runtime/process/mod.rs b/northstar/src/runtime/process/mod.rs deleted file mode 100644 index 03006a797..000000000 --- a/northstar/src/runtime/process/mod.rs +++ /dev/null @@ -1,569 +0,0 @@ -use super::{ - config::Config, - error::Error, - ipc::{ - channel, - condition::{self, ConditionNotify, ConditionWait}, - }, - ContainerEvent, Event, EventTx, ExitStatus, NotificationTx, Pid, ENV_NAME, ENV_VERSION, -}; -use crate::{ - common::{container::Container, non_null_string::NonNullString}, - npk::manifest::Manifest, - runtime::{ - console::{self, Peer}, - error::Context, - }, - seccomp, -}; -use async_trait::async_trait; -use futures::{future::ready, Future, FutureExt}; -use log::{debug, error, info, warn}; -use nix::{ - errno::Errno, - sys::{self, signal::Signal, socket}, - unistd, -}; -use std::{ - collections::HashMap, - ffi::{c_void, CString}, - fmt, - mem::forget, - os::unix::{ - net::UnixStream as StdUnixStream, - prelude::{AsRawFd, FromRawFd, RawFd}, - }, - path::Path, - ptr::null, -}; -use sys::wait; -use tokio::{net::UnixStream, task, time}; -use tokio_util::sync::CancellationToken; - -mod fs; -mod init; -mod io; -mod trampoline; - -#[derive(Debug)] -pub(super) struct Launcher { - tx: EventTx, - notification_tx: NotificationTx, - config: Config, -} - -pub(super) struct Process { - pid: Pid, - checkpoint: Option, - exit_status: Option + Send + Sync + Unpin>>, -} - -impl fmt::Debug for Process { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Process") - .field("pid", &self.pid) - .field("checkpoint", &self.checkpoint) - .finish() - } -} - -impl Launcher { - pub async fn start( - tx: EventTx, - config: Config, - notification_tx: NotificationTx, - ) -> Result { - set_child_subreaper(true)?; - - let launcher = Launcher { - tx, - config, - notification_tx, - }; - - Ok(launcher) - } - - pub async fn shutdown(self) -> Result<(), Error> { - Ok(()) - } - - /// Create a new container process set - pub async fn create( - &self, - root: &Path, - container: &Container, - manifest: &Manifest, - args: Option<&Vec>, - env: Option<&HashMap>, - ) -> Result { - // Token to stop the console task if any. This token is cancelled when - // the waitpid of this child process signals that the child is exited. See - // `wait`. - let stop = CancellationToken::new(); - let (init, argv) = init_argv(manifest, args); - let mut env = self::env(manifest, env); - - // Setup io and collect fd setup set - let (io, mut fds) = io::from_manifest(manifest).await?; - - // Pipe for sending the init pid from the intermediate process to the runtime - // and the exit status from init to the runtime - // - // Ensure the fds of the channel are *not* in the fds set. The list of fds that are - // closed by init is gathered above. Between the assembly of the list and the new pipes - // for the child pid and the condition variables a io task that forwards logs from containers - // can end. Those io tasks use pipes as well. If such a task ends it closes its fds. Those numbers - // can be in the list of to be closed fds but are reused when the pipe are created. - let channel = channel::Channel::new(); - fds.remove(&channel.as_raw_fd().0); - fds.remove(&channel.as_raw_fd().1); - - // Ensure that the checkpoint fds are not in the fds set and untouched - let (checkpoint_runtime, checkpoint_init) = checkpoints(); - fds.remove(&checkpoint_runtime.as_raw_fd().0); - fds.remove(&checkpoint_runtime.as_raw_fd().1); - - // Setup console if configured - let console_fd = console_fd( - self.tx.clone(), - manifest, - &mut env, - &mut fds, - stop.clone(), - &self.notification_tx, - ) - .await; - - let capabilities = manifest.capabilities.clone(); - let fds = fds.drain().collect::>(); - let uid = manifest.uid; - let gid = manifest.gid; - let groups = groups(manifest); - let mounts = fs::prepare_mounts(&self.config, root, manifest).await?; - let rlimits = manifest.rlimits.clone(); - let root = root.to_owned(); - let seccomp = seccomp_filter(manifest); - - debug!("{} init is {:?}", container, init); - debug!("{} argv is {:?}", container, argv); - debug!("{} env is {:?}", container, env); - - let init = init::Init { - root, - init, - argv, - env, - uid, - gid, - mounts, - fds, - groups, - capabilities, - rlimits, - seccomp, - }; - - // Fork trampoline process - match unsafe { unistd::fork() } { - Ok(result) => match result { - unistd::ForkResult::Parent { child } => { - let trampoline_pid = child; - // Close writing ends of log pipes (if any) - drop(io); - // Close child console socket (if any) - drop(console_fd); - // Close child checkpoint pipes - drop(checkpoint_init); - - // Receive the pid of the init process from the trampoline process - debug!("Waiting for the pid of init of {}", container); - let mut channel = channel.into_async_read(); - let pid = channel.recv::().await.expect("Failed to read pid") as Pid; - debug!("Created {} with pid {}", container, pid); - - // We're done reading the pid. The next information transferred via the - // channel is the exit status of the container process. - - // Reap the trampoline process which is (or will be) a zombie otherwise - debug!("Waiting for trampoline process {} to exit", trampoline_pid); - wait::waitpid(Some(trampoline_pid), None) - .expect("Failed to wait for trampoline process"); - - // Start a task that waits for the exit of the init process - let exit_status_fut = self.container_exit_status(container, channel, pid, stop); - - Ok(Process { - pid, - checkpoint: Some(checkpoint_runtime), - exit_status: Some(Box::new(exit_status_fut)), - }) - } - unistd::ForkResult::Child => { - // Forget writing ends of io which are stdout, stderr. The `forget` - // ensures that the file descriptors are not closed - forget(io); - - // Close checkpoint ends of the runtime - drop(checkpoint_runtime); - - trampoline::trampoline(init, channel, checkpoint_init) - } - }, - Err(e) => panic!("Fork error: {}", e), - } - } - - /// Spawn a task that waits for the containers exit status. If the receive operation - /// fails take the exit status of the init process `pid`. - fn container_exit_status( - &self, - container: &Container, - mut channel: channel::AsyncChannelRead, - pid: Pid, - stop: CancellationToken, - ) -> impl Future { - let container = container.clone(); - let tx = self.tx.clone(); - - // This task lives as long as the child process and doesn't need to be - // cancelled explicitly. - task::spawn(async move { - // Wait for an event on the channel - let status = match channel.recv::().await { - // Init sent something - Ok(exit_status) => { - debug!( - "Received exit status of {} ({}) via channel: {}", - container, pid, exit_status - ); - - // Wait for init to exit. This is needed to ensure the init process - // exited before the runtime starts to cleanup e.g remove cgroups - if let Err(e) = wait::waitpid(Some(unistd::Pid::from_raw(pid as i32)), None) { - panic!("Failed to wait for init process {}: {}", pid, e); - } - - exit_status - } - // The channel is closed before init sent something - Err(e) => { - // This is not an error. If for example the child process exited because - // of a SIGKILL the pipe is just closed and the init process cannot send - // anything there. In such a situation take the exit status of the init - // process as the exit status of the container process. - debug!( - "Failed to receive exit status of {} ({}) via channel: {}", - container, pid, e - ); - - let pid = unistd::Pid::from_raw(pid as i32); - let exit_status = loop { - match wait::waitpid(Some(pid), None) { - Ok(wait::WaitStatus::Exited(pid, code)) => { - debug!("Process {} exit code is {}", pid, code); - break ExitStatus::Exit(code); - } - Ok(wait::WaitStatus::Signaled(pid, signal, _dump)) => { - debug!("Process {} exit status is signal {}", pid, signal); - break ExitStatus::Signalled(signal as u8); - } - Ok(r) => unreachable!("Unexpected wait status of init: {:?}", r), - Err(nix::Error::EINTR) => continue, - Err(e) => panic!("Failed to waitpid on {}: {}", pid, e), - } - }; - debug!("Exit status of {} ({}): {}", container, pid, exit_status); - exit_status - } - }; - - // Stop console connection if any - stop.cancel(); - - // Send container exit event to the runtime main loop - let event = ContainerEvent::Exit(status.clone()); - tx.send(Event::Container(container, event)) - .await - .expect("Failed to send container event"); - - status - }) - .then(|r| match r { - Ok(r) => ready(r), - Err(_) => panic!("Task error"), - }) - } -} - -#[async_trait] -impl super::state::Process for Process { - fn pid(&self) -> Pid { - self.pid - } - - async fn spawn(&mut self) -> Result<(), Error> { - let checkpoint = self - .checkpoint - .take() - .expect("Attempt to start container twice. This is a bug."); - info!("Starting {}", self.pid()); - let wait = checkpoint.notify(); - - // If the child process refuses to start - kill it after 5 seconds - match time::timeout(time::Duration::from_secs(5), wait.async_wait()).await { - Ok(_) => (), - Err(_) => { - error!( - "Timeout while waiting for {} to start. Sending SIGKILL to {}", - self.pid, self.pid - ); - let process_group = unistd::Pid::from_raw(-(self.pid as i32)); - let sigkill = Some(sys::signal::SIGKILL); - sys::signal::kill(process_group, sigkill).ok(); - } - } - - Ok(()) - } - - async fn kill(&mut self, signal: Signal) -> Result<(), super::error::Error> { - debug!("Sending {} to {}", signal.as_str(), self.pid); - let process_group = unistd::Pid::from_raw(-(self.pid as i32)); - let sigterm = Some(signal); - match sys::signal::kill(process_group, sigterm) { - // The process is terminated already. Wait for the waittask to do it's job and resolve exit_status - Err(nix::Error::ESRCH) => { - debug!("Process {} already exited", self.pid); - Ok(()) - } - result => result.context(format!( - "Failed to send signal {} {}", - signal, process_group - )), - } - } - - async fn wait(&mut self) -> Result { - let exit_status = self.exit_status.take().expect("Wait called twice"); - Ok(exit_status.await) - } - - async fn destroy(&mut self) -> Result<(), Error> { - Ok(()) - } -} - -/// Construct the init and argv argument for the containers execve -fn init_argv(manifest: &Manifest, args: Option<&Vec>) -> (CString, Vec) { - // A container without an init shall not be started - // Validation of init is done in `Manifest` - let init = CString::new( - manifest - .init - .as_ref() - .expect("Attempt to use init from resource container") - .to_str() - .expect("Invalid init. This a bug in the manifest validation"), - ) - .expect("Invalid init"); - - // If optional arguments are defined, discard the values from the manifest. - // if there are no optional args - take the values from the manifest if present - // or nothing. - let args = match (manifest.args.as_ref(), args) { - (None, None) => &[], - (None, Some(a)) => a.as_slice(), - (Some(m), None) => m.as_slice(), - (Some(_), Some(a)) => a.as_slice(), - }; - - let mut argv = Vec::with_capacity(1 + args.len()); - argv.push(init.clone()); - argv.extend({ - args.iter().map(|arg| { - CString::new(arg.as_bytes()) - .expect("Invalid arg. This is a bug in the manifest or parameter validation") - }) - }); - - // argv - (init, argv) -} - -/// Construct the env argument for the containers execve. Optional args and env overwrite values from the -/// manifest. -fn env(manifest: &Manifest, env: Option<&HashMap>) -> Vec { - let mut result = Vec::with_capacity(2); - result.push( - CString::new(format!("{}={}", ENV_NAME, manifest.name)) - .expect("Invalid container name. This is a bug in the manifest validation"), - ); - result.push(CString::new(format!("{}={}", ENV_VERSION, manifest.version)).unwrap()); - - if let Some(ref e) = manifest.env { - result.extend({ - e.iter() - .filter(|(k, _)| { - // Skip the values declared in fn arguments - env.map(|env| !env.contains_key(k)).unwrap_or(true) - }) - .map(|(k, v)| { - CString::new(format!("{}={}", k, v)) - .expect("Invalid env. This is a bug in the manifest validation") - }) - }) - } - - // Add additional env variables passed - if let Some(env) = env { - result.extend( - env.iter().map(|(k, v)| { - CString::new(format!("{}={}", k, v)).expect("Invalid additional env") - }), - ); - } - - result -} - -/// Open a socket that is passed via env variable to the child. The peer of the -/// socket is a console connection handling task -async fn console_fd( - event_tx: EventTx, - manifest: &Manifest, - env: &mut Vec, - fds: &mut HashMap, - stop: CancellationToken, - notification_tx: &NotificationTx, -) -> Option { - if manifest.console { - let (runtime_socket, client_socket) = socket::socketpair( - socket::AddressFamily::Unix, - socket::SockType::Stream, - None, - socket::SockFlag::empty(), - ) - .expect("Failed to create socketpair"); - - // Add the fd number to the environment of the application - env.push(CString::new(format!("NORTHSTAR_CONSOLE={}", client_socket)).unwrap()); - - // Make sure that the server socket is closed in the child before exeve - fds.insert(runtime_socket, io::Fd::Close); - // Make sure the client socket is not included in the list to close fds - fds.remove(&client_socket.as_raw_fd()); - - // Convert std raw fd - let std = unsafe { StdUnixStream::from_raw_fd(runtime_socket) }; - std.set_nonblocking(true) - .expect("Failed to set socket into nonblocking mode"); - let io = UnixStream::from_std(std).expect("Failed to convert Unix socket"); - - let peer = Peer::from(format!("{}:{}", manifest.name, manifest.version).as_str()); - - // Start console - task::spawn(console::Console::connection( - io, - peer, - stop, - event_tx, - notification_tx.subscribe(), - None, - )); - - Some(unsafe { StdUnixStream::from_raw_fd(client_socket) }) - } else { - None - } -} - -/// Generate a list of supplementary gids if the groups info can be retrieved. This -/// must happen before the init `clone` because the group information cannot be gathered -/// without `/etc` etc... -fn groups(manifest: &Manifest) -> Vec { - if let Some(groups) = manifest.suppl_groups.as_ref() { - let mut result = Vec::with_capacity(groups.len()); - for group in groups { - let cgroup = CString::new(group.as_str()).unwrap(); // Check during manifest parsing - let group_info = - unsafe { nix::libc::getgrnam(cgroup.as_ptr() as *const nix::libc::c_char) }; - if group_info == (null::() as *mut nix::libc::group) { - warn!("Skipping invalid supplementary group {}", group); - } else { - let gid = unsafe { (*group_info).gr_gid }; - // TODO: Are there gids cannot use? - result.push(gid) - } - } - result - } else { - Vec::with_capacity(0) - } -} - -/// Generate seccomp filter applied in init -fn seccomp_filter(manifest: &Manifest) -> Option { - if let Some(seccomp) = manifest.seccomp.as_ref() { - return Some(seccomp::seccomp_filter( - seccomp.profile.as_ref(), - seccomp.allow.as_ref(), - manifest.capabilities.as_ref(), - )); - } - None -} - -// Set the child subreaper flag of the calling thread -fn set_child_subreaper(value: bool) -> Result<(), Error> { - #[cfg(target_os = "android")] - const PR_SET_CHILD_SUBREAPER: nix::libc::c_int = 36; - #[cfg(not(target_os = "android"))] - use nix::libc::PR_SET_CHILD_SUBREAPER; - - let value = if value { 1u64 } else { 0u64 }; - let result = unsafe { nix::libc::prctl(PR_SET_CHILD_SUBREAPER, value, 0, 0, 0) }; - Errno::result(result) - .map(drop) - .context("Set child subreaper flag") -} - -pub(super) struct Checkpoint(ConditionWait, ConditionNotify); - -fn checkpoints() -> (Checkpoint, Checkpoint) { - let a = condition::Condition::new().expect("Failed to create condition"); - a.set_cloexec(); - let b = condition::Condition::new().expect("Failed to create condition"); - b.set_cloexec(); - - let (aw, an) = a.split(); - let (bw, bn) = b.split(); - - (Checkpoint(aw, bn), Checkpoint(bw, an)) -} - -impl Checkpoint { - fn notify(self) -> ConditionWait { - self.1.notify(); - self.0 - } - - fn wait(self) -> ConditionNotify { - self.0.wait(); - self.1 - } - - /// Raw file descriptor number of the rx and tx pipe - fn as_raw_fd(&self) -> (RawFd, RawFd) { - (self.0.as_raw_fd(), self.1.as_raw_fd()) - } -} - -impl std::fmt::Debug for Checkpoint { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Checkpoint") - .field("wait", &self.0.as_raw_fd()) - .field("notifiy", &self.1.as_raw_fd()) - .finish() - } -} diff --git a/northstar/src/runtime/process/trampoline.rs b/northstar/src/runtime/process/trampoline.rs deleted file mode 100644 index 5e93b5895..000000000 --- a/northstar/src/runtime/process/trampoline.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::{init::Init, Checkpoint}; -use crate::runtime::ipc::channel::Channel; -use nix::{sched, unistd}; -use std::process::exit; - -pub(super) fn trampoline(init: Init, mut child_channel: Channel, checkpoint_init: Checkpoint) -> ! { - // Create pid namespace - sched::unshare(sched::CloneFlags::CLONE_NEWPID).expect("Failed to create pid namespace"); - - // Fork the init process - match unsafe { unistd::fork() }.expect("Failed to fork init") { - unistd::ForkResult::Parent { child } => { - // Send the pid of init to the runtime and exit - let pid = child.as_raw() as i32; - child_channel.send(&pid).expect("Failed to send init pid"); - exit(0); - } - unistd::ForkResult::Child => { - // Wait for the runtime to signal that init may start. - let condition_notify = checkpoint_init.wait(); - - // Dive into init and never return - init.run(condition_notify, child_channel); - } - } -} diff --git a/northstar/src/runtime/repository.rs b/northstar/src/runtime/repository.rs index be214b8ca..be515a262 100644 --- a/northstar/src/runtime/repository.rs +++ b/northstar/src/runtime/repository.rs @@ -3,17 +3,16 @@ use super::{ key::{self, PublicKey}, Container, }; -use crate::{ - npk::npk::{self}, - runtime::ipc::raw_fd_ext::RawFdExt, -}; +use crate::{npk::npk::Npk as NpkNpk, runtime::ipc::RawFdExt}; use bytes::Bytes; +use futures::{future::try_join_all, FutureExt}; use log::{debug, info}; use mpsc::Receiver; use nanoid::nanoid; use std::{ collections::HashMap, fmt, + future::ready, io::{BufReader, SeekFrom}, os::unix::prelude::{AsRawFd, FromRawFd, IntoRawFd}, path::{Path, PathBuf}, @@ -22,10 +21,11 @@ use tokio::{ fs::{self}, io::{AsyncSeekExt, AsyncWriteExt}, sync::mpsc, + task, time::Instant, }; -pub(super) type Npk = crate::npk::npk::Npk>; +pub(super) type Npk = NpkNpk>; #[async_trait::async_trait] pub(super) trait Repository: fmt::Debug { @@ -73,30 +73,40 @@ impl DirRepository { let mut readir = fs::read_dir(&dir).await.context("Repository read dir")?; let start = Instant::now(); + let mut tasks = Vec::new(); while let Ok(Some(entry)) = readir.next_entry().await { let file = entry.path(); - debug!( - "Loading {}{}", - file.display(), - if key.is_some() { " [verified]" } else { "" } - ); - let reader = std::fs::File::open(&file).context("Failed to open npk")?; - let reader = std::io::BufReader::new(reader); - let npk = crate::npk::npk::Npk::from_reader(reader, key.as_ref()) - .map_err(|e| Error::Npk(file.display().to_string(), e))?; - let name = npk.manifest().name.clone(); - let version = npk.manifest().version.clone(); - let container = Container::new(name, version); + let load_task = task::spawn_blocking(move || { + debug!( + "Loading {}{}", + file.display(), + if key.is_some() { " [verified]" } else { "" } + ); + let reader = std::fs::File::open(&file).context("Failed to open npk")?; + let reader = std::io::BufReader::new(reader); + let npk = NpkNpk::from_reader(reader, key.as_ref()) + .map_err(|e| Error::Npk(file.display().to_string(), e))?; + let name = npk.manifest().name.clone(); + let version = npk.manifest().version.clone(); + let container = Container::new(name, version); + Result::<_, Error>::Ok((container, (file, npk))) + }) + .then(|r| ready(r.expect("Task error"))); + + tasks.push(load_task); + } + + for result in try_join_all(tasks).await? { + let (container, (file, npk)) = result; containers.insert(container, (file, npk)); } let duration = start.elapsed(); info!( - "Loaded {} containers from {} in {:.03}s (avg: {:.05}s)", + "Loaded {} containers from {} in {:.03}s", containers.len(), dir.display(), duration.as_secs_f32(), - duration.as_secs_f32() / containers.len() as f32 ); Ok(DirRepository { @@ -206,7 +216,8 @@ impl<'a> Repository for MemRepository { // Write buffer to the memfd let mut file = unsafe { fs::File::from_raw_fd(fd.as_raw_fd()) }; - file.set_nonblocking(); + file.set_nonblocking(true) + .context("Failed to set nonblocking")?; while let Some(r) = rx.recv().await { file.write_all(&r).await.context("Failed stream npk")?; @@ -225,12 +236,13 @@ impl<'a> Repository for MemRepository { // Forget fd - it's owned by file fd.into_raw_fd(); - file.set_blocking(); + file.set_nonblocking(false) + .context("Failed to set blocking")?; let file = BufReader::new(file.into_std().await); // Load npk debug!("Loading memfd as npk"); - let npk = npk::Npk::from_reader(file, self.key.as_ref()) + let npk = NpkNpk::from_reader(file, self.key.as_ref()) .map_err(|e| Error::Npk("Memory".into(), e))?; let manifest = npk.manifest(); let container = Container::new(manifest.name.clone(), manifest.version.clone()); diff --git a/northstar/src/runtime/state.rs b/northstar/src/runtime/state.rs index b5fbf7331..0aa964e74 100644 --- a/northstar/src/runtime/state.rs +++ b/northstar/src/runtime/state.rs @@ -3,8 +3,9 @@ use super::{ config::{Config, RepositoryType}, console::Request, error::Error, + fork::Forker, + io, mount::MountControl, - process::Launcher, repository::{DirRepository, MemRepository, Npk}, stats::ContainerStats, Container, ContainerEvent, Event, EventTx, ExitStatus, NotificationTx, Pid, RepositoryId, @@ -13,32 +14,41 @@ use crate::{ api::{self, model}, common::non_null_string::NonNullString, npk::manifest::{Autostart, Manifest, Mount, Resource}, - runtime::{error::Context, CGroupEvent, ENV_NAME, ENV_VERSION}, + runtime::{ + console::{Console, Peer}, + io::ContainerIo, + ipc::owned_fd::OwnedFd, + CGroupEvent, ENV_CONTAINER, ENV_NAME, ENV_VERSION, + }, }; -use async_trait::async_trait; use bytes::Bytes; +use derive_new::new; use futures::{ - executor::{ThreadPool, ThreadPoolBuilder}, future::{join_all, ready, Either}, - task::SpawnExt, - Future, FutureExt, TryFutureExt, + Future, FutureExt, Stream, StreamExt, TryFutureExt, }; +use humantime::format_duration; +use itertools::Itertools; use log::{debug, error, info, warn}; use nix::sys::signal::Signal; use std::{ collections::{HashMap, HashSet}, convert::TryFrom, fmt::Debug, - iter::FromIterator, + iter::{once, FromIterator}, + os::unix::net::UnixStream as StdUnixStream, path::PathBuf, result, sync::Arc, }; use tokio::{ + net::UnixStream, + pin, sync::{mpsc, oneshot}, + task::{self, JoinHandle}, time, }; -use Signal::SIGKILL; +use tokio_util::sync::CancellationToken; /// Repository type Repository = Box; @@ -47,23 +57,13 @@ type Args<'a> = Option<&'a Vec>; /// Container environment variables set type Env<'a> = Option<&'a HashMap>; -#[async_trait] -pub(super) trait Process: Send + Sync + Debug { - fn pid(&self) -> Pid; - async fn spawn(&mut self) -> Result<(), Error>; - async fn kill(&mut self, signal: Signal) -> Result<(), Error>; - async fn wait(&mut self) -> Result; - async fn destroy(&mut self) -> Result<(), Error>; -} - #[derive(Debug)] -pub(super) struct State<'a> { - config: &'a Config, +pub(super) struct State { + config: Config, events_tx: EventTx, notification_tx: NotificationTx, mount_control: Arc, - launcher: Launcher, - executor: ThreadPool, + launcher: Forker, containers: HashMap, repositories: HashMap, } @@ -75,7 +75,7 @@ pub(super) struct ContainerState { /// Mount point of the root fs pub root: Option, /// Process information when started - pub process: Option, + pub process: Option, } impl ContainerState { @@ -84,24 +84,25 @@ impl ContainerState { } } -#[derive(Debug)] -pub(super) struct ProcessContext { - process: Box, +#[derive(new, Debug)] +pub(super) struct ContainerContext { + pid: Pid, started: time::Instant, debug: super::debug::Debug, cgroups: cgroups::CGroups, + stop: CancellationToken, + log_task: Option>>, } -impl ProcessContext { - async fn kill(&mut self, signal: Signal) -> Result<(), Error> { - self.process.kill(signal).await - } - +impl ContainerContext { async fn destroy(mut self) { - self.process - .destroy() - .await - .expect("Failed to destroy process"); + // Stop console if there's any any + self.stop.cancel(); + + if let Some(log_task) = self.log_task.take() { + // Wait for the pty to finish + drop(log_task.await); + } self.debug .destroy() @@ -112,13 +113,14 @@ impl ProcessContext { } } -impl<'a> State<'a> { +impl State { /// Create a new empty State instance pub(super) async fn new( - config: &'a Config, + config: Config, events_tx: EventTx, notification_tx: NotificationTx, - ) -> Result, Error> { + forker: Forker, + ) -> Result { let repositories = HashMap::new(); let containers = HashMap::new(); let mount_control = Arc::new( @@ -126,16 +128,6 @@ impl<'a> State<'a> { .await .expect("Failed to initialize mount control"), ); - let launcher = Launcher::start(events_tx.clone(), config.clone(), notification_tx.clone()) - .await - .expect("Failed to start launcher"); - - debug!("Initializing mount thread pool"); - let executor = ThreadPoolBuilder::new() - .name_prefix("northstar-mount-") - .pool_size(config.mount_parallel) - .create() - .expect("Failed to start mount thread pool"); let mut state = State { events_tx, @@ -143,9 +135,8 @@ impl<'a> State<'a> { repositories, containers, config, - launcher, + launcher: forker, mount_control, - executor, }; // Initialize repositories. This populates self.containers and self.repositories @@ -326,13 +317,13 @@ impl<'a> State<'a> { /// Start a container /// `container`: Container to start - /// `args`: Optional command line arguments that overwrite the values from the manifest - /// `env`: Optional env variables that overwrite the values from the manifest + /// `args_extra`: Optional command line arguments that overwrite the values from the manifest + /// `env_extra`: Optional env variables that overwrite the values from the manifest pub(super) async fn start( &mut self, container: &Container, - args: Args<'_>, - env: Env<'_>, + args_extra: Args<'_>, + env_extra: Env<'_>, ) -> Result<(), Error> { let start = time::Instant::now(); info!("Trying to start {}", container); @@ -345,11 +336,12 @@ impl<'a> State<'a> { } // Check optional env variables for reserved ENV_NAME or ENV_VERSION key which cannot be overwritten - if let Some(env) = env { - if env - .keys() - .any(|k| k.as_str() == ENV_NAME || k.as_str() == ENV_VERSION) - { + if let Some(env) = env_extra { + if env.keys().any(|k| { + k.as_str() == ENV_NAME + || k.as_str() == ENV_VERSION + || k.as_str() == "NORTHSTAR_CONSOLE" + }) { return Err(Error::InvalidArguments(format!( "env contains reserved key {} or {}", ENV_NAME, ENV_VERSION @@ -421,65 +413,110 @@ impl<'a> State<'a> { // Get a mutable reference to the container state in order to update the process field let container_state = self.containers.get_mut(container).expect("Internal error"); - // Root of container - let root = container_state - .root - .as_ref() - .map(|root| root.canonicalize().expect("Failed to canonicalize root")) - .unwrap(); - // Spawn process info!("Creating {}", container); - let mut process = match self - .launcher - .create(&root, container, &manifest, args, env) - .await - { - Ok(p) => p, - Err(e) => { - warn!("Failed to create process for {}", container); - return Err(e); - } + // Create a toke to stop tasks spawned related to this container + let stop = CancellationToken::new(); + + // We send the fd to the forker so that it can pass it to the init + let console_fd = if manifest.console { + let peer = Peer::from(container.to_string()); + let (runtime, container) = StdUnixStream::pair().expect("Failed to create socketpair"); + let container: OwnedFd = container.into(); + + let runtime = runtime + .set_nonblocking(true) + .and_then(|_| UnixStream::from_std(runtime)) + .expect("Failed to set socket into nonblocking mode"); + + let notifications = self.notification_tx.subscribe(); + let events_tx = self.events_tx.clone(); + let connection = + Console::connection(runtime, peer, stop.clone(), events_tx, notifications, None); + + // Start console task + task::spawn(connection); + + Some(container) + } else { + None }; - let pid = process.pid(); + // Create container + let config = &self.config; + let pid = self.launcher.create(config, &manifest, console_fd).await?; // Debug - let debug = super::debug::Debug::new(self.config, &manifest, pid).await?; + let debug = super::debug::Debug::new(&self.config, &manifest, pid).await?; // CGroups let cgroups = { debug!("Configuring CGroups for {}", container); let config = manifest.cgroups.clone().unwrap_or_default(); + let events_tx = self.events_tx.clone(); // Creating a cgroup is a northstar internal thing. If it fails it's not recoverable. - cgroups::CGroups::new( - &self.config.cgroup, - self.events_tx.clone(), - container, - &config, - process.pid(), - ) - .await - .expect("Failed to create cgroup") + cgroups::CGroups::new(&self.config.cgroup, events_tx, container, &config, pid) + .await + .expect("Failed to create cgroup") }; + // Open a file handle for stdin, stdout and stderr according to the manifest + let ContainerIo { io, log_task } = io::open(container, &manifest.io) + .await + .expect("IO setup error"); + // Signal the process to continue starting. This can fail because of the container content - if let Err(e) = process.spawn().await { - warn!("Failed to start {} ({}): {}", container, pid, e); + + let path = manifest.init.unwrap(); + let mut args = vec![path.display().to_string()]; + if let Some(extra_args) = args_extra { + args.extend(extra_args.iter().map(ToString::to_string)); + } else if let Some(manifest_args) = manifest.args { + args.extend(manifest_args.iter().map(ToString::to_string)); + }; + + // Prepare the environment for the container according to the manifest + let env = match (env_extra, &manifest.env) { + (Some(env), _) => env.clone(), + (None, Some(env_manifest)) => env_manifest.clone(), + (None, None) => HashMap::with_capacity(3), + }; + let env = env + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .chain(once(format!("{}={}", ENV_CONTAINER, container))) + .chain(once(format!("{}={}", ENV_NAME, container.name()))) + .chain(once(format!("{}={}", ENV_VERSION, container.version()))) + .collect::>(); + + debug!("Container {} init is {:?}", container, path.display()); + debug!("Container {} argv is {}", container, args.iter().join(" ")); + debug!("Container {} env is {}", container, env.iter().join(", ")); + + // Send exec request to launcher + if let Err(e) = self + .launcher + .exec(container.clone(), path, args, env, io) + .await + { + warn!("Failed to exec {} ({}): {}", container, pid, e); + + stop.cancel(); + + if let Some(log_task) = log_task { + drop(log_task.await); + } debug.destroy().await.expect("Failed to destroy debug"); cgroups.destroy().await; return Err(e); } // Add process context to process - container_state.process = Some(ProcessContext { - process: Box::new(process), - started: time::Instant::now(), - debug, - cgroups, - }); + let started = time::Instant::now(); + let context = ContainerContext::new(pid, started, debug, cgroups, stop, log_task); + container_state.process = Some(context); let duration = start.elapsed().as_secs_f32(); info!("Started {} ({}) in {:.03}s", container, pid, duration); @@ -499,16 +536,28 @@ impl<'a> State<'a> { let container_state = self.state_mut(container)?; match &mut container_state.process { - Some(process) => { + Some(context) => { info!("Killing {} with {}", container, signal.as_str()); - process.kill(signal).await + let pid = context.pid; + let process_group = nix::unistd::Pid::from_raw(-(pid as i32)); + match nix::sys::signal::kill(process_group, Some(signal)) { + Ok(_) => Ok(()), + Err(nix::Error::ESRCH) => { + debug!("Process {} already exited", pid); + Ok(()) + } + Err(e) => unimplemented!("Kill error {}", e), + } } None => Err(Error::StopContainerNotStarted(container.clone())), } } /// Shutdown the runtime: stop running applications and umount npks - pub(super) async fn shutdown(mut self) -> Result<(), Error> { + pub(super) async fn shutdown( + mut self, + event_rx: impl Stream, + ) -> Result<(), Error> { let to_umount = self .containers .iter() @@ -516,19 +565,39 @@ impl<'a> State<'a> { .map(|(container, _)| container.clone()) .collect::>(); - for (container, state) in &mut self.containers { - if let Some(mut context) = state.process.take() { - let pid = context.process.pid(); - info!("Sending SIGKILL to {} ({})", container, pid); - nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid as i32), SIGKILL).ok(); - - info!("Waiting for {} to exit", container); - let exit_status = - time::timeout(time::Duration::from_secs(10), context.process.wait()) - .await - .context(format!("Killing {}", container))?; - debug!("Container {} terminated with {:?}", container, exit_status); - context.destroy().await; + let to_kill = self + .containers + .iter() + .filter_map(|(container, state)| state.process.as_ref().map(|_| container.clone())) + .collect::>(); + + for container in &to_kill { + self.kill(container, Signal::SIGKILL).await?; + } + + // Wait until all processes are dead + if self + .containers + .values() + .any(|state| state.process.is_some()) + { + pin!(event_rx); + + loop { + if let Some(Event::Container(container, event)) = event_rx.next().await { + self.on_event(&container, &event, true).await?; + + // Check if all containers exited if this is a exit event + if let ContainerEvent::Exit(_) = event { + if self + .containers + .values() + .all(|state| state.process.is_none()) + { + break; + } + } + } } } @@ -536,8 +605,6 @@ impl<'a> State<'a> { self.umount(&container).await?; } - self.launcher.shutdown().await?; - Ok(()) } @@ -553,32 +620,38 @@ impl<'a> State<'a> { let container = repository.insert(rx).await?; // Check if container is already known and remove newly installed one if so - if let Ok(state) = self.state(&container) { + let already_installed = self + .state(&container) + .ok() + .map(|state| state.repository.clone()); + + if let Some(current_repository) = already_installed { warn!( "Skipping duplicate container {} which is already in repository {}", - container, state.repository + container, current_repository ); + let repository = self .repositories .get_mut(id) .ok_or_else(|| Error::InvalidRepository(id.to_string()))?; repository.remove(&container).await?; - Err(Error::InstallDuplicate(container)) - } else { - // Add the container to the state - self.containers.insert( - container.clone(), - ContainerState { - repository: id.into(), - ..Default::default() - }, - ); - info!("Successfully installed {}", container); + return Err(Error::InstallDuplicate(container)); + } - self.container_event(&container, ContainerEvent::Installed); + // Add the container to the state + self.containers.insert( + container.clone(), + ContainerState { + repository: id.into(), + ..Default::default() + }, + ); + info!("Successfully installed {}", container); - Ok(()) - } + self.container_event(&container, ContainerEvent::Installed); + + Ok(()) } /// Remove and umount a specific app @@ -632,6 +705,7 @@ impl<'a> State<'a> { &mut self, container: &Container, exit_status: &ExitStatus, + is_shutdown: bool, ) -> Result<(), Error> { let autostart = self .manifest(container) @@ -641,16 +715,21 @@ impl<'a> State<'a> { if let Ok(state) = self.state_mut(container) { if let Some(process) = state.process.take() { let is_critical = autostart == Some(Autostart::Critical); + let is_critical = is_critical && !is_shutdown; let duration = process.started.elapsed(); if is_critical { error!( - "Critical process {} exited after {:?} with status {}", - container, duration, exit_status, + "Critical process {} exited after {} with status {}", + container, + format_duration(duration), + exit_status, ); } else { info!( - "Process {} exited after {:?} with status {}", - container, duration, exit_status, + "Process {} exited after {} with status {}", + container, + format_duration(duration), + exit_status, ); } @@ -658,6 +737,8 @@ impl<'a> State<'a> { self.container_event(container, ContainerEvent::Exit(exit_status.clone())); + info!("Container {} exited with status {}", container, exit_status); + // This is a critical flagged container that exited with a error exit code. That's not good... if !exit_status.success() && is_critical { return Err(Error::CriticalContainer( @@ -675,11 +756,12 @@ impl<'a> State<'a> { &mut self, container: &Container, event: &ContainerEvent, + is_shutdown: bool, ) -> Result<(), Error> { match event { ContainerEvent::Started => (), ContainerEvent::Exit(exit_status) => { - self.on_exit(container, exit_status).await?; + self.on_exit(container, exit_status, is_shutdown).await?; } ContainerEvent::Installed => (), ContainerEvent::Uninstalled => (), @@ -695,7 +777,7 @@ impl<'a> State<'a> { pub(super) async fn on_request( &mut self, request: &mut Request, - response_tx: oneshot::Sender, + repsponse: oneshot::Sender, ) -> Result<(), Error> { match request { Request::Message(message) => { @@ -787,7 +869,7 @@ impl<'a> State<'a> { // A error on the response_tx means that the connection // was closed in the meantime. Ignore it. - response_tx.send(response).ok(); + repsponse.send(response).ok(); } else { warn!("Received message is not a request"); } @@ -800,7 +882,7 @@ impl<'a> State<'a> { // A error on the response_tx means that the connection // was closed in the meantime. Ignore it. - response_tx.send(payload).ok(); + repsponse.send(payload).ok(); } } Ok(()) @@ -829,11 +911,6 @@ impl<'a> State<'a> { } } - // Spawn mount tasks onto the executor - let mounts = mounts - .drain(..) - .map(|t| self.executor.spawn_with_handle(t).unwrap()); - // Process mount results let mut result = Vec::new(); for (container, mount_result) in containers.iter().zip(join_all(mounts).await) { @@ -855,7 +932,7 @@ impl<'a> State<'a> { warn!("Mount operation failed after {:.03}s", duration); } else { info!( - "Successfully {} container(s) in {:.03}s", + "Successfully mounted {} container(s) in {:.03}s", result.len(), duration ); @@ -869,7 +946,7 @@ impl<'a> State<'a> { for (container, state) in &self.containers { let manifest = self.manifest(container).expect("Internal error").clone(); let process = state.process.as_ref().map(|context| api::model::Process { - pid: context.process.pid(), + pid: context.pid, uptime: context.started.elapsed().as_nanos() as u64, }); let repository = state.repository.clone(); diff --git a/northstar/src/seccomp/bpf.rs b/northstar/src/seccomp/bpf.rs index 72f269087..b14fd9747 100644 --- a/northstar/src/seccomp/bpf.rs +++ b/northstar/src/seccomp/bpf.rs @@ -9,7 +9,7 @@ use bindings::{ }; use log::trace; use nix::errno::Errno; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::{ collections::{HashMap, HashSet}, mem::size_of, @@ -177,7 +177,7 @@ fn check_platform_requirements() { compile_error!("Seccomp is not supported on Big Endian architectures"); } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq)] pub struct SockFilter { pub code: u16, pub jt: u8, @@ -185,6 +185,32 @@ pub struct SockFilter { pub k: u32, } +impl Serialize for SockFilter { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let a = (self.code as u32) << 16 | (self.jt as u32) << 8 | self.jf as u32; + let value = (a as u64) << 32 | self.k as u64; + serializer.serialize_u64(value) + } +} + +impl<'de> Deserialize<'de> for SockFilter { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = u64::deserialize(deserializer)?; + let a = (value >> 32) as u32; + let code = ((a & 0xFFFF0000) >> 16) as u16; + let jt = ((a & 0xFF00) >> 8) as u8; + let jf = (a & 0xFF) as u8; + let k = (value & 0xFFFFFFFF) as u32; + Ok(SockFilter { code, jt, jf, k }) + } +} + impl From<&SockFilter> for sock_filter { fn from(s: &SockFilter) -> sock_filter { sock_filter { @@ -655,3 +681,24 @@ fn bpf_jump(code: u32, k: u32, jt: u8, jf: u8) -> SockFilter { jf, } } + +#[cfg(test)] +mod test { + use super::SockFilter; + use proptest::prelude::*; + + proptest! { + #[test] + fn sock_filter_serialize_deserialize(a in 0..100, b in 0i32..10) { + let filter = SockFilter { + code: (a + b) as u16, + jt: a as u8, + jf: b as u8, + k: (a * b) as u32, + }; + let serialized = serde_json::to_string(&filter).unwrap(); + let deserialized: SockFilter = serde_json::from_str(&serialized).unwrap(); + prop_assert_eq!(filter, deserialized); + } + } +} diff --git a/northstar/src/util.rs b/northstar/src/util.rs deleted file mode 100644 index 9596ee511..000000000 --- a/northstar/src/util.rs +++ /dev/null @@ -1,40 +0,0 @@ -use nix::{sys::stat, unistd}; -use std::{ - os::unix::prelude::{MetadataExt, PermissionsExt}, - path::{Path, PathBuf}, -}; -use tokio::fs; - -/// Return true if path is read and writeable -pub(crate) async fn is_rw(path: &Path) -> bool { - match fs::metadata(path).await { - Ok(stat) => { - let same_uid = stat.uid() == unistd::getuid().as_raw(); - let same_gid = stat.gid() == unistd::getgid().as_raw(); - let mode = stat::Mode::from_bits_truncate(stat.permissions().mode()); - - let is_readable = (same_uid && mode.contains(stat::Mode::S_IRUSR)) - || (same_gid && mode.contains(stat::Mode::S_IRGRP)) - || mode.contains(stat::Mode::S_IROTH); - let is_writable = (same_uid && mode.contains(stat::Mode::S_IWUSR)) - || (same_gid && mode.contains(stat::Mode::S_IWGRP)) - || mode.contains(stat::Mode::S_IWOTH); - - is_readable && is_writable - } - Err(_) => false, - } -} - -pub(crate) trait PathExt { - fn join_strip>(&self, w: T) -> PathBuf; -} - -impl PathExt for Path { - fn join_strip>(&self, w: T) -> PathBuf { - self.join(match w.as_ref().strip_prefix("/") { - Ok(stripped) => stripped, - Err(_) => w.as_ref(), - }) - } -} diff --git a/tools/nstar/src/main.rs b/tools/nstar/src/main.rs index c5cc3f4f2..93ed43460 100644 --- a/tools/nstar/src/main.rs +++ b/tools/nstar/src/main.rs @@ -23,7 +23,7 @@ use std::{ }; use tokio::{ fs, - io::{copy, AsyncBufReadExt, AsyncRead, AsyncWrite, BufReader}, + io::{copy, AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader}, net::{TcpStream, UnixStream}, time, }; @@ -286,8 +286,8 @@ async fn main() -> Result<()> { clap_complete::generate( shell, - &mut Opt::into_app(), - Opt::into_app().get_name().to_string(), + &mut Opt::command(), + Opt::command().get_name().to_string(), &mut output, ); @@ -319,12 +319,12 @@ async fn main() -> Result<()> { // Subscribe to notifications and print them Subcommand::Notifications { number } => { if opt.json { - let framed = Client::new(io, Some(100), opt.timeout) + let mut framed = Client::new(io, Some(100), opt.timeout) .await .with_context(|| format!("Failed to connect to {}", opt.url))? .framed(); - let mut lines = BufReader::new(framed).lines(); + let mut lines = BufReader::new(framed.get_mut()).lines(); for _ in 0..number.unwrap_or(usize::MAX) { match lines.next_line().await.context("Failed to read stream")? { Some(line) => println!("{}", line), @@ -366,16 +366,21 @@ async fn main() -> Result<()> { // Extra file transfer for install hack if let Subcommand::Install { npk, .. } = command { + framed.flush().await.context("Failed to flush")?; + framed.get_mut().flush().await.context("Failed to flush")?; + copy( &mut fs::File::open(npk).await.context("Failed to open npk")?, - &mut framed, + &mut framed.get_mut(), ) .await .context("Failed to stream npk")?; } + framed.get_mut().flush().await.context("Failed to flush")?; + if opt.json { - let response = BufReader::new(framed) + let response = BufReader::new(framed.get_mut()) .lines() .next_line() .await diff --git a/tools/sextant/src/pack.rs b/tools/sextant/src/pack.rs index 662531fcd..3b98e47f8 100644 --- a/tools/sextant/src/pack.rs +++ b/tools/sextant/src/pack.rs @@ -29,7 +29,7 @@ pub(crate) fn pack( if manifest.init.is_some() { let tmp = tempdir().context("Failed to create temporary directory")?; let name = manifest.name.clone(); - let num = clones.to_string().chars().count() - 1; + let num = clones.to_string().chars().count(); for n in 0..clones { manifest.name = format!("{}-{:0m$}", name, n, m = num) .try_into() diff --git a/tools/stress/Cargo.toml b/tools/stress/Cargo.toml index d635b29a1..6cd135f1f 100644 --- a/tools/stress/Cargo.toml +++ b/tools/stress/Cargo.toml @@ -10,6 +10,8 @@ anyhow = "1.0.54" clap = { version = "3.1.0", features = ["derive"] } env_logger = "0.9.0" futures = "0.3.21" +humantime = "2.1.0" +itertools = "0.10.3" log = "0.4.14" northstar = { path = "../../northstar", features = ["api"], default-features = false } rand = "0.8.5" diff --git a/tools/stress/src/main.rs b/tools/stress/src/main.rs index 2b351e4c8..ecc59f88d 100644 --- a/tools/stress/src/main.rs +++ b/tools/stress/src/main.rs @@ -1,21 +1,25 @@ use anyhow::{anyhow, Context, Result}; use clap::Parser; use futures::{ - future::{self, pending, ready, try_join_all}, + future::{self, pending, ready, try_join_all, Either}, FutureExt, StreamExt, }; +use humantime::parse_duration; +use itertools::Itertools; use log::{debug, info}; use northstar::api::{ client::{self, Client}, - model::{self, ExitStatus, Notification}, + model::{self, Container, ExitStatus, Notification}, }; -use std::{path::PathBuf, str::FromStr}; +use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use tokio::{ io::{AsyncRead, AsyncWrite}, net::{TcpStream, UnixStream}, - pin, select, task, time, + pin, select, + sync::Barrier, + task, time, }; -use tokio_util::{either::Either, sync::CancellationToken}; +use tokio_util::sync::CancellationToken; use url::Url; #[derive(Clone, Debug, PartialEq)] @@ -53,12 +57,16 @@ struct Opt { url: url::Url, /// Duration to run the test for in seconds - #[clap(short, long)] - duration: Option, + #[clap(short, long, parse(try_from_str = parse_duration))] + duration: Option, /// Random delay between each iteration within 0..value ms + #[clap(short, long, parse(try_from_str = parse_duration))] + random: Option, + + /// Single client #[clap(short, long)] - random: Option, + single: bool, /// Mode #[clap(short, long, default_value = "start-stop")] @@ -72,17 +80,13 @@ struct Opt { #[clap(long, required_if_eq("mode", "install-uninstall"))] repository: Option, - /// Relaxed result - #[clap(long)] - relaxed: bool, - /// Initial random delay in ms to randomize tasks #[clap(long)] initial_random_delay: Option, /// Notification timeout in seconds - #[clap(short, long, default_value = "60")] - timeout: u64, + #[clap(short, long, parse(try_from_str = parse_duration), default_value = "60s")] + timeout: Duration, } pub trait N: AsyncRead + AsyncWrite + Send + Unpin {} @@ -122,7 +126,6 @@ async fn main() -> Result<()> { debug!("address: {}", opt.url.to_string()); debug!("repository: {:?}", opt.repository); debug!("npk: {:?}", opt.npk); - debug!("relaxed: {}", opt.relaxed); debug!("random: {:?}", opt.random); debug!("timeout: {:?}", opt.timeout); @@ -140,125 +143,111 @@ async fn main() -> Result<()> { .iter() .filter(|c| c.manifest.init.is_some()) .map(|c| c.container.clone()) + .sorted() .collect::>(); drop(client); let mut tasks = Vec::new(); - let token = CancellationToken::new(); - - // Check random value that cannot be 0 - if let Some(delay) = opt.random { - assert!(delay > 0, "Invalid random value"); - } + let stop = CancellationToken::new(); - // Max string len of all containers - let len = containers - .iter() - .map(ToString::to_string) - .map(|s| s.len()) - .sum::(); - - let start = CancellationToken::new(); - - for container in &containers { - let container = container.clone(); - let initial_random_delay = opt.initial_random_delay; - let mode = opt.mode.clone(); - let random = opt.random; - let relaxed = opt.relaxed; - let start = start.clone(); - let token = token.clone(); - let url = opt.url.clone(); - let timeout = opt.timeout; - - debug!("Spawning task for {}", container); + if opt.single { + let stop = stop.clone(); let task = task::spawn(async move { - start.cancelled().await; - if let Some(initial_delay) = initial_random_delay { - time::sleep(time::Duration::from_millis( - rand::random::() % initial_delay, - )) - .await; - } - - let notifications = if relaxed { None } else { Some(100) }; let mut client = client::Client::new( - io(&url).await?, - notifications, + io(&opt.url).await?, + Some(containers.len() * 3), time::Duration::from_secs(30), ) .await?; + let mut iterations = 0; loop { - if mode == Mode::MountStartStopUmount || mode == Mode::MountUmount { - info!("{:() % delay); - info!("{: ready(r), - Err(e) => ready(Result::::Err(anyhow!("task error: {}", e))), + Err(e) => ready(Result::::Err(anyhow!("task error: {}", e))), }); - tasks.push(task); + + tasks.push(futures::future::Either::Right(task)); + } else { + // Sync the start of all tasks + let start_barrier = Arc::new(Barrier::new(containers.len())); + + for container in &containers { + let container = container.clone(); + let initial_random_delay = opt.initial_random_delay; + let mode = opt.mode.clone(); + let random = opt.random; + let start_barrier = start_barrier.clone(); + let timeout = opt.timeout; + let stop = stop.clone(); + let url = opt.url.clone(); + + debug!("Spawning task for {}", container); + let task = task::spawn(async move { + let mut client = + client::Client::new(io(&url).await?, Some(1000), time::Duration::from_secs(30)) + .await?; + + if let Some(initial_delay) = initial_random_delay { + time::sleep(time::Duration::from_millis( + rand::random::() % initial_delay, + )) + .await; + } + + let mut iterations = 0usize; + + start_barrier.wait().await; + + loop { + iteration(&mode, &container, &mut client, timeout, random).await?; + iterations += 1; + + if stop.is_cancelled() { + break Ok(iterations); + } + } + }) + .then(|r| match r { + Ok(r) => ready(r), + Err(e) => ready(Result::::Err(anyhow!("task error: {}", e))), + }); + + tasks.push(Either::Left(task)); + } } - info!("Starting {} tasks", containers.len()); - start.cancel(); + info!("Starting {} tasks", tasks.len()); let mut tasks = try_join_all(tasks); let ctrl_c = tokio::signal::ctrl_c(); let duration = opt .duration - .map(time::Duration::from_secs) .map(time::sleep) .map(Either::Left) .unwrap_or_else(|| Either::Right(future::pending::<()>())); @@ -266,18 +255,71 @@ async fn main() -> Result<()> { let result = select! { _ = duration => { info!("Stopping because test duration exceeded"); - token.cancel(); + stop.cancel(); tasks.await } _ = ctrl_c => { info!("Stopping because of ctrlc"); - token.cancel(); + stop.cancel(); tasks.await } r = &mut tasks => r, }; - info!("Total iterations: {}", result?.iter().sum::()); + info!("Total iterations: {}", result?.iter().sum::()); + Ok(()) +} + +async fn iteration( + mode: &Mode, + container: &Container, + client: &mut Client>, + timeout: Duration, + random: Option, +) -> Result<()> { + if *mode == Mode::MountStartStopUmount || *mode == Mode::MountUmount { + info!("{} mount", &container); + client.mount(vec![container.clone()]).await?; + } + + if *mode != Mode::MountUmount { + info!("{}: start", container); + client.start(container).await?; + let started = Notification::Started { + container: container.clone(), + }; + await_notification(client, started, timeout).await?; + } + + if let Some(delay) = random { + info!("{}: sleeping for {:?}", container, delay); + time::sleep(delay).await; + } + + if *mode != Mode::MountUmount { + info!("{}: stopping", container); + client + .kill(container.clone(), 15) + .await + .context("Failed to stop container")?; + + info!("{}: waiting for termination", container); + let stopped = Notification::Exit { + container: container.clone(), + status: ExitStatus::Signalled { signal: 15 }, + }; + await_notification(client, stopped, timeout).await?; + } + + // Check if we need to umount + if *mode != Mode::StartStop { + info!("{}: umounting", container); + client + .umount(container.clone()) + .await + .context("Failed to umount")?; + } + Ok(()) } @@ -285,10 +327,8 @@ async fn main() -> Result<()> { async fn await_notification( client: &mut Client, notification: Notification, - duration: u64, + duration: Duration, ) -> Result<()> { - let duration = time::Duration::from_secs(duration); - time::timeout(duration, async { loop { match client.next().await { @@ -300,7 +340,7 @@ async fn await_notification( } }) .await - .context("Failed to wait for notification")? + .with_context(|| format!("Failed to wait for notification: {:?}", notification))? } /// Install and uninstall an npk in a loop @@ -310,7 +350,7 @@ async fn install_uninstall(opt: &Opt) -> Result<()> { let timeout = opt .duration - .map(|d| Either::Left(time::sleep(time::Duration::from_secs(d)))) + .map(|d| Either::Left(time::sleep(d))) .unwrap_or_else(|| Either::Right(pending())); pin!(timeout);