Forker process

The logic for container process setup and launching is moved to a separate process. This process runs in parallel to the runtime and communicates to it via Unix Domain sockets. This enables running the northstar runtime in a multithreaded way without the danger of deadlocking with libc. ┌───────────┐ ┌────────┐ ┌────────────────────────────┐ │ Northstar ├────┤ Forker │ │ Container A │ │ Runtime │ └────┬───┘ │ ┌──────┐ ┌───────────────┐ │ └───────────┘ ├───────┼►│ Init ├─┤ Application A │ │ │ │ └──────┘ └───────────────┘ │ │ └────────────────────────────┘ │ │ ┌────────────────────────────┐ │ │ Container B │ │ │ ┌──────┐ ┌───────────────┐ │ ├───────┼►│ Init ├─┤ Application B │ │ │ │ └──────┘ └───────────────┘ │ │ └────────────────────────────┘ ▼ ... The `Forker` process must consequently be single threaded. Additionally, this patch introduces a small API for the container Init processes that enables the request to start new processes inside the container. This is a prerequisite to #454.
esrlabs · Jan 21, 2022 · 8a02168 · 8a02168
1 parent 92189a7
commit 8a02168
Show file tree

Hide file tree

Showing 59 changed files with 2,745 additions and 2,262 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -85,7 +85,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --all-features
+          args: --all-features -- --test-threads=1
 
   doc:
     name: Documentation

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -264,7 +264,7 @@ kernel configuration with the `CONFIG_` entries in the `check_conf.sh` script.
 
 ### Container launch sequence
 
-**TODO**: <br/><img src="images/container-startup.png" class="inline" width=600/>
+<br/><img src="images/container-startup.png" class="inline" width=600/>
 
 ### Manifest Format
 

diff --git a/doc/diagrams/container_startup.png b/doc/diagrams/container_startup.png
diff --git a/doc/diagrams/container_startup.puml b/doc/diagrams/container_startup.puml
diff --git a/images/container-startup.png b/images/container-startup.png
diff --git a/images/container-startup.puml b/images/container-startup.puml
@@ -1,33 +1,70 @@
 @startuml container_startup
 
+create Client
+activate Client
+
+create Runtime
 activate Runtime
-Runtime -> Runtime: Check and Mount container
+
+create Forker
+Runtime -> Forker: Fork
+activate Forker
+
+Client -> Runtime: Connect: Hello
+Client <- Runtime: ConnectAck
+Client -> Runtime: Start container
+Runtime -> Runtime: Check and mount container(s)
+Runtime -> Runtime: Open PTY
+
+Runtime -> Forker: Create container
+
 create Trampoline
-Runtime -> Trampoline: Fork
+Forker -> Trampoline: Fork
 activate Trampoline
+Trampoline -> Trampoline: Create PID namespace
+
 create Init
 Trampoline -> Init: Fork
 activate Init
-Trampoline -> Runtime: Init PID
+Init -> Init: Mount, Chroot, UID / GID,\ndrop privileges, file descriptors
+
+Trampoline -> Forker: Forked init with PID 
 destroy Trampoline
-Runtime -> Runtime: Wait for Trampoline exit (waitpid)
-Init -> Init: Wait for run signal (Condition::wait)
+
+Forker -> Forker: reap Trampoline
+
+Forker -> Runtime: Created init with PID
+
 Runtime -> Runtime: Configure cgroups
-Runtime -> Init: Signal run (Condition::notify)
-Runtime -> Runtime: Wait for execve (Condition::wait)
-Init -> Init: Mount, Chroot, UID / GID,\ndrop privileges, file descriptors
+Runtime -> Runtime: Configure debug
+Runtime -> Runtime: Configure PTY forward
+
+Runtime -> Forker: Exec container
+Forker -> Init: Exec Container
 create Container
 Init -> Container: Fork
 activate Container
+Forker <- Init: Exec
+Runtime <- Forker: Exec
+Client <- Runtime: Started
+Client <- Runtime: Notification: Started
+
 Init -> Init: Wait for container to exit (waitpid)
+Container -> Container: Setup PTY
 Container -> Container: Set seccomp filter
 Container -> : Execve(..)
-Runtime -> Runtime: Condition pipe closed: Container is started
-note left: Condition pipe is CLOEXEC
-Container -> Init: Exit
+...
+Container -> Init: SIGCHLD
 destroy Container
-Init -> Runtime: Exit
-Runtime -> Runtime: Read exit status from pipe or waitpid on pid of init
+
+Init -> Init: waitpid: Exit status of container
+Init -> Forker: Container exit status
 destroy Init
 
+Forker -> Runtime: Container exit status
+Runtime -> Runtime: Stop PTY thread
+Runtime -> Runtime: Destroy cgroups
+Runtime -> Runtime: Destroy cgroups
+Client <- Runtime: Notification: Exit
+
 @enduml
diff --git a/main/Cargo.toml b/main/Cargo.toml
@@ -15,7 +15,7 @@ log = "0.4.14"
 nix = "0.23.0"
 northstar = { path = "../northstar", features = ["runtime"] }
 structopt = "0.3.25"
-tokio = { version = "1.12.0", features = ["rt", "macros", "signal"] }
+tokio = { version = "1.12.0", features = ["rt-multi-thread", "macros", "signal"] }
 toml = "0.5.8"
 
 [target.'cfg(not(target_os = "android"))'.dependencies]

diff --git a/main/src/logger.rs b/main/src/logger.rs
@@ -9,59 +9,71 @@ pub fn init() {
 }
 
 #[cfg(not(target_os = "android"))]
-static TAG_SIZE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(20);
+static TAG_SIZE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(28);
 
 /// Initialize the logger
 #[cfg(not(target_os = "android"))]
 pub fn init() {
     use env_logger::fmt::Color;
     use std::{io::Write, sync::atomic::Ordering};
 
+    fn color(target: &str) -> Color {
+        // Some colors are hard to read on (at least) dark terminals
+        // and I consider some others as ugly ;-)
+        let hash = target.bytes().fold(42u8, |c, x| c ^ x);
+        Color::Ansi256(match hash {
+            c @ 0..=1 => c + 2,
+            c @ 16..=21 => c + 6,
+            c @ 52..=55 | c @ 126..=129 => c + 4,
+            c @ 163..=165 | c @ 200..=201 => c + 3,
+            c @ 207 => c + 1,
+            c @ 232..=240 => c + 9,
+            c => c,
+        })
+    }
+
     let mut builder = env_logger::Builder::new();
     builder.parse_filters("northstar=debug");
 
     builder.format(|buf, record| {
-        let mut style = buf.style();
+        let timestamp = buf.timestamp_millis().to_string();
+        let timestamp = timestamp.strip_suffix('Z').unwrap();
+
+        let mut level = buf.default_level_style(record.metadata().level());
+        level.set_bold(true);
+        let level = level.value(record.metadata().level().as_str());
 
-        let timestamp = buf.timestamp_millis();
-        let level = buf.default_styled_level(record.metadata().level());
+        let pid = std::process::id().to_string();
+        let mut pid_style = buf.style();
+        pid_style.set_color(color(&pid));
 
-        if let Some(module_path) = record
-            .module_path()
+        if let Some(target) = Option::from(record.target().is_empty())
+            .map(|_| record.target())
+            .or_else(|| record.module_path())
             .and_then(|module_path| module_path.find(&"::").map(|p| &module_path[p + 2..]))
         {
-            TAG_SIZE.fetch_max(module_path.len(), Ordering::SeqCst);
+            let mut tag_style = buf.style();
+            TAG_SIZE.fetch_max(target.len(), Ordering::SeqCst);
             let tag_size = TAG_SIZE.load(Ordering::SeqCst);
-            fn hashed_color(i: &str) -> Color {
-                // Some colors are hard to read on (at least) dark terminals
-                // and I consider some others as ugly ;-)
-                Color::Ansi256(match i.bytes().fold(42u8, |c, x| c ^ x) {
-                    c @ 0..=1 => c + 2,
-                    c @ 16..=21 => c + 6,
-                    c @ 52..=55 | c @ 126..=129 => c + 4,
-                    c @ 163..=165 | c @ 200..=201 => c + 3,
-                    c @ 207 => c + 1,
-                    c @ 232..=240 => c + 9,
-                    c => c,
-                })
-            }
-            style.set_color(hashed_color(module_path));
+            tag_style.set_color(color(target));
 
             writeln!(
                 buf,
-                "{}: {:>s$} {:<5}: {}",
+                "{} {:>s$} {}  {:<5}: {}",
                 timestamp,
-                style.value(module_path),
+                tag_style.value(target),
+                pid_style.value("⬤"),
                 level,
                 record.args(),
-                s = tag_size
+                s = tag_size,
             )
         } else {
             writeln!(
                 buf,
-                "{}: {} {:<5}: {}",
+                "{} {} {}  {:<5}: {}",
                 timestamp,
                 " ".repeat(TAG_SIZE.load(Ordering::SeqCst)),
+                pid_style.value("⬤"),
                 level,
                 record.args(),
             )

diff --git a/main/src/main.rs b/main/src/main.rs
@@ -6,7 +6,7 @@
 use anyhow::{anyhow, Context, Error};
 use log::{debug, info, warn};
 use nix::mount::MsFlags;
-use northstar::runtime;
+use northstar::{runtime, runtime::Runtime as Northstar};
 use runtime::config::Config;
 use std::{
     fs::{self, read_to_string},
@@ -32,16 +32,32 @@ struct Opt {
     pub disable_mount_namespace: bool,
 }
 
-#[tokio::main(flavor = "current_thread")]
-async fn main() -> Result<(), Error> {
+fn main() -> Result<(), Error> {
+    // Initialize logging
+    logger::init();
+
+    // Parse command line arguments and prepare the environment
+    let config = init()?;
+
+    // Create the runtime launcher. This must be done *before* spawning the tokio threadpool.
+    let northstar = Northstar::new(config)?;
+
+    tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_all()
+        .thread_name("northstar")
+        .build()
+        .context("Failed to create runtime")?
+        .block_on(run(northstar))
+}
+
+fn init() -> Result<Config, Error> {
     let opt = Opt::from_args();
     let config = read_to_string(&opt.config)
         .with_context(|| format!("Failed to read configuration file {}", opt.config.display()))?;
     let config: Config = toml::from_str(&config)
         .with_context(|| format!("Failed to read configuration file {}", opt.config.display()))?;
 
-    logger::init();
-
     fs::create_dir_all(&config.run_dir).context("Failed to create run_dir")?;
     fs::create_dir_all(&config.data_dir).context("Failed to create data_dir")?;
     fs::create_dir_all(&config.log_dir).context("Failed to create log dir")?;
@@ -64,9 +80,15 @@ async fn main() -> Result<(), Error> {
         debug!("Mount namespace is disabled");
     }
 
-    let mut runtime = runtime::Runtime::start(config)
+    Ok(config)
+}
+
+async fn run(northstar: Northstar) -> Result<(), Error> {
+    let mut runtime = northstar
+        .start()
         .await
-        .context("Failed to start runtime")?;
+        .context("Failed to start Northstar")?;
+
     let mut sigint = tokio::signal::unix::signal(SignalKind::interrupt())
         .context("Failed to install sigint handler")?;
     let mut sigterm = tokio::signal::unix::signal(SignalKind::terminate())
@@ -87,7 +109,7 @@ async fn main() -> Result<(), Error> {
             info!("Received SIGHUP. Stopping Northstar runtime");
             runtime.shutdown().await
         }
-        status = &mut runtime => status,
+        status = runtime.stopped() => status,
     };
 
     match status {

diff --git a/northstar-tests/Cargo.toml b/northstar-tests/Cargo.toml
@@ -10,6 +10,7 @@ env_logger = "0.9.0"
 futures = "0.3.17"
 lazy_static = "1.4.0"
 log = "0.4.14"
+nanoid = "0.4.0"
 nix = "0.23.0"
 northstar = { path = "../northstar", features = ["api", "runtime"] }
 regex = "1.5.4"