Add support for bootstrap containers via API settings

README.md

@@ -478,6 +478,46 @@ We use it for the control container because it needs to be available early to gi
 
 Be careful, and make sure you have a similar low-level use case before reaching for host containers.
 
+#### Bootstrap containers settings
+* `settings.bootstrap-containers.<name>.source`: the image for the container
+* `settings.bootstrap-containers.<name>.mode`: the mode of the container, it could be one of `off`, `once` or `always`. See below for a description of modes.
+* `settings.bootstrap-containers.<name>.essential`: whether or not the container should fail the boot process, defaults to `false`
+* `settings.bootstrap-containers.<name>.user-data`: field with arbitrary base64-encoded data
+
+Bootstrap containers are host containers that can be used to "bootstrap" the host before services like ECS Agent, Kubernetes, and Docker start.
+
+Bootstrap containers are very similar to normal host containers; they come with persistent storage and with optional user data.
+Unlike normal host containers, bootstrap containers can't be treated as `superpowered` containers.
+However, these containers have access to the underlying root filesystem on `/.bottlerocket/rootfs`.
+Bootstrap containers are set up to run after the systemd `configured.target` unit is active.
+The containers' systemd unit depends on this target (and not on any of the bootstrap containers' peers) which means that bootstrap containers will not execute in a deterministic order
+The boot process will "wait" for as long as the bootstrap containers run.
+Bootstrap containers configured with `essential=true` will stop the boot process if they exit code is a non-zero value.
+
+Bootstrap containers have three different modes:
+
+* `always`: with this setting, the container is executed on every boot.
+* `off`: the container won't run
+* `once`: with this setting, the container only runs on the first boot where the container is defined. Upon completion, the mode is changed to `off`.
+
+Here's an example of adding a bootstrap container with API calls:
+
+```
+apiclient set \
+   bootstrap-containers.bootstrap.source=MY-CONTAINER-URI \
+   bootstrap-containers.bootstrap.mode=once \
+   bootstrap-containers.bootstrap.essential=true
+```
+
+Here's the same example, but with the settings you'd add to user data:
+
+```
+[settings.bootstrap-containers.bootstrap]
+source = "MY-CONTAINER-URI"
+mode = "once"
+essential = true
+```
+
 #### Platform-specific settings
 
 Platform-specific settings are automatically set at boot time by [early-boot-config](sources/api/early-boot-config) based on metadata available on the running platform.

Release.toml

@@ -1,4 +1,4 @@
-version = "1.0.7"
+version = "1.0.8"
 
 [migrations]
 "(0.3.1, 0.3.2)" = ["migrate_v0.3.2_admin-container-v0-5-0.lz4"]
@@ -36,4 +36,5 @@ version = "1.0.7"
     "migrate_v1.0.8_proxy-affect-host-containers.lz4",
     "migrate_v1.0.8_control-container-v0-5-0.lz4",
     "migrate_v1.0.8_admin-container-v0-7-0.lz4",
+    "migrate_v1.0.8_add-bootstrap-containers.lz4"
 ]

packages/os/bootstrap-containers-tmpfiles.conf

@@ -0,0 +1,3 @@
+d /etc/bootstrap-containers 0750 root root -
+d /local/bootstrap-containers 0700 root root -
+T /local/bootstrap-containers - - - - security.selinux=system_u:object_r:state_t:s0

packages/os/bootstrap-containers@.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=bootstrap container %i
+Before=configured.target
+After=host-containerd.service
+Wants=host-containers.service
+# Block manual interactions with bootstrap containers, since they should only be
+# started by systemd
+RefuseManualStart=true
+RefuseManualStop=true
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/bootstrap-containers/%i.env
+ExecStart=/usr/bin/host-ctr run \
+    --container-id='%i' \
+    --source='${CTR_SOURCE}' \
+    --container-type='bootstrap'
+ExecStartPost=/usr/bin/bootstrap-containers mark-bootstrap \
+    --container-id '%i' \
+    --mode '${CTR_MODE}'
+RemainAfterExit=true
+StandardError=journal+console

packages/os/host-containers-tmpfiles.conf

@@ -1,3 +1,3 @@
-d /etc/host-containers 0755 root root -
+d /etc/host-containers 0750 root root -
 d /local/host-containers 0700 root root -
 T /local/host-containers - - - - security.selinux=system_u:object_r:state_t:s0

packages/os/os.spec

@@ -35,11 +35,13 @@ Source110: mark-successful-boot.service
 Source111: metricdog.service
 Source112: metricdog.timer
 Source113: send-boot-success.service
+Source114: [email protected]
 
 # 2xx sources: tmpfilesd configs
 Source200: migration-tmpfiles.conf
 Source201: host-containers-tmpfiles.conf
 Source202: thar-be-updates-tmpfiles.conf
+Source203: bootstrap-containers-tmpfiles.conf
 
 # 3xx sources: udev rules
 Source300: ephemeral-storage.rules
@@ -200,6 +202,12 @@ Requires: %{_cross_os}apiserver = %{version}-%{release}
 %{summary}.
 %endif
 
+%package -n %{_cross_os}bootstrap-containers
+Summary: Manages bootstrap-containers
+Requires: %{_cross_os}apiserver = %{version}-%{release}
+%description -n %{_cross_os}bootstrap-containers
+%{summary}.
+
 %prep
 %setup -T -c
 %cargo_prep
@@ -257,6 +265,7 @@ echo "** Output from non-static builds:"
     -p ghostdog \
     -p growpart \
     -p corndog \
+    -p bootstrap-containers \
 %if "%{_cross_variant}" == "aws-ecs-1"
     -p ecs-settings-applier \
 %endif
@@ -283,7 +292,7 @@ for p in \
   storewolf settings-committer \
   migrator \
   signpost updog metricdog logdog \
-  ghostdog \
+  ghostdog bootstrap-containers \
 %if "%{_cross_variant}" == "aws-ecs-1"
   ecs-settings-applier \
 %endif
@@ -338,13 +347,15 @@ install -p -m 0644 %{S:5} %{S:6} %{buildroot}%{_cross_templatedir}
 install -d %{buildroot}%{_cross_unitdir}
 install -p -m 0644 \
   %{S:100} %{S:101} %{S:102} %{S:103} %{S:105} \
-  %{S:106} %{S:107} %{S:110} %{S:111} %{S:112} %{S:113}\
+  %{S:106} %{S:107} %{S:110} %{S:111} %{S:112} \
+  %{S:113} %{S:114} \
   %{buildroot}%{_cross_unitdir}
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:200} %{buildroot}%{_cross_tmpfilesdir}/migration.conf
 install -p -m 0644 %{S:201} %{buildroot}%{_cross_tmpfilesdir}/host-containers.conf
 install -p -m 0644 %{S:202} %{buildroot}%{_cross_tmpfilesdir}/thar-be-updates.conf
+install -p -m 0644 %{S:203} %{buildroot}%{_cross_tmpfilesdir}/bootstrap-containers.conf
 
 install -d %{buildroot}%{_cross_udevrulesdir}
 install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-storage.rules
@@ -461,4 +472,9 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor
 %{_cross_bindir}/static-pods
 %endif
 
+%files -n %{_cross_os}bootstrap-containers
+%{_cross_bindir}/bootstrap-containers
+%{_cross_unitdir}/[email protected]
+%{_cross_tmpfilesdir}/bootstrap-containers.conf
+
 %changelog

packages/release/release.spec

@@ -81,6 +81,7 @@ Requires: %{_cross_os}logdog
 Requires: %{_cross_os}util-linux
 Requires: %{_cross_os}wicked
 Requires: %{_cross_os}os
+Requires: %{_cross_os}bootstrap-containers
 
 %description
 %{summary}.

sources/Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "api/apiserver",
     "api/apiclient",
+    "api/bootstrap-containers",
     "api/bork",
     "api/corndog",
     "api/datastore",
@@ -51,6 +52,7 @@ members = [
     "api/migration/migrations/v1.0.8/proxy-affect-host-containers",
     "api/migration/migrations/v1.0.8/control-container-v0-5-0",
     "api/migration/migrations/v1.0.8/admin-container-v0-7-0",
+    "api/migration/migrations/v1.0.8/add-bootstrap-containers",
 
     "bottlerocket-release",
 

sources/api/bootstrap-containers/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "bootstrap-containers"
+version = "0.1.0"
+authors = ["Arnaldo Garcia Rincon <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+build = "build.rs"
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+apiclient = { path = "../apiclient" }
+datastore = { path = "../datastore" }
+base64 = "0.13"
+http = "0.2"
+log = "0.4"
+models = { path = "../../models" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+simplelog = "0.10"
+snafu = "0.6"
+tokio = { version = "0.2", default-features = false, features = ["macros", "rt-threaded"] }
+# When hyper updates to tokio 0.3:
+#tokio = { version = "1", default-features = false, features = ["macros", "rt-multi-thread"] }
+
+[build-dependencies]
+cargo-readme = "3.1"

sources/api/bootstrap-containers/README.md

@@ -0,0 +1,77 @@
+# bootstrap-containers
+
+Current version: 0.1.0
+
+## Bootstrap containers
+
+bootstrap-containers ensures that bootstrap containers are executed as defined in the system settings
+
+It queries the API for their settings, then configures the system by:
+
+* creating a user-data file in the host container's persistent storage area, if a base64-encoded
+  user-data setting is set for the host container.  (The decoded contents are available to the
+  container at /.bottlerocket/bootstrap-containers/<name>/user-data)
+* creating an environment file used by a bootstrap-container-specific instance of a systemd service
+* creating a systemd drop-in configureation file used by a bootstrap-container-specific
+instance of a systemd service
+* ensuring that the bootstap container's systemd service is enabled/disabled for the next boot
+
+## Examples
+Given a bootstrap container called `bear` with the following configuration:
+
+```toml
+[settings.bootstrap-containers.bear]
+source="<SOURCE>"
+mode="once"
+user-data="ypXCt82h4bSlwrfKlA=="
+```
+
+Where `<SOURCE>`, is the url of an image with the following definition:
+
+```Dockerfile
+FROM alpine
+ADD bootstrap-script /
+RUN chmod +x /bootstrap-script
+ENTRYPOINT ["sh", "bootstrap-script"]
+```
+
+And `bootstrap-script` as:
+
+```sh
+#!/usr/bin/env sh
+# We'll read some data to be written out from given user-data.
+USER_DATA_DIR=/.bottlerocket/bootstrap-containers/current
+# This is the in-container view of where the host's `/var` can be accessed.
+HOST_VAR_DIR=/.bottlerocket/rootfs/var
+# The directory that'll be created by this bootstrap container
+MY_HOST_DIR=$VAR_DIR/lib/my_directory
+# Create it!
+mkdir -p "$MY_HOST_DIR"
+# Write the user-data to stdout (to the journal) and to our new path:
+tee /dev/stdout "$MY_HOST_DIR/bear.txt" < "$USER_DATA_DIR/user-data"
+# The bootstrap container can set the permissions which are seen by the host:
+chmod -R o+r "$MY_HOST_DIR"
+chown -R 1000:1000 "$MY_HOST_DIR"
+# Bootstrap containers *must* finish before boot continues.
+#
+# With this, the boot process will be delayed 120 seconds. You can check the
+# status of `preconfigured.target` and `bootstrap-containers@bear` to see
+# that this sleep kept the system from starting up the apiserver.
+#
+# From the admin container:
+#
+# systemctl status preconfigured.target bootstrap-containers@bear
+sleep 120
+```
+
+You should see a new directory under `/var/lib` called `my_directory`, a file in that
+directory called `bear.txt` and the following command should show `ʕ·͡ᴥ·ʔ` in the bootstrap
+containers logs:
+
+```sh
+journalctl -u [email protected]
+```
+
+## Colophon
+
+This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`.

sources/api/bootstrap-containers/README.tpl

@@ -0,0 +1,9 @@
+# {{crate}}
+
+Current version: {{version}}
+
+{{readme}}
+
+## Colophon
+
+This text was generated using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`.

sources/api/bootstrap-containers/build.rs

@@ -0,0 +1,32 @@
+// Automatically generate README.md from rustdoc.
+
+use std::env;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+fn main() {
+    // Check for environment variable "SKIP_README". If it is set,
+    // skip README generation
+    if env::var_os("SKIP_README").is_some() {
+        return;
+    }
+
+    let mut source = File::open("src/main.rs").unwrap();
+    let mut template = File::open("README.tpl").unwrap();
+
+    let content = cargo_readme::generate_readme(
+        &PathBuf::from("."), // root
+        &mut source,         // source
+        Some(&mut template), // template
+        // The "add x" arguments don't apply when using a template.
+        true,  // add title
+        false, // add badges
+        false, // add license
+        true,  // indent headings
+    )
+    .unwrap();
+
+    let mut readme = File::create("README.md").unwrap();
+    readme.write_all(content.as_bytes()).unwrap();
+}