buildsys: add repack-variant target

Add the the initial target for repackaging a Bottlerocket variant. This
target will skip the traditional workflow of building packages and
using `rpm2img`. Instead the build system will point to a yet to be
implemented `img2img` tool.

Signed-off-by: Patrick J.P. Culp <[email protected]>

docs/design/README.md

@@ -278,7 +278,7 @@ These will not be available and need to be added to the Bottlerocket SDK:
 The existing build system relies on certain binaries, scripts and static files, such as
 
 - buildsys, pubsys, testsys, etc.
-- rpm2img, the Dockerfile, and Makefile.toml
+- rpm2img, the Dockerfiles, and Makefile.toml
 
 These tools and files will be embedded into the Twoliter binary and  installed on-the-fly when Twoliter runs.
 The binaries will be built and embedded using the unstable Cargo feature [bindeps].

tools/buildsys/src/args.rs

@@ -41,13 +41,15 @@ pub(crate) struct Buildsys {
 pub(crate) enum Command {
     BuildPackage(Box<BuildPackageArgs>),
     BuildVariant(Box<BuildVariantArgs>),
+    RepackVariant(Box<RepackVariantArgs>),
 }
 
 impl Command {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             Command::BuildPackage(_) => BuildType::Package,
             Command::BuildVariant(_) => BuildType::Variant,
+            Command::RepackVariant(_) => BuildType::Repack,
         }
     }
 }
@@ -160,6 +162,25 @@ pub(crate) struct BuildVariantArgs {
     pub(crate) common: Common,
 }
 
+/// Repack variant from prebuilt images.
+#[derive(Debug, Parser)]
+pub(crate) struct RepackVariantArgs {
+    #[arg(long, env = "BUILDSYS_NAME")]
+    pub(crate) name: String,
+
+    #[arg(long, env = "BUILDSYS_VARIANT")]
+    pub(crate) variant: String,
+
+    #[arg(long, env = "BUILDSYS_VERSION_BUILD")]
+    pub(crate) version_build: String,
+
+    #[arg(long, env = "BUILDSYS_VERSION_IMAGE")]
+    pub(crate) version_image: String,
+
+    #[command(flatten)]
+    pub(crate) common: Common,
+}
+
 /// Returns the environment variables that need to be watched for a given `[BuildType]`.
 fn sensitive_env_vars(build_type: BuildFlags) -> impl Iterator<Item = &'static str> {
     REBUILD_VARS
@@ -186,11 +207,13 @@ enum BuildFlags {
     Package = 0b00000001,
     Kit = 0b00000010,
     Variant = 0b00000100,
+    Repack = 0b0001000,
 }
 
 impl From<BuildType> for BuildFlags {
     fn from(value: BuildType) -> Self {
         match value {
+            BuildType::Repack => BuildFlags::Repack,
             BuildType::Package => BuildFlags::Package,
             BuildType::Kit => BuildFlags::Kit,
             BuildType::Variant => BuildFlags::Variant,
@@ -212,11 +235,14 @@ const VARIANT: u8 = BuildFlags::Variant as u8;
 #[test]
 fn build_type_includes_test() {
     // true
+    assert!(BuildFlags::Repack.includes(8));
     assert!(BuildFlags::Package.includes(PACKAGE | VARIANT));
     assert!(BuildFlags::Variant.includes(VARIANT));
     assert!(BuildFlags::Variant.includes(VARIANT | PACKAGE));
 
     // false
+    assert!(!BuildFlags::Repack.includes(PACKAGE));
+    assert!(!BuildFlags::Repack.includes(VARIANT));
     assert!(!BuildFlags::Package.includes(VARIANT));
     assert!(!BuildFlags::Variant.includes(PACKAGE));
     assert!(!BuildFlags::Variant.includes(32));

tools/buildsys/src/builder.rs

@@ -6,7 +6,7 @@ the repository's top-level Dockerfile.
 */
 pub(crate) mod error;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs};
+use crate::args::{BuildPackageArgs, BuildVariantArgs, RepackVariantArgs};
 use buildsys::manifest::{
     ImageFeature, ImageFormat, ImageLayout, Manifest, PartitionPlan, SupportedArch,
 };
@@ -86,7 +86,6 @@ lazy_static! {
 
 static DOCKER_BUILD_MAX_ATTEMPTS: NonZeroU16 = nonzero!(10u16);
 
-#[allow(dead_code)]
 enum OutputCleanup {
     BeforeBuild,
     None,
@@ -224,17 +223,60 @@ impl VariantBuildArgs {
     }
 }
 
+struct VariantRepackArgs {
+    data_image_publish_size_gib: i32,
+    data_image_size_gib: String,
+    image_features: HashSet<ImageFeature>,
+    image_format: String,
+    name: String,
+    os_image_publish_size_gib: String,
+    os_image_size_gib: String,
+    partition_plan: String,
+    variant: String,
+    version_build: String,
+    version_image: String,
+}
+
+impl VariantRepackArgs {
+    fn build_args(&self) -> Vec<String> {
+        let mut args = Vec::new();
+        args.push("--network".into());
+        args.push("host".into());
+        args.build_arg(
+            "DATA_IMAGE_PUBLISH_SIZE_GIB",
+            self.data_image_publish_size_gib.to_string(),
+        );
+        args.build_arg("DATA_IMAGE_SIZE_GIB", &self.data_image_size_gib);
+        args.build_arg("IMAGE_FORMAT", &self.image_format);
+        args.build_arg("IMAGE_NAME", &self.name);
+        args.build_arg("OS_IMAGE_PUBLISH_SIZE_GIB", &self.os_image_publish_size_gib);
+        args.build_arg("OS_IMAGE_SIZE_GIB", &self.os_image_size_gib);
+        args.build_arg("PARTITION_PLAN", &self.partition_plan);
+        args.build_arg("VARIANT", &self.variant);
+        args.build_arg("BUILD_ID", &self.version_build);
+        args.build_arg("VERSION_ID", &self.version_image);
+
+        for image_feature in self.image_features.iter() {
+            args.build_arg(format!("{}", image_feature), "1");
+        }
+
+        args
+    }
+}
+
 #[allow(clippy::large_enum_variant)]
 enum TargetBuildArgs {
     Package(PackageBuildArgs),
     Variant(VariantBuildArgs),
+    Repack(VariantRepackArgs),
 }
 
 impl TargetBuildArgs {
     pub(crate) fn build_type(&self) -> BuildType {
         match self {
             TargetBuildArgs::Package(_) => BuildType::Package,
             TargetBuildArgs::Variant(_) => BuildType::Variant,
+            TargetBuildArgs::Repack(_) => BuildType::Repack,
         }
     }
 }
@@ -384,6 +426,67 @@ impl DockerBuild {
         })
     }
 
+    /// Create a new `DockerBuild` that can repackage a variant image.
+    pub(crate) fn repack_variant(args: RepackVariantArgs, manifest: &Manifest) -> Result<Self> {
+        let image_layout = manifest.info().image_layout().cloned().unwrap_or_default();
+        let ImageLayout {
+            os_image_size_gib,
+            data_image_size_gib,
+            partition_plan,
+            ..
+        } = image_layout;
+
+        let (os_image_publish_size_gib, data_image_publish_size_gib) =
+            image_layout.publish_image_sizes_gib();
+
+        Ok(Self {
+            dockerfile: args.common.tools_dir.join("repack.Dockerfile"),
+            context: args.common.root_dir.clone(),
+            target: "repack".to_string(),
+            tag: append_token(
+                format!(
+                    "buildsys-repack-{variant}-{arch}",
+                    variant = args.variant,
+                    arch = args.common.arch
+                ),
+                &args.common.root_dir,
+            ),
+            root_dir: args.common.root_dir.clone(),
+            artifacts_dir: args.common.image_arch_variant_dir,
+            state_dir: args.common.state_dir,
+            artifact_name: args.variant.clone(),
+            common_build_args: CommonBuildArgs::new(
+                &args.common.root_dir,
+                args.common.sdk_image,
+                args.common.arch,
+                OutputCleanup::None,
+            ),
+            target_build_args: TargetBuildArgs::Repack(VariantRepackArgs {
+                data_image_publish_size_gib,
+                data_image_size_gib: data_image_size_gib.to_string(),
+                image_features: manifest.info().image_features().unwrap_or_default(),
+                image_format: match manifest.info().image_format() {
+                    Some(ImageFormat::Raw) | None => "raw",
+                    Some(ImageFormat::Qcow2) => "qcow2",
+                    Some(ImageFormat::Vmdk) => "vmdk",
+                }
+                .to_string(),
+                name: args.name,
+                os_image_publish_size_gib: os_image_publish_size_gib.to_string(),
+                os_image_size_gib: os_image_size_gib.to_string(),
+                partition_plan: match partition_plan {
+                    PartitionPlan::Split => "split",
+                    PartitionPlan::Unified => "unified",
+                }
+                .to_string(),
+                variant: args.variant,
+                version_build: args.version_build,
+                version_image: args.version_image,
+            }),
+            secrets_args: secrets_args()?,
+        })
+    }
+
     pub(crate) fn build(&self) -> Result<()> {
         env::set_current_dir(&self.root_dir).context(error::DirectoryChangeSnafu {
             path: &self.root_dir,
@@ -468,6 +571,7 @@ impl DockerBuild {
         let mut args = match &self.target_build_args {
             TargetBuildArgs::Package(p) => p.build_args(),
             TargetBuildArgs::Variant(v) => v.build_args(),
+            TargetBuildArgs::Repack(r) => r.build_args(),
         };
         args.build_arg("ARCH", self.common_build_args.arch.to_string());
         args.build_arg("GOARCH", self.common_build_args.arch.goarch());
@@ -571,6 +675,7 @@ fn create_marker_dir(
         BuildType::Package => "packages",
         BuildType::Kit => "kits",
         BuildType::Variant => "variants",
+        BuildType::Repack => "variants",
     };
 
     let path = [&state_dir.display().to_string(), arch, prefix, name]

tools/buildsys/src/lib.rs

@@ -6,4 +6,5 @@ pub enum BuildType {
     Package,
     Kit,
     Variant,
+    Repack,
 }

tools/buildsys/src/main.rs

@@ -15,7 +15,7 @@ mod gomod;
 mod project;
 mod spec;
 
-use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command};
+use crate::args::{BuildPackageArgs, BuildVariantArgs, Buildsys, Command, RepackVariantArgs};
 use crate::builder::DockerBuild;
 use buildsys::manifest::{BundleModule, Manifest, ManifestInfo, SupportedArch};
 use cache::LookasideCache;
@@ -103,6 +103,7 @@ fn run(args: Buildsys) -> Result<()> {
     match args.command {
         Command::BuildPackage(args) => build_package(*args),
         Command::BuildVariant(args) => build_variant(*args),
+        Command::RepackVariant(args) => repack_variant(*args),
     }
 }
 
@@ -256,6 +257,23 @@ fn build_variant(args: BuildVariantArgs) -> Result<()> {
         .context(error::BuildAttemptSnafu)
 }
 
+fn repack_variant(args: RepackVariantArgs) -> Result<()> {
+    let manifest_file = "Cargo.toml";
+
+    let manifest = Manifest::new(
+        args.common.cargo_manifest_dir.join(manifest_file),
+        &args.common.cargo_metadata_path,
+    )
+    .context(error::ManifestParseSnafu)?;
+
+    supported_arch(manifest.info(), args.common.arch)?;
+
+    DockerBuild::repack_variant(args, &manifest)
+        .context(error::BuilderInstantiationSnafu)?
+        .build()
+        .context(error::BuildAttemptSnafu)
+}
+
 /// Ensure that the current arch is supported by the current variant
 fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> {
     if let Some(supported_arches) = manifest.supported_arches() {

tools/buildsys/src/manifest.rs

@@ -504,6 +504,7 @@ fn is_manifest_type(pkg_metadata: &PackageMetadata, manifest_type: BuildType) ->
         BuildType::Package => metadata_table.get("build-package").is_some(),
         BuildType::Kit => metadata_table.get("build-kit").is_some(),
         BuildType::Variant => metadata_table.get("build-variant").is_some(),
+        BuildType::Repack => metadata_table.get("build-variant").is_some(),
     }
 }
 

twoliter/build.rs

@@ -30,8 +30,11 @@ fn main() {
     paths.copy_file("build.Dockerfile");
     paths.copy_file("build.Dockerfile.dockerignore");
     paths.copy_file("docker-go");
+    paths.copy_file("img2img");
     paths.copy_file("imghelper");
     paths.copy_file("partyplanner");
+    paths.copy_file("repack.Dockerfile");
+    paths.copy_file("repack.Dockerfile.dockerignore");
     paths.copy_file("rpm2img");
     paths.copy_file("rpm2kmodkit");
     paths.copy_file("rpm2migrations");

twoliter/embedded/Makefile.toml

@@ -811,6 +811,20 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${BUILDSYS_OUTPUT_DIR}/latest"
 '''
 ]
 
+[tasks.repack-variant]
+dependencies = ["fetch-sdk", "build-sbkeys", "publish-setup"]
+script = [
+'''
+export PATH="${TWOLITER_TOOLS_DIR}:${PATH}"
+
+# Parse the variant into its components and set additional variables.
+eval "$(bottlerocket-variant)"
+
+buildsys repack-variant \
+  --cargo-manifest-dir variants/${BUILDSYS_VARIANT}
+'''
+]
+
 [tasks.check-licenses]
 dependencies = ["fetch"]
 script = [

twoliter/embedded/img2img

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "img2img not yet implemented" >&2 ; exit 1

twoliter/embedded/repack.Dockerfile

@@ -0,0 +1,58 @@
+# syntax=docker/dockerfile:1.4.3
+#
+# Several commands start with RUN --mount=target=/host, which mounts the docker build
+# context (which in practice is the root of the Bottlerocket repository) as a read-only
+# filesystem at /host.
+ARG SDK
+
+FROM ${SDK} as imgrepack
+
+ARG ARCH
+ARG VERSION_ID
+ARG BUILD_ID
+ARG NOCACHE
+ARG VARIANT
+ARG IMAGE_NAME
+ARG IMAGE_FORMAT
+ARG OS_IMAGE_SIZE_GIB
+ARG DATA_IMAGE_SIZE_GIB
+ARG PARTITION_PLAN
+ARG OS_IMAGE_PUBLISH_SIZE_GIB
+ARG DATA_IMAGE_PUBLISH_SIZE_GIB
+ARG UEFI_SECURE_BOOT
+ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID}
+WORKDIR /root
+
+USER root
+RUN --mount=target=/host \
+    --mount=type=secret,id=PK.crt,target=/root/sbkeys/PK.crt \
+    --mount=type=secret,id=KEK.crt,target=/root/sbkeys/KEK.crt \
+    --mount=type=secret,id=db.crt,target=/root/sbkeys/db.crt \
+    --mount=type=secret,id=vendor.crt,target=/root/sbkeys/vendor.crt \
+    --mount=type=secret,id=shim-sign.key,target=/root/sbkeys/shim-sign.key \
+    --mount=type=secret,id=shim-sign.crt,target=/root/sbkeys/shim-sign.crt \
+    --mount=type=secret,id=code-sign.key,target=/root/sbkeys/code-sign.key \
+    --mount=type=secret,id=code-sign.crt,target=/root/sbkeys/code-sign.crt \
+    --mount=type=secret,id=config-sign.key,target=/root/sbkeys/config-sign.key \
+    --mount=type=secret,id=kms-sign.json,target=/root/.config/aws-kms-pkcs11/config.json \
+    --mount=type=secret,id=aws-access-key-id.env,target=/root/.aws/aws-access-key-id.env \
+    --mount=type=secret,id=aws-secret-access-key.env,target=/root/.aws/aws-secret-access-key.env \
+    --mount=type=secret,id=aws-session-token.env,target=/root/.aws/aws-session-token.env \
+    /host/build/tools/img2img \
+      --input-dir="/host/build/images/${ARCH}-${VARIANT}/${VERSION_ID}-${BUILD_ID}" \
+      --output-dir=/local/output \
+      --output-fmt="${IMAGE_FORMAT}" \
+      --os-image-size-gib="${OS_IMAGE_SIZE_GIB}" \
+      --data-image-size-gib="${DATA_IMAGE_SIZE_GIB}" \
+      --os-image-publish-size-gib="${OS_IMAGE_PUBLISH_SIZE_GIB}" \
+      --data-image-publish-size-gib="${DATA_IMAGE_PUBLISH_SIZE_GIB}" \
+      --partition-plan="${PARTITION_PLAN}" \
+      --ovf-template="/host/variants/${VARIANT}/template.ovf" \
+      ${UEFI_SECURE_BOOT:+--with-uefi-secure-boot=yes} \
+    && echo ${NOCACHE}
+
+# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
+# Copies the repackaged artifacts to their expected location so that buildsys can find them
+# and copy them out.
+FROM scratch AS repack
+COPY --from=imgrepack /local/output/. /output/

twoliter/embedded/repack.Dockerfile.dockerignore

@@ -0,0 +1,4 @@
+/
+!/build/images/*
+!/build/tools/*
+!/variants/*/template*.ovf