From a04c3e90d5b2e1e405a63e6334a2159659cbc21a Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 25 Jul 2024 14:32:12 +0300 Subject: [PATCH 01/12] refactor: create separation between ostree / vanilla / generic templates --- template/src/lib.rs | 6 ++- ...ntainerfile.j2 => Containerfile.ostree.j2} | 0 template/templates/Containerfile.vanilla.j2 | 49 +++++++++++++++++++ template/templates/modules/modules.j2 | 24 ++++----- template/templates/stages.j2 | 2 +- 5 files changed, 68 insertions(+), 13 deletions(-) rename template/templates/{Containerfile.j2 => Containerfile.ostree.j2} (100%) create mode 100644 template/templates/Containerfile.vanilla.j2 diff --git a/template/src/lib.rs b/template/src/lib.rs index a9f43360..76f8257a 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -13,7 +13,11 @@ use uuid::Uuid; pub use askama::Template; #[derive(Debug, Clone, Template, TypedBuilder)] -#[template(path = "Containerfile.j2", escape = "none", whitespace = "minimize")] +#[template( + path = "Containerfile.vanilla.j2", + escape = "none", + whitespace = "minimize" +)] pub struct ContainerFileTemplate<'a> { recipe: &'a Recipe<'a>, diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.ostree.j2 similarity index 100% rename from template/templates/Containerfile.j2 rename to template/templates/Containerfile.ostree.j2 diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 new file mode 100644 index 00000000..4f94e1b8 --- /dev/null +++ b/template/templates/Containerfile.vanilla.j2 @@ -0,0 +1,49 @@ +{%- import "modules/modules.j2" as modules -%} +{%- include "stages.j2" %} + +# Main image +FROM {{ recipe.base_image }}:{{ recipe.image_version }} AS {{ recipe.name|replace('/', "-") }} + +ARG RECIPE={{ recipe_path.display() }} +ARG IMAGE_REGISTRY={{ registry }} + +{%- if self::files_dir_exists() %} +ARG CONFIG_DIRECTORY="/tmp/files" +{%- else if self::config_dir_exists() %} +ARG CONFIG_DIRECTORY="/tmp/config" +{%- endif %} +ARG MODULE_DIRECTORY="/tmp/modules" +ARG IMAGE_NAME="{{ recipe.name }}" +ARG BASE_IMAGE="{{ recipe.base_image }}" + +# Key RUN +RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ + mkdir -p /usr/etc/pki/containers/ \ + && cp /tmp/keys/* /usr/etc/pki/containers/ + +# Bin RUN +RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ + mkdir -p /usr/bin/ \ + && cp /tmp/bins/* /usr/bin/ + +# Init step copied from VanillaOS template +RUN lpkg --unlock && apt-get update + +{% call modules::generic_modules_run(recipe.modules_ext, os_version) %} + +# Cleanup step copied from VanillaOS template +RUN apt-get autoremove -y && apt-get clean && lpkg --lock + +# FsGuard step copied from VanillaOS template +RUN rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && minisign -Sm /FsGuard/filelist -p .//minisign.pub -s .//minisign.key && touch /FsGuard/signature && echo -n "----begin attach----" >> /FsGuard/signature && cat /FsGuard/filelist.minisig >> /FsGuard/signature && echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && cat /FsGuard/signature >> /sources/fsguard/FsGuard && mv /sources/fsguard/FsGuard /usr/sbin/FsGuard && rm ./minisign.key ./minisign.pub + +RUN rm -fr /tmp/* /var/tmp/* + +# Labels are added last since they cause cache misses with buildah +LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" +LABEL org.opencontainers.image.title="{{ recipe.name }}" +LABEL org.opencontainers.image.description="{{ recipe.description }}" +{%- if let Some(repo) = self::get_repo_url() %} +LABEL org.opencontainers.image.source="{{ repo }}" +{%- endif %} +LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index c2236652..fdcb7491 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -1,7 +1,8 @@ -{% macro main_modules_run(modules_ext, os_version) %} +{% macro generic_modules_run(modules_ext, os_version) %} # Module RUNs {%- for module in modules_ext.modules %} {%- if let Some(module) = module.required_fields %} + {%- if module.no_cache %} ARG CACHEBUST="{{ build_id }}" {%- endif %} @@ -22,22 +23,18 @@ RUN \ {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ {%- endif %} - {%- if module.module_type == "akmods" %} - --mount=type=bind,from=stage-akmods-{{ module.generate_akmods_info(os_version).stage_name }},src=/rpms,dst=/tmp/rpms,rw \ - {%- endif %} --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ - --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ - /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' \ - && ostree container commit + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' {%- endif %} {%- endif %} {%- endfor %} {% endmacro %} -{% macro stage_modules_run(modules_ext, os_version) %} + + +{% macro ostree_modules_run(modules_ext, os_version) %} # Module RUNs {%- for module in modules_ext.modules %} {%- if let Some(module) = module.required_fields %} - {%- if module.no_cache %} ARG CACHEBUST="{{ build_id }}" {%- endif %} @@ -58,9 +55,14 @@ RUN \ {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ {%- endif %} + {%- if module.module_type == "akmods" %} + --mount=type=bind,from=stage-akmods-{{ module.generate_akmods_info(os_version).stage_name }},src=/rpms,dst=/tmp/rpms,rw \ + {%- endif %} --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ - /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' + --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ + /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module.print_module_context() }}' \ + && ostree container commit {%- endif %} {%- endif %} {%- endfor %} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/template/templates/stages.j2 b/template/templates/stages.j2 index c1a689a1..9505e9fb 100644 --- a/template/templates/stages.j2 +++ b/template/templates/stages.j2 @@ -73,7 +73,7 @@ SHELL ["bash", "-c"] {%- endif %} {%- endif %} - {% call modules::stage_modules_run(stage.modules_ext, os_version) %} + {% call modules::generic_modules_run(stage.modules_ext, os_version) %} {%- endif %} {%- endfor %} {%- endif %} From 019ab77b8bf0dbcc6b9051706328d8e5abf338fa Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 25 Jul 2024 14:56:31 +0300 Subject: [PATCH 02/12] =?UTF-8?q?fix:=20circumvent=20vanilla=20not=20havin?= =?UTF-8?q?g=20os=20versions=20by=20providing=20a=20default=20value=20?= =?UTF-8?q?=F0=9F=97=91=EF=B8=8F=F0=9F=97=91=EF=B8=8F=F0=9F=97=91=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this patch should be replaced by making os_version optional (at least for some image types) --- src/drivers.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index cc0bfc24..f076367d 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -11,7 +11,7 @@ use std::{ }; use blue_build_recipe::Recipe; -use blue_build_utils::constants::IMAGE_VERSION_LABEL; +// use blue_build_utils::constants::IMAGE_VERSION_LABEL; use log::{debug, info, trace}; use miette::{bail, miette, Result}; use once_cell::sync::Lazy; @@ -389,12 +389,7 @@ impl Driver<'_> { .build(); let inspection = INSPECT_DRIVER.get_metadata(&inspect_opts)?; - let os_version = inspection.get_version().ok_or_else(|| { - miette!( - help = format!("Please check with the image author about using '{IMAGE_VERSION_LABEL}' to report the os version."), - "Unable to get the OS version from the labels" - ) - })?; + let os_version = inspection.get_version().unwrap_or(0); trace!("os_version: {os_version}"); os_version From 22e599d4cf7a7c7d0a96219748eeac0339b2065b Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 25 Jul 2024 15:19:59 +0300 Subject: [PATCH 03/12] fix(vanilla fsguard): download genfilelist.py from vib-fsguard module --- template/templates/Containerfile.vanilla.j2 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 4f94e1b8..805c7ca2 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -35,9 +35,17 @@ RUN lpkg --unlock && apt-get update RUN apt-get autoremove -y && apt-get clean && lpkg --lock # FsGuard step copied from VanillaOS template -RUN rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && minisign -Sm /FsGuard/filelist -p .//minisign.pub -s .//minisign.key && touch /FsGuard/signature && echo -n "----begin attach----" >> /FsGuard/signature && cat /FsGuard/filelist.minisig >> /FsGuard/signature && echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && cat /FsGuard/signature >> /sources/fsguard/FsGuard && mv /sources/fsguard/FsGuard /usr/sbin/FsGuard && rm ./minisign.key ./minisign.pub - -RUN rm -fr /tmp/* /var/tmp/* +# first download the required python script from the vib-fsguard module +RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ + rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && \ + chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && \ + python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && \ + minisign -Sm /FsGuard/filelist -p .//minisign.pub -s .//minisign.key && touch /FsGuard/signature && \ + echo -n "----begin attach----" >> /FsGuard/signature && cat /FsGuard/filelist.minisig >> /FsGuard/signature && \ + echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && \ + cat /FsGuard/signature >> /sources/fsguard/FsGuard && mv /sources/fsguard/FsGuard /usr/sbin/FsGuard && rm ./minisign.key ./minisign.pub + +RUN rm -fr /tmp/* /var/tmp/* /sources/* # Labels are added last since they cause cache misses with buildah LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" From bf8c41f9dd5d2b0d94b1cd314527de88ab66c284 Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 25 Jul 2024 21:37:40 +0300 Subject: [PATCH 04/12] fix: fsguard integrity check for whole /usr --- template/templates/Containerfile.vanilla.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 805c7ca2..1078261b 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -39,7 +39,7 @@ RUN apt-get autoremove -y && apt-get clean && lpkg --lock RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && \ chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && \ - python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && \ + python3 /sources/fsguard/genfilelist.py /usr /FsGuard/filelist /usr/sbin/FsGuard && \ minisign -Sm /FsGuard/filelist -p .//minisign.pub -s .//minisign.key && touch /FsGuard/signature && \ echo -n "----begin attach----" >> /FsGuard/signature && cat /FsGuard/filelist.minisig >> /FsGuard/signature && \ echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && \ From 60169a55819fcd156dc2ea41c5fb174dee43a5d3 Mon Sep 17 00:00:00 2001 From: xyny Date: Tue, 30 Jul 2024 20:15:54 +0300 Subject: [PATCH 05/12] chore: try moving & disabling some of the 'cruft' --- template/templates/Containerfile.vanilla.j2 | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 1078261b..75c2486e 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -17,9 +17,9 @@ ARG IMAGE_NAME="{{ recipe.name }}" ARG BASE_IMAGE="{{ recipe.base_image }}" # Key RUN -RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ - mkdir -p /usr/etc/pki/containers/ \ - && cp /tmp/keys/* /usr/etc/pki/containers/ +# RUN --mount=type=bind,from=stage-keys,src=/keys,dst=/tmp/keys \ +# mkdir -p /usr/etc/pki/containers/ \ +# && cp /tmp/keys/* /usr/etc/pki/containers/ # Bin RUN RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ @@ -34,6 +34,16 @@ RUN lpkg --unlock && apt-get update # Cleanup step copied from VanillaOS template RUN apt-get autoremove -y && apt-get clean && lpkg --lock +# Labels are(n't) added last since they (may cause issues with FsGuard) (eventhough they) cause cache misses with buildah +LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" +LABEL org.opencontainers.image.title="{{ recipe.name }}" +LABEL org.opencontainers.image.description="{{ recipe.description }}" +{%- if let Some(repo) = self::get_repo_url() %} +LABEL org.opencontainers.image.source="{{ repo }}" +{%- endif %} +LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md + + # FsGuard step copied from VanillaOS template # first download the required python script from the vib-fsguard module RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ @@ -45,13 +55,4 @@ RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && \ cat /FsGuard/signature >> /sources/fsguard/FsGuard && mv /sources/fsguard/FsGuard /usr/sbin/FsGuard && rm ./minisign.key ./minisign.pub -RUN rm -fr /tmp/* /var/tmp/* /sources/* - -# Labels are added last since they cause cache misses with buildah -LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" -LABEL org.opencontainers.image.title="{{ recipe.name }}" -LABEL org.opencontainers.image.description="{{ recipe.description }}" -{%- if let Some(repo) = self::get_repo_url() %} -LABEL org.opencontainers.image.source="{{ repo }}" -{%- endif %} -LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md +RUN rm -fr /tmp/* /var/tmp/* /sources/* \ No newline at end of file From e65af406f22d3b35f9699a51b8e4f598c1185d8c Mon Sep 17 00:00:00 2001 From: xyny Date: Tue, 30 Jul 2024 20:20:45 +0300 Subject: [PATCH 06/12] chore: try generating file list for only /usr/bin just like was done originally --- template/templates/Containerfile.vanilla.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 75c2486e..e64d1b0f 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -49,7 +49,7 @@ LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-b RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && \ chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && \ - python3 /sources/fsguard/genfilelist.py /usr /FsGuard/filelist /usr/sbin/FsGuard && \ + python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && \ minisign -Sm /FsGuard/filelist -p .//minisign.pub -s .//minisign.key && touch /FsGuard/signature && \ echo -n "----begin attach----" >> /FsGuard/signature && cat /FsGuard/filelist.minisig >> /FsGuard/signature && \ echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && \ From 2e4ecf80a703ba0702b1ffe12c0e8e88bf7290a4 Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 1 Aug 2024 17:41:40 +0300 Subject: [PATCH 07/12] fix(fsguard): fetch FsGuard binary --- template/templates/Containerfile.vanilla.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index e64d1b0f..806291d0 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -45,8 +45,10 @@ LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-b # FsGuard step copied from VanillaOS template -# first download the required python script from the vib-fsguard module -RUN mkdir -p /sources/fsguard/ && curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ +# first download the required python script from the vib-fsguard module and the FsGuard binary +RUN mkdir -p /sources/fsguard/ && \ + wget https://github.com/linux-immutability-tools/FsGuard/releases/download/v0.1.2-2/FsGuard_0.1.2-2_linux_arm64.tar.gz -O /tmp/fsguard.tar.gz && tar -xf /tmp/fsguard.tar.gz -C /sources/fsguard/ && \ + curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && \ chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && \ python3 /sources/fsguard/genfilelist.py /usr/bin /FsGuard/filelist /usr/sbin/FsGuard && \ From 3dcefae50ab612bf8250fa70b304175af126dac6 Mon Sep 17 00:00:00 2001 From: xyny Date: Wed, 7 Aug 2024 11:28:45 +0300 Subject: [PATCH 08/12] fix(fsguard): correct URL --- template/templates/Containerfile.vanilla.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 806291d0..8ad1d763 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -47,7 +47,7 @@ LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-b # FsGuard step copied from VanillaOS template # first download the required python script from the vib-fsguard module and the FsGuard binary RUN mkdir -p /sources/fsguard/ && \ - wget https://github.com/linux-immutability-tools/FsGuard/releases/download/v0.1.2-2/FsGuard_0.1.2-2_linux_arm64.tar.gz -O /tmp/fsguard.tar.gz && tar -xf /tmp/fsguard.tar.gz -C /sources/fsguard/ && \ + wget https://github.com/linux-immutability-tools/FsGuard/releases/download/v0.1.2-2/FsGuard_0.1.2-2_linux_amd64.tar.gz -O /tmp/fsguard.tar.gz && tar -xf /tmp/fsguard.tar.gz -C /sources/fsguard/ && \ curl https://raw.githubusercontent.com/Vanilla-OS/vib-fsguard/main/genfilelist.py -o /sources/fsguard/genfilelist.py && \ rm -rf /FsGuard && rm -f ./minisign.pub ./minisign.key && chmod +x /usr/sbin/init && mkdir /FsGuard && \ chmod +x /sources/fsguard/genfilelist.py && minisign -WG -s ./minisign.key && \ From 622f89599ffafc4c9d459539f6b257737ee7a7ed Mon Sep 17 00:00:00 2001 From: xyny Date: Wed, 7 Aug 2024 11:40:39 +0300 Subject: [PATCH 09/12] refactor: create new template struct for vanilla while keeping old template struct (currently switches to using the vanilla template struct only, need to add ability for user to specify struct) --- src/commands/generate.rs | 4 ++-- template/src/lib.rs | 26 +++++++++++++++++++++- template/templates/Containerfile.ostree.j2 | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/commands/generate.rs b/src/commands/generate.rs index 1f3170cb..e8a23a21 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -4,7 +4,7 @@ use std::{ }; use blue_build_recipe::Recipe; -use blue_build_template::{ContainerFileTemplate, Template}; +use blue_build_template::{Template, VanillaContainerFileTemplate}; use blue_build_utils::{ constants::{ CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_REGISTRY, CONFIG_PATH, GITHUB_REPOSITORY_OWNER, @@ -114,7 +114,7 @@ impl GenerateCommand { info!("Templating for recipe at {}", recipe_path.display()); - let template = ContainerFileTemplate::builder() + let template = VanillaContainerFileTemplate::builder() .os_version(Driver::get_os_version(&recipe_de)?) .build_id(Driver::get_build_id()) .recipe(&recipe_de) diff --git a/template/src/lib.rs b/template/src/lib.rs index a7d38d84..6cba89a3 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -12,13 +12,37 @@ use uuid::Uuid; pub use rinja::Template; +#[derive(Debug, Clone, Template, TypedBuilder)] +#[template( + path = "Containerfile.ostree.j2", + escape = "none", + whitespace = "minimize" +)] +pub struct OstreeContainerFileTemplate<'a> { + recipe: &'a Recipe<'a>, + + #[builder(setter(into))] + recipe_path: &'a Path, + + #[builder(setter(into))] + build_id: Uuid, + + os_version: u64, + + #[builder(setter(into))] + registry: Cow<'a, str>, + + #[builder(setter(into))] + exports_tag: Cow<'a, str>, +} + #[derive(Debug, Clone, Template, TypedBuilder)] #[template( path = "Containerfile.vanilla.j2", escape = "none", whitespace = "minimize" )] -pub struct ContainerFileTemplate<'a> { +pub struct VanillaContainerFileTemplate<'a> { recipe: &'a Recipe<'a>, #[builder(setter(into))] diff --git a/template/templates/Containerfile.ostree.j2 b/template/templates/Containerfile.ostree.j2 index e83e2087..78f98ca7 100644 --- a/template/templates/Containerfile.ostree.j2 +++ b/template/templates/Containerfile.ostree.j2 @@ -28,7 +28,7 @@ RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ && cp /tmp/bins/* /usr/bin/ \ && ostree container commit -{% call modules::main_modules_run(recipe.modules_ext, os_version) %} +{% call modules::ostree_modules_run(recipe.modules_ext, os_version) %} RUN rm -fr /tmp/* /var/* && ostree container commit From 90c14b6d7dfae7830828a3906c8b5adacd526654 Mon Sep 17 00:00:00 2001 From: xyny Date: Wed, 7 Aug 2024 12:59:38 +0300 Subject: [PATCH 10/12] feat: allow specifying template to use in recipe with base-image-type key this key name is temporary and shall be changed with recipe v2 --- recipe/src/recipe.rs | 5 +++ src/commands/generate.rs | 75 +++++++++++++++++++++++++++++----------- template/src/lib.rs | 18 +++++++++- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index 85168973..8400133e 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -39,6 +39,11 @@ pub struct Recipe<'a> { #[builder(setter(into))] pub base_image: Cow<'a, str>, + /// The base image type for user's image. + #[serde(alias = "base-image-type", skip_serializing_if = "Option::is_none")] + #[builder(setter(into))] + pub base_image_type: Option>, + /// The version/tag of the base image. #[serde(alias = "image-version")] #[builder(setter(into))] diff --git a/src/commands/generate.rs b/src/commands/generate.rs index e8a23a21..9fcb41ff 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -1,10 +1,9 @@ use std::{ - env, - path::{Path, PathBuf}, + env, path::{Path, PathBuf} }; -use blue_build_recipe::Recipe; -use blue_build_template::{Template, VanillaContainerFileTemplate}; +use blue_build_recipe::{Recipe}; +use blue_build_template::{ContainerFileTemplate, OstreeContainerFileTemplate, VanillaContainerFileTemplate}; use blue_build_utils::{ constants::{ CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_REGISTRY, CONFIG_PATH, GITHUB_REPOSITORY_OWNER, @@ -114,11 +113,36 @@ impl GenerateCommand { info!("Templating for recipe at {}", recipe_path.display()); - let template = VanillaContainerFileTemplate::builder() - .os_version(Driver::get_os_version(&recipe_de)?) + let template: Box = match &recipe_de.base_image_type { + Some(cow) => match cow.as_ref() { + "vanilla" => Box::new(self.build_vanilla_template(&recipe_de, &recipe_path)?), + "ostree" => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), + _ => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), + } + None => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), + }; + + let output_str = template.render().into_diagnostic()?; + if let Some(output) = self.output.as_ref() { + debug!("Templating to file {}", output.display()); + trace!("Containerfile:\n{output_str}"); + + std::fs::write(output, output_str).into_diagnostic()?; + } else { + debug!("Templating to stdout"); + syntax_highlighting::print(&output_str, "Dockerfile", self.syntax_theme)?; + } + + Ok(()) + } + + fn build_ostree_template<'a>(&self, recipe_de: &'a Recipe<'a>, recipe_path: &'a Path) -> Result> { + info!("Using ostree template"); + Ok(OstreeContainerFileTemplate::builder() + .os_version(Driver::get_os_version(recipe_de)?) .build_id(Driver::get_build_id()) - .recipe(&recipe_de) - .recipe_path(recipe_path.as_path()) + .recipe(recipe_de) + .recipe_path(recipe_path) .registry(self.get_registry()) .exports_tag({ #[allow(clippy::const_is_empty)] @@ -131,20 +155,29 @@ impl GenerateCommand { shadow::COMMIT_HASH.to_string() } }) - .build(); - - let output_str = template.render().into_diagnostic()?; - if let Some(output) = self.output.as_ref() { - debug!("Templating to file {}", output.display()); - trace!("Containerfile:\n{output_str}"); - - std::fs::write(output, output_str).into_diagnostic()?; - } else { - debug!("Templating to stdout"); - syntax_highlighting::print(&output_str, "Dockerfile", self.syntax_theme)?; - } + .build()) + } - Ok(()) + fn build_vanilla_template<'a>(&self, recipe_de: &'a Recipe<'a>, recipe_path: &'a Path) -> Result> { + info!("Using vanilla template"); + Ok(VanillaContainerFileTemplate::builder() + .os_version(Driver::get_os_version(recipe_de)?) + .build_id(Driver::get_build_id()) + .recipe(recipe_de) + .recipe_path(recipe_path) + .registry(self.get_registry()) + .exports_tag({ + #[allow(clippy::const_is_empty)] + if shadow::COMMIT_HASH.is_empty() { + // This is done for users who install via + // cargo. Cargo installs do not carry git + // information via shadow + format!("v{}", crate_version!()) + } else { + shadow::COMMIT_HASH.to_string() + } + }) + .build()) } fn get_registry(&self) -> String { diff --git a/template/src/lib.rs b/template/src/lib.rs index 6cba89a3..0aeebff4 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -12,6 +12,13 @@ use uuid::Uuid; pub use rinja::Template; +pub trait ContainerFileTemplate { + /// # Errors + /// + /// Will return error from rinja if there is an issue rendering the template. + fn render(&self) -> Result; +} + #[derive(Debug, Clone, Template, TypedBuilder)] #[template( path = "Containerfile.ostree.j2", @@ -35,6 +42,11 @@ pub struct OstreeContainerFileTemplate<'a> { #[builder(setter(into))] exports_tag: Cow<'a, str>, } +impl ContainerFileTemplate for OstreeContainerFileTemplate<'_> { + fn render(&self) -> Result { + Template::render(&self) + } +} #[derive(Debug, Clone, Template, TypedBuilder)] #[template( @@ -59,7 +71,11 @@ pub struct VanillaContainerFileTemplate<'a> { #[builder(setter(into))] exports_tag: Cow<'a, str>, } - +impl ContainerFileTemplate for VanillaContainerFileTemplate<'_> { + fn render(&self) -> Result { + Template::render(&self) + } +} #[derive(Debug, Clone, Template, TypedBuilder)] #[template(path = "github_issue.j2", escape = "md")] pub struct GithubIssueTemplate<'a> { From 772137df6775caedd26c9ad181ef5bdeb0e075b7 Mon Sep 17 00:00:00 2001 From: xyny Date: Wed, 7 Aug 2024 13:00:36 +0300 Subject: [PATCH 11/12] style(generate.rs): format --- src/commands/generate.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/commands/generate.rs b/src/commands/generate.rs index 9fcb41ff..73af20e6 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -1,9 +1,12 @@ use std::{ - env, path::{Path, PathBuf} + env, + path::{Path, PathBuf}, }; -use blue_build_recipe::{Recipe}; -use blue_build_template::{ContainerFileTemplate, OstreeContainerFileTemplate, VanillaContainerFileTemplate}; +use blue_build_recipe::Recipe; +use blue_build_template::{ + ContainerFileTemplate, OstreeContainerFileTemplate, VanillaContainerFileTemplate, +}; use blue_build_utils::{ constants::{ CI_PROJECT_NAME, CI_PROJECT_NAMESPACE, CI_REGISTRY, CONFIG_PATH, GITHUB_REPOSITORY_OWNER, @@ -118,7 +121,7 @@ impl GenerateCommand { "vanilla" => Box::new(self.build_vanilla_template(&recipe_de, &recipe_path)?), "ostree" => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), _ => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), - } + }, None => Box::new(self.build_ostree_template(&recipe_de, &recipe_path)?), }; @@ -136,8 +139,12 @@ impl GenerateCommand { Ok(()) } - fn build_ostree_template<'a>(&self, recipe_de: &'a Recipe<'a>, recipe_path: &'a Path) -> Result> { - info!("Using ostree template"); + fn build_ostree_template<'a>( + &self, + recipe_de: &'a Recipe<'a>, + recipe_path: &'a Path, + ) -> Result> { + info!("Using ostree template"); Ok(OstreeContainerFileTemplate::builder() .os_version(Driver::get_os_version(recipe_de)?) .build_id(Driver::get_build_id()) @@ -158,8 +165,12 @@ impl GenerateCommand { .build()) } - fn build_vanilla_template<'a>(&self, recipe_de: &'a Recipe<'a>, recipe_path: &'a Path) -> Result> { - info!("Using vanilla template"); + fn build_vanilla_template<'a>( + &self, + recipe_de: &'a Recipe<'a>, + recipe_path: &'a Path, + ) -> Result> { + info!("Using vanilla template"); Ok(VanillaContainerFileTemplate::builder() .os_version(Driver::get_os_version(recipe_de)?) .build_id(Driver::get_build_id()) From 077ace9da409caa556a578d7ebe53ebe9eec2f25 Mon Sep 17 00:00:00 2001 From: xyny Date: Thu, 8 Aug 2024 18:07:22 +0300 Subject: [PATCH 12/12] fix: move LABELs back to the bottom --- template/templates/Containerfile.vanilla.j2 | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/template/templates/Containerfile.vanilla.j2 b/template/templates/Containerfile.vanilla.j2 index 8ad1d763..46ff8440 100644 --- a/template/templates/Containerfile.vanilla.j2 +++ b/template/templates/Containerfile.vanilla.j2 @@ -34,16 +34,6 @@ RUN lpkg --unlock && apt-get update # Cleanup step copied from VanillaOS template RUN apt-get autoremove -y && apt-get clean && lpkg --lock -# Labels are(n't) added last since they (may cause issues with FsGuard) (eventhough they) cause cache misses with buildah -LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" -LABEL org.opencontainers.image.title="{{ recipe.name }}" -LABEL org.opencontainers.image.description="{{ recipe.description }}" -{%- if let Some(repo) = self::get_repo_url() %} -LABEL org.opencontainers.image.source="{{ repo }}" -{%- endif %} -LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md - - # FsGuard step copied from VanillaOS template # first download the required python script from the vib-fsguard module and the FsGuard binary RUN mkdir -p /sources/fsguard/ && \ @@ -57,4 +47,13 @@ RUN mkdir -p /sources/fsguard/ && \ echo -n "----begin second attach----" >> /FsGuard/signature && tail -n1 .//minisign.pub >> /FsGuard/signature && \ cat /FsGuard/signature >> /sources/fsguard/FsGuard && mv /sources/fsguard/FsGuard /usr/sbin/FsGuard && rm ./minisign.key ./minisign.pub -RUN rm -fr /tmp/* /var/tmp/* /sources/* \ No newline at end of file +RUN rm -fr /tmp/* /var/tmp/* /sources/* + +# Labels are added last since they cause cache misses with buildah +LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}" +LABEL org.opencontainers.image.title="{{ recipe.name }}" +LABEL org.opencontainers.image.description="{{ recipe.description }}" +{%- if let Some(repo) = self::get_repo_url() %} +LABEL org.opencontainers.image.source="{{ repo }}" +{%- endif %} +LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md \ No newline at end of file