From 9a3ad0ae17eff97d339c69c9b85036e29874bcf1 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Fri, 11 Oct 2024 19:37:28 -0400 Subject: [PATCH] feat: Add validation command --- .gitignore | 2 + Cargo.lock | 605 ++++++++++++++---- Cargo.toml | 21 +- Earthfile | 415 ++++++------ bacon.toml | 9 +- integration-tests/Earthfile | 163 ++--- .../legacy-test-repo/config/recipe.yml | 1 + .../test-repo/recipes/akmods.yml | 2 + .../test-repo/recipes/bluebuild.yml | 4 +- .../test-repo/recipes/flatpaks.yml | 2 + .../test-repo/recipes/recipe-39.yml | 1 + .../test-repo/recipes/recipe-arm64.yml | 7 +- .../recipes/recipe-invalid-module.yml | 57 ++ .../recipes/recipe-invalid-stage.yml | 61 ++ .../test-repo/recipes/recipe-invalid.yml | 59 ++ .../test-repo/recipes/recipe.yml | 1 + .../test-repo/recipes/stages.yml | 3 + process/Cargo.toml | 4 +- process/drivers.rs | 4 +- process/drivers/sigstore_driver.rs | 42 +- process/process.rs | 8 +- recipe/src/lib.rs | 23 + recipe/src/module.rs | 24 +- recipe/src/module_ext.rs | 54 +- recipe/src/recipe.rs | 9 - recipe/src/stage.rs | 13 +- recipe/src/stages_ext.rs | 52 +- src/bin/bluebuild.rs | 4 + src/commands.rs | 9 +- src/commands/bug_report.rs | 2 +- src/commands/generate.rs | 11 +- src/commands/validate.rs | 405 ++++++++++++ src/commands/validate/schema_validator.rs | 105 +++ src/lib.rs | 5 +- template/templates/modules/modules.j2 | 5 +- 35 files changed, 1675 insertions(+), 517 deletions(-) create mode 100644 integration-tests/test-repo/recipes/recipe-invalid-module.yml create mode 100644 integration-tests/test-repo/recipes/recipe-invalid-stage.yml create mode 100644 integration-tests/test-repo/recipes/recipe-invalid.yml create mode 100644 src/commands/validate.rs create mode 100644 src/commands/validate/schema_validator.rs diff --git a/.gitignore b/.gitignore index 1ad22d54..61ae6973 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ .vscode/ result* .direnv/ +.arg +.secret cosign.key !test-files/keys/cosign.key diff --git a/Cargo.lock b/Cargo.lock index 0a8a1fea..81672c77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,7 @@ dependencies = [ "cfg-if", "getrandom", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -264,6 +265,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -327,7 +343,9 @@ dependencies = [ "clap_complete", "colored", "fuzzy-matcher", + "indexmap 2.6.0", "indicatif", + "jsonschema", "log", "miette", "oci-distribution", @@ -335,12 +353,14 @@ dependencies = [ "os_info", "rayon", "requestty", + "reqwest 0.12.8", "rusty-hook", "serde", "serde_json", "serde_yaml 0.9.34+deprecated", "shadow-rs", "tempdir", + "tokio", "urlencoding", "users", ] @@ -369,6 +389,7 @@ dependencies = [ "once_cell", "os_pipe", "rand 0.8.5", + "reqwest 0.12.8", "rstest", "semver", "serde", @@ -458,6 +479,12 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "borrow-or-share" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" + [[package]] name = "bstr" version = "1.10.0" @@ -474,6 +501,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "byteorder" version = "1.5.0" @@ -537,8 +570,6 @@ version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58e804ac3194a48bb129643eb1d62fcc20d18c6b8c181704489353d13120bcd1" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -701,12 +732,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "const_fn" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" - [[package]] name = "const_format" version = "0.2.33" @@ -1011,6 +1036,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "docker_credential" version = "1.3.1" @@ -1094,6 +1130,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -1135,6 +1180,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.1" @@ -1179,6 +1235,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1207,6 +1274,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fsio" version = "0.1.3" @@ -1362,19 +1439,6 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" -[[package]] -name = "git2" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" -dependencies = [ - "bitflags 2.6.0", - "libc", - "libgit2-sys", - "log", - "url", -] - [[package]] name = "glob" version = "0.3.1" @@ -1663,7 +1727,6 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "rustls 0.23.14", - "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -1713,6 +1776,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1729,6 +1910,18 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd69211b9b519e98303c015e21a007e293db403b6c85b9b124e133d25e242cdd" +dependencies = [ + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1884,15 +2077,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.72" @@ -1933,6 +2117,31 @@ dependencies = [ "utf8-decode", ] +[[package]] +name = "jsonschema" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ae663abb3bb9e77538ee88a0eb69cbd3f62a8bf2018f848fbc60c2cdec024d" +dependencies = [ + "ahash 0.8.11", + "base64 0.22.1", + "bytecount", + "email_address", + "fancy-regex", + "fraction", + "idna 1.0.2", + "itoa", + "num-cmp", + "once_cell", + "percent-encoding", + "referencing", + "regex-syntax", + "reqwest 0.12.8", + "serde", + "serde_json", + "uuid-simd", +] + [[package]] name = "jwt" version = "0.16.0" @@ -2065,18 +2274,6 @@ version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" -[[package]] -name = "libgit2-sys" -version = "0.16.2+1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - [[package]] name = "libm" version = "0.2.8" @@ -2093,18 +2290,6 @@ dependencies = [ "libc", ] -[[package]] -name = "libz-sys" -version = "1.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2117,6 +2302,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -2329,6 +2520,30 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -2346,6 +2561,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2372,6 +2602,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2591,12 +2832,6 @@ dependencies = [ "url", ] -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - [[package]] name = "option-ext" version = "0.2.0" @@ -2633,6 +2868,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "owo-colors" version = "4.1.0" @@ -3209,6 +3450,39 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "referencing" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c152a23ee0e5947ee31d9cfebc873a5aa3a249da9e59d2e76cd7416a13cc9a5d" +dependencies = [ + "ahash 0.8.11", + "fluent-uri", + "once_cell", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.11.0" @@ -3345,6 +3619,7 @@ checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", "futures-util", "http 1.1.0", @@ -3363,7 +3638,6 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.14", - "rustls-native-certs", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -3562,19 +3836,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.2.0", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3668,15 +3929,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -3719,29 +3971,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.6.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "1.0.23" @@ -3919,10 +4148,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e5c5c8276991763b44ede03efaf966eaa0412fafbf299e6380704678ca3b997" dependencies = [ "const_format", - "git2", "is_debug", "time", - "tzdb", ] [[package]] @@ -4228,6 +4455,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "syntect" version = "5.2.0" @@ -4411,6 +4649,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -4617,35 +4865,6 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "tz-rs" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" -dependencies = [ - "const_fn", -] - -[[package]] -name = "tzdb" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b580f6b365fa89f5767cdb619a55d534d04a4e14c2d7e5b9a31e94598687fb1" -dependencies = [ - "iana-time-zone", - "tz-rs", - "tzdb_data", -] - -[[package]] -name = "tzdb_data" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "654c1ec546942ce0594e8d220e6b8e3899e0a0a8fe70ddd54d32a376dfefe3f8" -dependencies = [ - "tz-rs", -] - [[package]] name = "unicase" version = "2.7.0" @@ -4744,7 +4963,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -4765,12 +4984,24 @@ dependencies = [ "log", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8-decode" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca61eb27fa339aa08826a29f03e87b99b4d8f0fc2255306fd266bb1b6a9de498" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4787,10 +5018,15 @@ dependencies = [ ] [[package]] -name = "vcpkg" -version = "0.2.15" +name = "uuid-simd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] [[package]] name = "version_check" @@ -4798,6 +5034,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -5274,6 +5516,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ab703352da6a72f35c39a533526393725640575bb211f61987a2748323ad956" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "x509-cert" version = "0.2.5" @@ -5297,6 +5551,30 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -5318,6 +5596,27 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" @@ -5338,3 +5637,25 @@ dependencies = [ "quote", "syn 2.0.79", ] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] diff --git a/Cargo.toml b/Cargo.toml index ecac5147..cf9e9017 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,15 @@ colored = "2" indexmap = { version = "2", features = ["serde"] } indicatif = { version = "0.17", features = ["improved_unicode"] } log = "0.4" -oci-distribution = { version = "0.11.0", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots"] } +oci-distribution = { version = "0.11", default-features = false } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } miette = "7" rstest = "0.18" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" tempdir = "0.3" +tokio = { version = "1", features = ["rt", "rt-multi-thread"] } users = "0.11" uuid = { version = "1", features = ["v4"] } @@ -65,24 +67,28 @@ blue-build-process-management = { version = "=0.8.20", path = "./process" } clap-verbosity-flag = "2" clap_complete = "4" fuzzy-matcher = "0.3" +jsonschema = { version = "0.26", optional = true } open = "5" os_info = "3" rayon = { version = "1.10.0", optional = true } requestty = { version = "0.5", features = ["macros", "termion"] } -shadow-rs = "0.26" +shadow-rs = { version = "0.26", default-features = false } urlencoding = "2" cached.workspace = true clap = { workspace = true, features = ["derive", "cargo", "unicode", "env"] } colored.workspace = true +indexmap.workspace = true indicatif.workspace = true log.workspace = true miette = { workspace = true, features = ["fancy"] } oci-distribution.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true tempdir.workspace = true +tokio = { workspace = true, optional = true } bon.workspace = true users.workspace = true @@ -90,17 +96,24 @@ users.workspace = true default = [] stages = ["blue-build-recipe/stages"] copy = ["blue-build-recipe/copy"] -multi-recipe = ["rayon", "indicatif/rayon"] +multi-recipe = ["dep:rayon", "indicatif/rayon"] iso = [] switch = [] sigstore = ["blue-build-process-management/sigstore"] login = [] +validate = [ + "dep:jsonschema", + "dep:rayon", + "dep:tokio", + "cached/async", + "blue-build-process-management/validate" +] [dev-dependencies] rusty-hook = "0.11" [build-dependencies] -shadow-rs = "0.26" +shadow-rs = { version = "0.26", default-features = false } [lints] workspace = true diff --git a/Earthfile b/Earthfile index b6a29c85..9ed3670c 100644 --- a/Earthfile +++ b/Earthfile @@ -6,270 +6,281 @@ IMPORT github.com/earthly/lib/rust AS rust ARG --global IMAGE=ghcr.io/blue-build/cli all: - WAIT - BUILD --platform=linux/amd64 --platform=linux/arm64 +prebuild - END - BUILD +build - BUILD ./integration-tests+all + WAIT + BUILD --platform=linux/amd64 --platform=linux/arm64 +prebuild + END + BUILD +build + BUILD ./integration-tests+all run-checks: - BUILD +lint - BUILD +test + BUILD +lint + BUILD +test build-images: - BUILD +blue-build-cli - BUILD +blue-build-cli-alpine - BUILD +installer + BUILD +blue-build-cli + BUILD +blue-build-cli-alpine + BUILD +installer prebuild: - BUILD +blue-build-cli-prebuild - BUILD +blue-build-cli-alpine-prebuild + BUILD +blue-build-cli-prebuild + BUILD +blue-build-cli-alpine-prebuild lint: - FROM +common - RUN cargo fmt --check - DO rust+CARGO --args="clippy" - DO rust+CARGO --args="clippy --all-features" - DO rust+CARGO --args="clippy --no-default-features" - DO rust+CARGO --args="clippy --no-default-features --features stages" - DO rust+CARGO --args="clippy --no-default-features --features copy" - DO rust+CARGO --args="clippy --no-default-features --features multi-recipe" - DO rust+CARGO --args="clippy --no-default-features --features iso" - DO rust+CARGO --args="clippy --no-default-features --features switch" - DO rust+CARGO --args="clippy --no-default-features --features sigstore" + FROM +common + RUN cargo fmt --check + DO rust+CARGO --args="clippy" + DO rust+CARGO --args="clippy --all-features" + DO rust+CARGO --args="clippy --no-default-features" + FOR feat IN $( \ + cargo metadata --format-version 1 \ + | jq -cr '.packages[] | select(.name == "blue-build") | .features | keys | .[] | select(. != "default")' \ + ) + DO rust+CARGO --args="clippy --no-default-features --features $feat" + END test: - FROM +common - COPY --dir test-files/ integration-tests/ /app - COPY +cosign/cosign /usr/bin/cosign - - DO rust+CARGO --args="test --workspace" - DO rust+CARGO --args="test --workspace --all-features" - DO rust+CARGO --args="test --workspace --no-default-features" - DO rust+CARGO --args="test --workspace --no-default-features --features stages" - DO rust+CARGO --args="test --workspace --no-default-features --features copy" - DO rust+CARGO --args="test --workspace --no-default-features --features multi-recipe" - DO rust+CARGO --args="test --workspace --no-default-features --features iso" - DO rust+CARGO --args="test --workspace --no-default-features --features switch" - DO rust+CARGO --args="test --workspace --no-default-features --features sigstore" + FROM +common + COPY --dir test-files/ integration-tests/ /app + COPY +cosign/cosign /usr/bin/cosign + + DO rust+CARGO --args="test --workspace" + DO rust+CARGO --args="test --workspace --all-features" + DO rust+CARGO --args="test --workspace --no-default-features" + FOR feat IN $( \ + cargo metadata --format-version 1 \ + | jq -cr '.packages[] | select(.name == "blue-build") | .features | keys | .[] | select(. != "default")' \ + ) + DO rust+CARGO --args="test --workspace --features $feat" + END install: - FROM +common - ARG --required BUILD_TARGET - - DO rust+CROSS --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" - - SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + FROM +common + ARG --required BUILD_TARGET + ARG --required RELEASE + + IF [ "$RELEASE" = "true" ] + DO rust+CROSS --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + ELSE + DO rust+CROSS --args="build" --target="$BUILD_TARGET" --output="$BUILD_TARGET/debug/[^\./]+" + SAVE ARTIFACT target/$BUILD_TARGET/debug/bluebuild + END install-all-features: - FROM +common - ARG --required BUILD_TARGET - - DO rust+CROSS --args="build --all-features --release" --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" - - SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + FROM +common + ARG --required BUILD_TARGET + ARG --required RELEASE + + IF [ "$RELEASE" = "true" ] + DO rust+CROSS --args="build --all-features --release" --target="$BUILD_TARGET" --output="$BUILD_TARGET/release/[^\./]+" + SAVE ARTIFACT target/$BUILD_TARGET/release/bluebuild + ELSE + DO rust+CROSS --args="build --all-features" --target="$BUILD_TARGET" --output="$BUILD_TARGET/debug/[^\./]+" + SAVE ARTIFACT target/$BUILD_TARGET/debug/bluebuild + END common: - FROM --platform=native ghcr.io/blue-build/earthly-lib/cargo-builder + FROM --platform=native ghcr.io/blue-build/earthly-lib/cargo-builder - WORKDIR /app - COPY --keep-ts --dir src/ template/ recipe/ utils/ process/ /app - COPY --keep-ts Cargo.* /app - COPY --keep-ts *.md /app - COPY --keep-ts LICENSE /app - COPY --keep-ts build.rs /app - COPY --keep-ts --dir .git/ /app - RUN touch build.rs + WORKDIR /app + COPY --keep-ts --dir src/ template/ recipe/ utils/ process/ /app + COPY --keep-ts Cargo.* /app + COPY --keep-ts *.md /app + COPY --keep-ts LICENSE /app + COPY --keep-ts build.rs /app + COPY --keep-ts --dir .git/ /app + RUN touch build.rs - DO rust+INIT --keep_fingerprints=true + DO rust+INIT --keep_fingerprints=true build-scripts: - ARG BASE_IMAGE="alpine" - FROM $BASE_IMAGE + ARG BASE_IMAGE="alpine" + FROM $BASE_IMAGE - COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest - LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" + COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest + LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - COPY --dir scripts/ / - FOR script IN "$(ls /scripts | grep -e '.*\.sh$')" - RUN echo "Making ${script} executable" && \ - chmod +x "scripts/${script}" - END + COPY --dir scripts/ / + FOR script IN "$(ls /scripts | grep -e '.*\.sh$')" + RUN echo "Making ${script} executable" && \ + chmod +x "scripts/${script}" + END - DO --pass-args +SAVE_IMAGE --IMAGE="$IMAGE/build-scripts" + DO --pass-args +SAVE_IMAGE --IMAGE="$IMAGE/build-scripts" blue-build-cli-prebuild: - ARG BASE_IMAGE="registry.fedoraproject.org/fedora-toolbox" - FROM DOCKERFILE -f Dockerfile.fedora . + ARG BASE_IMAGE="registry.fedoraproject.org/fedora-toolbox" + FROM DOCKERFILE -f Dockerfile.fedora . - COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest - LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" + COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest + LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - COPY +cosign/cosign /usr/bin/cosign - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" + COPY +cosign/cosign /usr/bin/cosign + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" blue-build-cli: - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - FROM "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + FROM "$IMAGE:$EARTHLY_GIT_HASH-prebuild-$TARGETARCH" - IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-gnu" - ELSE - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" - END + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-gnu" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-gnu" + END - RUN mkdir -p /bluebuild - WORKDIR /bluebuild - CMD ["bluebuild"] + RUN mkdir -p /bluebuild + WORKDIR /bluebuild + CMD ["bluebuild"] - DO --pass-args +SAVE_IMAGE + DO --pass-args +SAVE_IMAGE blue-build-cli-alpine-prebuild: - ARG BASE_IMAGE="alpine" - FROM DOCKERFILE -f Dockerfile.alpine . + ARG BASE_IMAGE="alpine" + FROM DOCKERFILE -f Dockerfile.alpine . - COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest - LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" + COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest + LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - COPY +cosign/cosign /usr/bin/cosign + COPY +cosign/cosign /usr/bin/cosign - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + SAVE IMAGE --push "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" blue-build-cli-alpine: - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - FROM "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + FROM "$IMAGE:$EARTHLY_GIT_HASH-alpine-prebuild-$TARGETARCH" - IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" - ELSE - DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" - END + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="aarch64-unknown-linux-musl" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" + END - RUN mkdir -p /bluebuild - WORKDIR /bluebuild - CMD ["bluebuild"] + RUN mkdir -p /bluebuild + WORKDIR /bluebuild + CMD ["bluebuild"] - DO --pass-args +SAVE_IMAGE --SUFFIX="-alpine" + DO --pass-args +SAVE_IMAGE --SUFFIX="-alpine" installer: - ARG BASE_IMAGE="alpine" - FROM $BASE_IMAGE + ARG BASE_IMAGE="alpine" + FROM $BASE_IMAGE - COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest - LABEL org.opencontainers.image.base.name="$BASE_IMAGE" - LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" + COPY --platform=native (+digest/base-image-digest --BASE_IMAGE=$BASE_IMAGE) /base-image-digest + LABEL org.opencontainers.image.base.name="$BASE_IMAGE" + LABEL org.opencontainers.image.base.digest="$(cat /base-image-digest)" - ARG TARGETARCH - IF [ "$TARGETARCH" = "arm64" ] - DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="aarch64-unknown-linux-musl" - ELSE - DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" - END + ARG TARGETARCH + IF [ "$TARGETARCH" = "arm64" ] + DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="aarch64-unknown-linux-musl" + ELSE + DO --pass-args +INSTALL --OUT_DIR="/out/" --BUILD_TARGET="x86_64-unknown-linux-musl" + END - COPY install.sh /install.sh + COPY install.sh /install.sh - CMD ["cat", "/install.sh"] + CMD ["cat", "/install.sh"] - DO --pass-args +SAVE_IMAGE --SUFFIX="-installer" - SAVE ARTIFACT /out/bluebuild + DO --pass-args +SAVE_IMAGE --SUFFIX="-installer" + SAVE ARTIFACT /out/bluebuild cosign: - FROM gcr.io/projectsigstore/cosign - SAVE ARTIFACT /ko-app/cosign + FROM gcr.io/projectsigstore/cosign + SAVE ARTIFACT /ko-app/cosign digest: - FROM alpine - RUN apk update && apk add skopeo jq + FROM alpine + RUN apk update && apk add skopeo jq - ARG --required BASE_IMAGE - RUN skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest' > /base-image-digest - SAVE ARTIFACT /base-image-digest - + ARG --required BASE_IMAGE + RUN skopeo inspect "docker://$BASE_IMAGE" | jq -r '.Digest' > /base-image-digest + SAVE ARTIFACT /base-image-digest + version: - FROM rust + FROM rust - RUN apt-get update && apt-get install -y jq + RUN apt-get update && apt-get install -y jq - WORKDIR /app - COPY --keep-ts --dir src/ template/ recipe/ utils/ process/ /app - COPY --keep-ts Cargo.* /app + WORKDIR /app + COPY --keep-ts --dir src/ template/ recipe/ utils/ process/ /app + COPY --keep-ts Cargo.* /app - RUN /bin/bash -c 'set -eo pipefail; cargo metadata --no-deps --format-version 1 \ - | jq -r ".packages[] | select(.name == \"blue-build\") .version" > /version' + RUN /bin/bash -c 'set -eo pipefail; cargo metadata --no-deps --format-version 1 \ + | jq -r ".packages[] | select(.name == \"blue-build\") .version" > /version' - SAVE ARTIFACT /version + SAVE ARTIFACT /version INSTALL: - FUNCTION - ARG TAGGED="false" - ARG --required BUILD_TARGET - ARG --required OUT_DIR - - IF [ "$TAGGED" = "true" ] - COPY --platform=native (+install/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR - ELSE - COPY --platform=native (+install-all-features/bluebuild --BUILD_TARGET="$BUILD_TARGET") $OUT_DIR - END + FUNCTION + ARG TAGGED="false" + ARG --required BUILD_TARGET + ARG --required OUT_DIR + ARG RELEASE="true" + + IF [ "$TAGGED" = "true" ] + COPY --platform=native --pass-args +install/bluebuild $OUT_DIR + ELSE + COPY --platform=native --pass-args +install-all-features/bluebuild $OUT_DIR + END SAVE_IMAGE: - FUNCTION - ARG SUFFIX="" - ARG IMAGE="$IMAGE" - ARG TAGGED="false" - - COPY --platform=native +version/version / - ARG VERSION="$(cat /version)" - ARG MAJOR_VERSION="$(echo "$VERSION" | cut -d'.' -f1)" - ARG MINOR_VERSION="$(echo "$VERSION" | cut -d'.' -f2)" - ARG PATCH_VERSION="$(echo "$VERSION" | cut -d'.' -f3)" - ARG BUILD_TIME="$(date -Iseconds)" - DO --pass-args +LABELS - - IF [ "$TAGGED" = "true" ] - SAVE IMAGE --push "${IMAGE}:v${VERSION}${SUFFIX}" - - ARG LATEST=false - IF [ "$LATEST" = "true" ] - SAVE IMAGE --push "${IMAGE}:latest${SUFFIX}" - SAVE IMAGE --push "${IMAGE}:v${MAJOR_VERSION}.${MINOR_VERSION}${SUFFIX}" - SAVE IMAGE --push "${IMAGE}:v${MAJOR_VERSION}${SUFFIX}" - END - ELSE - ARG EARTHLY_GIT_BRANCH - SAVE IMAGE --push "${IMAGE}:${EARTHLY_GIT_BRANCH}${SUFFIX}" - END - ARG EARTHLY_GIT_HASH - SAVE IMAGE --push "${IMAGE}:${EARTHLY_GIT_HASH}${SUFFIX}" + FUNCTION + ARG SUFFIX="" + ARG IMAGE="$IMAGE" + ARG TAGGED="false" + + COPY --platform=native +version/version / + ARG VERSION="$(cat /version)" + ARG MAJOR_VERSION="$(echo "$VERSION" | cut -d'.' -f1)" + ARG MINOR_VERSION="$(echo "$VERSION" | cut -d'.' -f2)" + ARG PATCH_VERSION="$(echo "$VERSION" | cut -d'.' -f3)" + ARG BUILD_TIME="$(date -Iseconds)" + DO --pass-args +LABELS + + IF [ "$TAGGED" = "true" ] + SAVE IMAGE --push "${IMAGE}:v${VERSION}${SUFFIX}" + + ARG LATEST=false + IF [ "$LATEST" = "true" ] + SAVE IMAGE --push "${IMAGE}:latest${SUFFIX}" + SAVE IMAGE --push "${IMAGE}:v${MAJOR_VERSION}.${MINOR_VERSION}${SUFFIX}" + SAVE IMAGE --push "${IMAGE}:v${MAJOR_VERSION}${SUFFIX}" + END + ELSE + ARG EARTHLY_GIT_BRANCH + SAVE IMAGE --push "${IMAGE}:${EARTHLY_GIT_BRANCH}${SUFFIX}" + END + ARG EARTHLY_GIT_HASH + SAVE IMAGE --push "${IMAGE}:${EARTHLY_GIT_HASH}${SUFFIX}" LABELS: - FUNCTION - LABEL org.opencontainers.image.created="$BUILD_TIME" - LABEL org.opencontainers.image.url="https://github.com/blue-build/cli" - LABEL org.opencontainers.image.source="https://github.com/blue-build/cli" - LABEL org.opencontainers.image.version="$VERSION" - LABEL version="$VERSION" - LABEL org.opencontainers.image.vendor="BlueBuild" - LABEL vendor="BlueBuild" - LABEL org.opencontainers.image.licenses="Apache-2.0" - LABEL license="Apache-2.0" - LABEL org.opencontainers.image.title="BlueBuild CLI tool" - LABEL name="blue-build/cli" - LABEL org.opencontainers.image.description="A CLI tool built for creating Containerfile templates for ostree based atomic distros" - LABEL org.opencontainers.image.documentation="https://raw.githubusercontent.com/blue-build/cli/main/README.md" - - ARG TAGGED="false" - IF [ "$TAGGED" = "true" ] - ARG EARTHLY_GIT_BRANCH - LABEL org.opencontainers.image.ref.name="$EARTHLY_GIT_BRANCH" - ELSE - LABEL org.opencontainers.image.ref.name="v$VERSION" - END + FUNCTION + LABEL org.opencontainers.image.created="$BUILD_TIME" + LABEL org.opencontainers.image.url="https://github.com/blue-build/cli" + LABEL org.opencontainers.image.source="https://github.com/blue-build/cli" + LABEL org.opencontainers.image.version="$VERSION" + LABEL version="$VERSION" + LABEL org.opencontainers.image.vendor="BlueBuild" + LABEL vendor="BlueBuild" + LABEL org.opencontainers.image.licenses="Apache-2.0" + LABEL license="Apache-2.0" + LABEL org.opencontainers.image.title="BlueBuild CLI tool" + LABEL name="blue-build/cli" + LABEL org.opencontainers.image.description="A CLI tool built for creating Containerfile templates for ostree based atomic distros" + LABEL org.opencontainers.image.documentation="https://raw.githubusercontent.com/blue-build/cli/main/README.md" + + ARG TAGGED="false" + IF [ "$TAGGED" = "true" ] + ARG EARTHLY_GIT_BRANCH + LABEL org.opencontainers.image.ref.name="$EARTHLY_GIT_BRANCH" + ELSE + LABEL org.opencontainers.image.ref.name="v$VERSION" + END diff --git a/bacon.toml b/bacon.toml index c2b2ae81..b5c835ac 100644 --- a/bacon.toml +++ b/bacon.toml @@ -89,6 +89,9 @@ watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.r # should go in your personal global prefs.toml file instead. [keybindings] # alt-m = "job:my-job" -c = "job:clippy-all" # comment this to have 'c' run clippy on only the default target -i = "job:install-all" -t = "job:test-all" +c = "job:clippy" +shift-c = "job:clippy-all" # comment this to have 'c' run clippy on only the default target +i = "job:install" +shift-i = "job:install-all" +t = "job:test" +shift-t = "job:test-all" diff --git a/integration-tests/Earthfile b/integration-tests/Earthfile index 38055a7a..1e4b0974 100644 --- a/integration-tests/Earthfile +++ b/integration-tests/Earthfile @@ -4,126 +4,137 @@ PROJECT blue-build/cli IMPORT github.com/earthly/lib/utils/dind AS dind all: - BUILD +test-image - BUILD +test-legacy-image - BUILD +build - BUILD +rebase - BUILD +upgrade - BUILD +switch + BUILD +test-image + BUILD +test-legacy-image + BUILD +build + BUILD +rebase + BUILD +upgrade + BUILD +switch + BUILD +validate test-image: - FROM +build-template --src=template-containerfile - WORKDIR /tmp/test - COPY ./test-scripts/*.sh ./ + FROM +build-template --src=template-containerfile + WORKDIR /tmp/test + COPY ./test-scripts/*.sh ./ - DO +RUN_TESTS + DO +RUN_TESTS test-legacy-image: - FROM +build-template --src=template-legacy-containerfile - WORKDIR /tmp/test - COPY ./test-scripts/*.sh ./ + FROM +build-template --src=template-legacy-containerfile + WORKDIR /tmp/test + COPY ./test-scripts/*.sh ./ - DO +RUN_TESTS + DO +RUN_TESTS build-template: - ARG --required src - FROM DOCKERFILE \ - -f +$src/test/Containerfile \ - +$src/test/* + ARG --required src + FROM DOCKERFILE \ + -f +$src/test/Containerfile \ + +$src/test/* template-containerfile: - FROM +test-base - RUN bluebuild -v generate recipes/recipe.yml | tee Containerfile + FROM +test-base + RUN --no-cache bluebuild -v generate recipes/recipe.yml | tee Containerfile - SAVE ARTIFACT /test + SAVE ARTIFACT /test template-legacy-containerfile: - FROM +legacy-base - RUN bluebuild -v template config/recipe.yml | tee Containerfile + FROM +legacy-base + RUN --no-cache bluebuild -v template config/recipe.yml | tee Containerfile - SAVE ARTIFACT /test + SAVE ARTIFACT /test build: - FROM +test-base + FROM +test-base - RUN bluebuild -v build recipes/recipe.yml + RUN bluebuild -v build recipes/recipe.yml build-full: - FROM +test-base --MOCK="false" + FROM +test-base --MOCK="false" - DO dind+INSTALL + DO dind+INSTALL - ENV BB_USERNAME=gmpinder - ENV BB_REGISTRY=ghcr.io - ENV BB_REGISTRY_NAMESPACE=blue-build + ENV BB_USERNAME=gmpinder + ENV BB_REGISTRY=ghcr.io + ENV BB_REGISTRY_NAMESPACE=blue-build - WITH DOCKER - RUN --secret BB_PASSWORD=github/registry bluebuild build --push -S sigstore -vv recipes/recipe.yml - END - + WITH DOCKER + RUN --secret BB_PASSWORD=github/registry bluebuild build --push -S sigstore -vv recipes/recipe.yml + END + rebase: - FROM +legacy-base + FROM +legacy-base - RUN bluebuild -v rebase config/recipe.yml + RUN --no-cache bluebuild -v rebase config/recipe.yml upgrade: - FROM +legacy-base + FROM +legacy-base - RUN mkdir -p /etc/bluebuild && touch $BB_TEST_LOCAL_IMAGE - RUN bluebuild -v upgrade config/recipe.yml + RUN mkdir -p /etc/bluebuild && touch $BB_TEST_LOCAL_IMAGE + RUN --no-cache bluebuild -v upgrade config/recipe.yml switch: - FROM +test-base + FROM +test-base - RUN mkdir -p /etc/bluebuild && touch $BB_TEST_LOCAL_IMAGE - RUN bluebuild -v switch recipes/recipe.yml + RUN mkdir -p /etc/bluebuild && touch $BB_TEST_LOCAL_IMAGE + RUN --no-cache bluebuild -v switch recipes/recipe.yml + +validate: + FROM +test-base + + RUN --no-cache bluebuild -v validate recipes/recipe.yml + RUN --no-cache bluebuild -v validate recipes/recipe-39.yml + RUN --no-cache bluebuild -v validate recipes/recipe-arm64.yml + RUN --no-cache bluebuild -v validate recipes/recipe-invalid.yml && exit 1 || exit 0 + RUN --no-cache bluebuild -v validate recipes/recipe-invalid-module.yml && exit 1 || exit 0 + RUN --no-cache bluebuild -v validate recipes/recipe-invalid-stage.yml && exit 1 || exit 0 legacy-base: - FROM ../+blue-build-cli-alpine - RUN apk update --no-cache && apk add bash grep jq sudo coreutils - ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test-legacy.tar.gz - ENV CLICOLOR_FORCE=1 + FROM ../+blue-build-cli-alpine --RELEASE=false + RUN apk update --no-cache && apk add bash grep jq sudo coreutils + ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test-legacy.tar.gz + ENV CLICOLOR_FORCE=1 - COPY ./mock-scripts/ /usr/bin/ + COPY ./mock-scripts/ /usr/bin/ - WORKDIR /test - COPY ./legacy-test-repo /test + WORKDIR /test + COPY ./legacy-test-repo /test - DO ../+INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" --TAGGED="true" + DO ../+INSTALL --OUT_DIR="/usr/bin/" --BUILD_TARGET="x86_64-unknown-linux-musl" --TAGGED="true" - DO +GEN_KEYPAIR + DO +GEN_KEYPAIR test-base: - FROM ../+blue-build-cli-alpine - RUN apk update --no-cache && apk add bash grep jq sudo coreutils - ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test.tar.gz - ENV CLICOLOR_FORCE=1 + FROM ../+blue-build-cli-alpine --RELEASE=false + RUN apk update --no-cache && apk add bash grep jq sudo coreutils + ENV BB_TEST_LOCAL_IMAGE=/etc/bluebuild/cli_test.tar.gz + ENV CLICOLOR_FORCE=1 - ARG MOCK="true" - IF [ "$MOCK" = "true" ] - COPY ./mock-scripts/ /usr/bin/ - END + ARG MOCK="true" + IF [ "$MOCK" = "true" ] + COPY ./mock-scripts/ /usr/bin/ + END - WORKDIR /test - COPY ./test-repo /test + WORKDIR /test + COPY ./test-repo /test - DO +GEN_KEYPAIR + DO +GEN_KEYPAIR GEN_KEYPAIR: - FUNCTION - # Setup a cosign key pair - ENV COSIGN_PASSWORD="" - ENV COSIGN_YES="true" - RUN cosign generate-key-pair - ENV COSIGN_PRIVATE_KEY=$(cat cosign.key) - RUN rm cosign.key + FUNCTION + # Setup a cosign key pair + ENV COSIGN_PASSWORD="" + ENV COSIGN_YES="true" + RUN cosign generate-key-pair + ENV COSIGN_PRIVATE_KEY=$(cat cosign.key) + RUN rm cosign.key RUN_TESTS: - FUNCTION - FOR script IN $(ls *.sh) - RUN --no-cache chmod +x $script \ - && echo "Running test $script" \ - && ./$script - END + FUNCTION + FOR script IN $(ls *.sh) + RUN --no-cache chmod +x $script \ + && echo "Running test $script" \ + && ./$script + END diff --git a/integration-tests/legacy-test-repo/config/recipe.yml b/integration-tests/legacy-test-repo/config/recipe.yml index e6943995..c8b48de5 100644 --- a/integration-tests/legacy-test-repo/config/recipe.yml +++ b/integration-tests/legacy-test-repo/config/recipe.yml @@ -35,6 +35,7 @@ modules: - type: signing - type: test-module + source: local - type: containerfile containerfiles: diff --git a/integration-tests/test-repo/recipes/akmods.yml b/integration-tests/test-repo/recipes/akmods.yml index 10e11054..33382992 100644 --- a/integration-tests/test-repo/recipes/akmods.yml +++ b/integration-tests/test-repo/recipes/akmods.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/module-list-v1.json modules: # Tests installing rpms from a combo image stage - type: akmods diff --git a/integration-tests/test-repo/recipes/bluebuild.yml b/integration-tests/test-repo/recipes/bluebuild.yml index 29595601..6bcb1e9d 100644 --- a/integration-tests/test-repo/recipes/bluebuild.yml +++ b/integration-tests/test-repo/recipes/bluebuild.yml @@ -1,6 +1,8 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/module-stage-list-v1.json stages: - name: blue-build - image: rust + from: rust modules: - type: script scripts: diff --git a/integration-tests/test-repo/recipes/flatpaks.yml b/integration-tests/test-repo/recipes/flatpaks.yml index e32cc8b4..39de41ce 100644 --- a/integration-tests/test-repo/recipes/flatpaks.yml +++ b/integration-tests/test-repo/recipes/flatpaks.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/module-v1.json type: default-flatpaks notify: true system: diff --git a/integration-tests/test-repo/recipes/recipe-39.yml b/integration-tests/test-repo/recipes/recipe-39.yml index 9ad3c499..366c6bed 100644 --- a/integration-tests/test-repo/recipes/recipe-39.yml +++ b/integration-tests/test-repo/recipes/recipe-39.yml @@ -33,6 +33,7 @@ modules: - type: signing - type: test-module + source: local - type: containerfile containerfiles: diff --git a/integration-tests/test-repo/recipes/recipe-arm64.yml b/integration-tests/test-repo/recipes/recipe-arm64.yml index 66539261..b22589b2 100644 --- a/integration-tests/test-repo/recipes/recipe-arm64.yml +++ b/integration-tests/test-repo/recipes/recipe-arm64.yml @@ -1,10 +1,12 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json name: cli/test description: This is my personal OS image. base-image: quay.io/fedora/fedora-silverblue image-version: 40 -alt_tags: +alt-tags: - arm64 -stages: +stages: [] modules: - from-file: flatpaks.yml @@ -29,6 +31,7 @@ modules: - type: signing - type: test-module + source: local - type: containerfile containerfiles: diff --git a/integration-tests/test-repo/recipes/recipe-invalid-module.yml b/integration-tests/test-repo/recipes/recipe-invalid-module.yml new file mode 100644 index 00000000..f2ec2493 --- /dev/null +++ b/integration-tests/test-repo/recipes/recipe-invalid-module.yml @@ -0,0 +1,57 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json +name: cli/test-invalid-module +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-main +image-version: 40 +stages: + - from-file: stages.yml +modules: + - from-file: akmods.yml + - from-file: flatpaks.yml + + - type: files + files: + - source: usr + destination: /usr + + - type: script + scripts: + - example.sh + + - type: rpm-ostree + repos: + - https://copr.fedorainfracloud.org/coprs/atim/starship/repo/fedora-%OS_VERSION%/atim-starship-fedora-%OS_VERSION%.repo + install: micro + installer: test + remove: + - firefox + - firefox-langpacks + + - type: signing + + - type: test-module + source: local + + - type: containerfile + containerfiles: + labels: labels + snippets: + - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/integration-tests/test-repo/recipes/recipe-invalid-stage.yml b/integration-tests/test-repo/recipes/recipe-invalid-stage.yml new file mode 100644 index 00000000..4398e53f --- /dev/null +++ b/integration-tests/test-repo/recipes/recipe-invalid-stage.yml @@ -0,0 +1,61 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json +name: cli/test-invalid-stage +description: This is my personal OS image. +base-image: ghcr.io/ublue-os/silverblue-main +image-version: 40 +stages: + - name: ubuntu-test + from: + - ubuntu + modules: {} +modules: + - from-file: akmods.yml + - from-file: flatpaks.yml + + - type: files + files: + - source: usr + destination: /usr + + - type: script + scripts: + - example.sh + + - type: rpm-ostree + repos: + - https://copr.fedorainfracloud.org/coprs/atim/starship/repo/fedora-%OS_VERSION%/atim-starship-fedora-%OS_VERSION%.repo + install: + - micro + - starship + remove: + - firefox + - firefox-langpacks + + - type: signing + + - type: test-module + source: local + + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/integration-tests/test-repo/recipes/recipe-invalid.yml b/integration-tests/test-repo/recipes/recipe-invalid.yml new file mode 100644 index 00000000..9f29c46e --- /dev/null +++ b/integration-tests/test-repo/recipes/recipe-invalid.yml @@ -0,0 +1,59 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json +name: cli/test-invalid +description: 10 +base-image: ghcr.io/ublue-os/silverblue-main +image-version: + - 40 + - 39 +stages: {} +modules: + - from-file: akmods.yml + - from-file: flatpaks.yml + + - type: files + files: + - source: usr + destination: /usr + + - type: script + scripts: + - example.sh + + - type: rpm-ostree + repos: + - https://copr.fedorainfracloud.org/coprs/atim/starship/repo/fedora-%OS_VERSION%/atim-starship-fedora-%OS_VERSION%.repo + install: + - micro + - starship + remove: + - firefox + - firefox-langpacks + + - type: signing + + - type: test-module + source: local + + - type: containerfile + containerfiles: + - labels + snippets: + - RUN echo "This is a snippet" && ostree container commit + + - type: copy + from: alpine-test + src: /test.txt + dest: / + - type: copy + from: ubuntu-test + src: /test.txt + dest: / + - type: copy + from: debian-test + src: /test.txt + dest: / + - type: copy + from: fedora-test + src: /test.txt + dest: / diff --git a/integration-tests/test-repo/recipes/recipe.yml b/integration-tests/test-repo/recipes/recipe.yml index 19e23da8..dcf7ec7f 100644 --- a/integration-tests/test-repo/recipes/recipe.yml +++ b/integration-tests/test-repo/recipes/recipe.yml @@ -32,6 +32,7 @@ modules: - type: signing - type: test-module + source: local - type: containerfile containerfiles: diff --git a/integration-tests/test-repo/recipes/stages.yml b/integration-tests/test-repo/recipes/stages.yml index e2851c52..31ea534f 100644 --- a/integration-tests/test-repo/recipes/stages.yml +++ b/integration-tests/test-repo/recipes/stages.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/module-stage-list-v1.json stages: - name: ubuntu-test from: ubuntu @@ -25,6 +27,7 @@ modules: snippets: - echo "test" > /test.txt - type: test-module + source: local - type: containerfile containerfiles: - labels diff --git a/process/Cargo.toml b/process/Cargo.toml index b722c02f..d1968b03 100644 --- a/process/Cargo.toml +++ b/process/Cargo.toml @@ -24,7 +24,6 @@ rand = "0.8" semver = { version = "1", features = ["serde"] } signal-hook = { version = "0.3", features = ["extended-siginfo"] } sigstore = { version = "0.10", features = ["full-rustls-tls", "cached-client", "sigstore-trust-root", "sign"], default-features = false, optional = true } -tokio = { version = "1.39.2", features = ["rt", "rt-multi-thread"], optional = true } zeroize = { version = "1", features = ["aarch64", "derive", "serde"] } cached.workspace = true @@ -36,9 +35,11 @@ indexmap.workspace = true log.workspace = true miette.workspace = true oci-distribution.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true tempdir.workspace = true +tokio = { workspace = true, optional = true } bon.workspace = true users.workspace = true uuid.workspace = true @@ -52,3 +53,4 @@ workspace = true [features] sigstore = ["dep:tokio", "dep:sigstore"] +validate = ["dep:tokio"] diff --git a/process/drivers.rs b/process/drivers.rs index 4db0523a..d1948ba1 100644 --- a/process/drivers.rs +++ b/process/drivers.rs @@ -1,5 +1,7 @@ //! This module is responsible for managing various strategies -//! to perform actions throughout the program. This hides all +//! to perform actions throughout the program. +//! +//! This hides all //! the implementation details from the command logic and allows //! for caching certain long execution tasks like inspecting the //! labels for an image. diff --git a/process/drivers/sigstore_driver.rs b/process/drivers/sigstore_driver.rs index 5b93a934..611120fd 100644 --- a/process/drivers/sigstore_driver.rs +++ b/process/drivers/sigstore_driver.rs @@ -2,7 +2,7 @@ use std::{fs, path::Path}; use crate::{ drivers::opts::{PrivateKeyContents, VerifyType}, - RT, + ASYNC_RUNTIME, }; use super::{ @@ -135,7 +135,8 @@ impl SigningDriver for SigstoreDriver { debug!("Credentials retrieved"); let (cosign_signature_image, source_image_digest) = retry(2, 5, || { - RT.block_on(client.triangulate(&image_digest, &auth)) + ASYNC_RUNTIME + .block_on(client.triangulate(&image_digest, &auth)) .into_diagnostic() .with_context(|| format!("Failed to triangulate image {image_digest}")) })?; @@ -151,18 +152,19 @@ impl SigningDriver for SigstoreDriver { debug!("Pushing signature"); retry(2, 5, || { - RT.block_on(client.push_signature( - None, - &auth, - &cosign_signature_image, - vec![signature_layer.clone()], - )) - .into_diagnostic() - .with_context(|| { - format!( + ASYNC_RUNTIME + .block_on(client.push_signature( + None, + &auth, + &cosign_signature_image, + vec![signature_layer.clone()], + )) + .into_diagnostic() + .with_context(|| { + format!( "Failed to push signature {cosign_signature_image} for image {image_digest}" ) - }) + }) })?; debug!("Successfully pushed signature"); @@ -196,19 +198,21 @@ impl SigningDriver for SigstoreDriver { debug!("Triangulating image"); let auth = Auth::Anonymous; let (cosign_signature_image, source_image_digest) = retry(2, 5, || { - RT.block_on(client.triangulate(&image_digest, &auth)) + ASYNC_RUNTIME + .block_on(client.triangulate(&image_digest, &auth)) .into_diagnostic() .with_context(|| format!("Failed to triangulate image {image_digest}")) })?; trace!("{cosign_signature_image}, {source_image_digest}"); let trusted_layers = retry(2, 5, || { - RT.block_on(client.trusted_signature_layers( - &auth, - &source_image_digest, - &cosign_signature_image, - )) - .into_diagnostic() + ASYNC_RUNTIME + .block_on(client.trusted_signature_layers( + &auth, + &source_image_digest, + &cosign_signature_image, + )) + .into_diagnostic() })?; sigstore::cosign::verify_constraints(&trusted_layers, verification_constraints.iter()) diff --git a/process/process.rs b/process/process.rs index 100bc6ee..693c1bad 100644 --- a/process/process.rs +++ b/process/process.rs @@ -2,17 +2,17 @@ //! by this tool. It contains drivers for running, building, inspecting, and signing //! images that interface with tools like docker or podman. -#[cfg(feature = "sigstore")] +#[cfg(any(feature = "sigstore", feature = "validate"))] use once_cell::sync::Lazy; -#[cfg(feature = "sigstore")] +#[cfg(any(feature = "sigstore", feature = "validate"))] use tokio::runtime::Runtime; pub mod drivers; pub mod logging; pub mod signal_handler; -#[cfg(feature = "sigstore")] -pub(crate) static RT: Lazy = Lazy::new(|| { +#[cfg(any(feature = "sigstore", feature = "validate"))] +pub static ASYNC_RUNTIME: Lazy = Lazy::new(|| { tokio::runtime::Builder::new_multi_thread() .enable_all() .build() diff --git a/recipe/src/lib.rs b/recipe/src/lib.rs index 4d041191..4d685cbc 100644 --- a/recipe/src/lib.rs +++ b/recipe/src/lib.rs @@ -5,9 +5,32 @@ pub mod recipe; pub mod stage; pub mod stages_ext; +use std::path::{Path, PathBuf}; + +use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; +use log::warn; + pub use akmods_info::*; pub use module::*; pub use module_ext::*; pub use recipe::*; pub use stage::*; pub use stages_ext::*; + +pub trait FromFileList { + const LIST_KEY: &str; + + fn get_from_file_paths(&self) -> Vec; +} + +pub(crate) fn base_recipe_path() -> &'static Path { + let legacy_path = Path::new(CONFIG_PATH); + let recipe_path = Path::new(RECIPE_PATH); + + if recipe_path.exists() && recipe_path.is_dir() { + recipe_path + } else { + warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); + legacy_path + } +} diff --git a/recipe/src/module.rs b/recipe/src/module.rs index ddfb390b..39b575b4 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -9,7 +9,7 @@ use miette::{bail, Result}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; -use crate::{AkmodsInfo, ModuleExt}; +use crate::{base_recipe_path, AkmodsInfo, ModuleExt}; #[derive(Serialize, Deserialize, Debug, Clone, Builder, Default)] pub struct ModuleRequiredFields<'a> { @@ -80,6 +80,17 @@ impl<'a> ModuleRequiredFields<'a> { } } + #[must_use] + pub fn get_non_local_source(&'a self) -> Option<&'a str> { + let source = self.source.as_deref()?; + + if source == "local" { + None + } else { + Some(source) + } + } + #[must_use] pub fn generate_akmods_info(&'a self, os_version: &u64) -> AkmodsInfo { #[derive(Debug, Copy, Clone)] @@ -164,7 +175,7 @@ pub struct Module<'a> { pub from_file: Option>, } -impl<'a> Module<'a> { +impl Module<'_> { /// Get's any child modules. /// /// # Errors @@ -202,7 +213,7 @@ impl<'a> Module<'a> { traversed_files.push(file_name.clone()); Self::get_modules( - &ModuleExt::parse(&file_name)?.modules, + &ModuleExt::try_from(&file_name)?.modules, Some(traversed_files), )? } @@ -224,6 +235,13 @@ impl<'a> Module<'a> { Ok(found_modules) } + #[must_use] + pub fn get_from_file_path(&self) -> Option { + self.from_file + .as_ref() + .map(|path| base_recipe_path().join(&**path)) + } + #[must_use] pub fn example() -> Self { Self::builder() diff --git a/recipe/src/module_ext.rs b/recipe/src/module_ext.rs index 2e45dab6..5e871345 100644 --- a/recipe/src/module_ext.rs +++ b/recipe/src/module_ext.rs @@ -1,12 +1,15 @@ -use std::{collections::HashSet, fs, path::Path}; +use std::{ + collections::HashSet, + fs, + path::{Path, PathBuf}, +}; -use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; use bon::Builder; -use log::{trace, warn}; -use miette::{Context, IntoDiagnostic, Result}; +use log::trace; +use miette::{Context, IntoDiagnostic, Report, Result}; use serde::{Deserialize, Serialize}; -use crate::{AkmodsInfo, Module}; +use crate::{base_recipe_path, AkmodsInfo, FromFileList, Module}; #[derive(Default, Serialize, Clone, Deserialize, Debug, Builder)] pub struct ModuleExt<'a> { @@ -14,22 +17,31 @@ pub struct ModuleExt<'a> { pub modules: Vec>, } -impl ModuleExt<'_> { - /// Parse a module file returning a [`ModuleExt`] - /// - /// # Errors - /// Can return an `anyhow` Error if the file cannot be read or deserialized - /// into a [`ModuleExt`] - pub fn parse(file_name: &Path) -> Result { - let legacy_path = Path::new(CONFIG_PATH); - let recipe_path = Path::new(RECIPE_PATH); +impl FromFileList for ModuleExt<'_> { + const LIST_KEY: &'static str = "modules"; + + #[must_use] + fn get_from_file_paths(&self) -> Vec { + self.modules + .iter() + .filter_map(Module::get_from_file_path) + .collect() + } +} - let file_path = if recipe_path.exists() && recipe_path.is_dir() { - recipe_path.join(file_name) - } else { - warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); - legacy_path.join(file_name) - }; +impl TryFrom<&PathBuf> for ModuleExt<'_> { + type Error = Report; + + fn try_from(value: &PathBuf) -> std::result::Result { + Self::try_from(value.as_path()) + } +} + +impl TryFrom<&Path> for ModuleExt<'_> { + type Error = Report; + + fn try_from(file_name: &Path) -> Result { + let file_path = base_recipe_path().join(file_name); let file = fs::read_to_string(&file_path) .into_diagnostic() @@ -45,7 +57,9 @@ impl ModuleExt<'_> { Ok, ) } +} +impl ModuleExt<'_> { #[must_use] pub fn get_akmods_info_list(&self, os_version: &u64) -> Vec { trace!("get_akmods_image_list({self:#?}, {os_version})"); diff --git a/recipe/src/recipe.rs b/recipe/src/recipe.rs index 4fe54ee5..122d75d3 100644 --- a/recipe/src/recipe.rs +++ b/recipe/src/recipe.rs @@ -1,12 +1,10 @@ use std::{borrow::Cow, fs, path::Path}; use bon::Builder; -use indexmap::IndexMap; use log::{debug, trace}; use miette::{Context, IntoDiagnostic, Result}; use oci_distribution::Reference; use serde::{Deserialize, Serialize}; -use serde_yaml::Value; use crate::{Module, ModuleExt, StagesExt}; @@ -69,13 +67,6 @@ pub struct Recipe<'a> { /// This holds the list of modules to be run on the image. #[serde(flatten)] pub modules_ext: ModuleExt<'a>, - - /// Extra data that the user might have added. This is - /// done in case we serialize the data to a yaml file - /// so that we retain any unused information. - #[serde(flatten)] - #[builder(into)] - pub extra: IndexMap, } impl<'a> Recipe<'a> { diff --git a/recipe/src/stage.rs b/recipe/src/stage.rs index dff8875b..0b4bb6b4 100644 --- a/recipe/src/stage.rs +++ b/recipe/src/stage.rs @@ -6,7 +6,7 @@ use colored::Colorize; use miette::{bail, Result}; use serde::{Deserialize, Serialize}; -use crate::{Module, ModuleExt, StagesExt}; +use crate::{base_recipe_path, Module, ModuleExt, StagesExt}; /// Contains the required fields for a stage. #[derive(Serialize, Deserialize, Debug, Clone, Builder)] @@ -86,7 +86,7 @@ pub struct Stage<'a> { pub from_file: Option>, } -impl<'a> Stage<'a> { +impl Stage<'_> { /// Get's any child stages. /// /// # Errors @@ -119,7 +119,7 @@ impl<'a> Stage<'a> { let mut tf = traversed_files.clone(); tf.push(file_name.clone()); - Self::get_stages(&StagesExt::parse(&file_name)?.stages, Some(tf))? + Self::get_stages(&StagesExt::try_from(&file_name)?.stages, Some(tf))? } _ => { let from_example = Stage::builder().from_file("path/to/stage.yml").build(); @@ -139,6 +139,13 @@ impl<'a> Stage<'a> { Ok(found_stages) } + #[must_use] + pub fn get_from_file_path(&self) -> Option { + self.from_file + .as_ref() + .map(|path| base_recipe_path().join(&**path)) + } + #[must_use] pub fn example() -> Self { Stage::builder() diff --git a/recipe/src/stages_ext.rs b/recipe/src/stages_ext.rs index ae8ebd7a..a9a5333c 100644 --- a/recipe/src/stages_ext.rs +++ b/recipe/src/stages_ext.rs @@ -1,12 +1,13 @@ -use std::{fs, path::Path}; +use std::{ + fs, + path::{Path, PathBuf}, +}; -use blue_build_utils::constants::{CONFIG_PATH, RECIPE_PATH}; use bon::Builder; -use log::warn; -use miette::{Context, IntoDiagnostic, Result}; +use miette::{Context, IntoDiagnostic, Report, Result}; use serde::{Deserialize, Serialize}; -use crate::{Module, Stage}; +use crate::{base_recipe_path, FromFileList, Module, Stage}; #[derive(Default, Serialize, Clone, Deserialize, Debug, Builder)] pub struct StagesExt<'a> { @@ -14,22 +15,31 @@ pub struct StagesExt<'a> { pub stages: Vec>, } -impl<'a> StagesExt<'a> { - /// Parse a module file returning a [`StagesExt`] - /// - /// # Errors - /// Can return an `anyhow` Error if the file cannot be read or deserialized - /// into a [`StagesExt`] - pub fn parse(file_name: &Path) -> Result { - let legacy_path = Path::new(CONFIG_PATH); - let recipe_path = Path::new(RECIPE_PATH); - - let file_path = if recipe_path.exists() && recipe_path.is_dir() { - recipe_path.join(file_name) - } else { - warn!("Use of {CONFIG_PATH} for recipes is deprecated, please move your recipe files into {RECIPE_PATH}"); - legacy_path.join(file_name) - }; +impl FromFileList for StagesExt<'_> { + const LIST_KEY: &'static str = "stages"; + + #[must_use] + fn get_from_file_paths(&self) -> Vec { + self.stages + .iter() + .filter_map(Stage::get_from_file_path) + .collect() + } +} + +impl TryFrom<&PathBuf> for StagesExt<'_> { + type Error = Report; + + fn try_from(value: &PathBuf) -> Result { + Self::try_from(value.as_path()) + } +} + +impl TryFrom<&Path> for StagesExt<'_> { + type Error = Report; + + fn try_from(file_name: &Path) -> Result { + let file_path = base_recipe_path().join(file_name); let file = fs::read_to_string(&file_path) .into_diagnostic() diff --git a/src/bin/bluebuild.rs b/src/bin/bluebuild.rs index 06776492..879af089 100644 --- a/src/bin/bluebuild.rs +++ b/src/bin/bluebuild.rs @@ -12,6 +12,7 @@ fn main() { ("hyper::proto", LevelFilter::Off), ("hyper_util", LevelFilter::Off), ("oci_distribution", LevelFilter::Off), + ("reqwest", LevelFilter::Off), ]) .log_out_dir(args.log_out.clone()) .init(); @@ -42,6 +43,9 @@ fn main() { #[cfg(feature = "iso")] CommandArgs::GenerateIso(mut command) => command.run(), + #[cfg(feature = "validate")] + CommandArgs::Validate(mut command) => command.run(), + CommandArgs::BugReport(mut command) => command.run(), CommandArgs::Completions(mut command) => command.run(), diff --git a/src/commands.rs b/src/commands.rs index 5c6c2b97..a95dbe49 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -5,7 +5,7 @@ use log::error; use clap::{command, crate_authors, Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; -use crate::shadow; +use crate::info::shadow; pub mod bug_report; pub mod build; @@ -15,6 +15,8 @@ pub mod generate; pub mod generate_iso; #[cfg(feature = "login")] pub mod login; +#[cfg(feature = "validate")] +pub mod validate; // #[cfg(feature = "init")] // pub mod init; #[cfg(not(feature = "switch"))] @@ -115,6 +117,11 @@ pub enum CommandArgs { #[cfg(feature = "login")] Login(login::LoginCommand), + /// Validate your recipe file and display + /// errors to help fix problems. + #[cfg(feature = "validate")] + Validate(Box), + // /// Initialize a new Ublue Starting Point repo // #[cfg(feature = "init")] // Init(init::InitCommand), diff --git a/src/commands/bug_report.rs b/src/commands/bug_report.rs index ebbe7787..1552fde2 100644 --- a/src/commands/bug_report.rs +++ b/src/commands/bug_report.rs @@ -16,7 +16,7 @@ use std::time::Duration; use super::BlueBuildCommand; -use crate::shadow; +use crate::info::shadow; #[derive(Default, Debug, Clone, Builder, Args)] pub struct BugReportRecipe { diff --git a/src/commands/generate.rs b/src/commands/generate.rs index 95736771..bf61f1f2 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -18,7 +18,9 @@ use clap::{crate_version, Args}; use log::{debug, info, trace, warn}; use miette::{IntoDiagnostic, Result}; -use crate::shadow; +#[cfg(feature = "validate")] +use crate::commands::validate::ValidateCommand; +use crate::info::shadow; use super::BlueBuildCommand; @@ -99,6 +101,13 @@ impl GenerateCommand { legacy_path.join(RECIPE_FILE) } }); + + #[cfg(feature = "validate")] + ValidateCommand::builder() + .recipe(recipe_path.clone()) + .build() + .try_run()?; + let registry = if let (Some(registry), Some(registry_namespace)) = (&self.registry, &self.registry_namespace) { diff --git a/src/commands/validate.rs b/src/commands/validate.rs new file mode 100644 index 00000000..f19013d7 --- /dev/null +++ b/src/commands/validate.rs @@ -0,0 +1,405 @@ +use std::{ + fs::OpenOptions, + io::{BufReader, Read}, + path::{Path, PathBuf}, +}; + +use blue_build_process_management::ASYNC_RUNTIME; +use blue_build_recipe::{FromFileList, ModuleExt, Recipe, StagesExt}; +use blue_build_utils::{ + string, + syntax_highlighting::{self}, +}; +use bon::Builder; +use clap::Args; +use colored::Colorize; +use indexmap::IndexMap; +use jsonschema::{BasicOutput, ValidationError}; +use log::{debug, info, trace}; +use miette::{bail, miette, Context, IntoDiagnostic, Report}; +use rayon::prelude::*; +use schema_validator::{ + build_validator, SchemaValidator, MODULE_LIST_V1_SCHEMA_URL, MODULE_V1_SCHEMA_URL, + RECIPE_V1_SCHEMA_URL, STAGE_LIST_V1_SCHEMA_URL, STAGE_V1_SCHEMA_URL, +}; +use serde::de::DeserializeOwned; +use serde_json::Value; + +use super::BlueBuildCommand; + +mod schema_validator; + +#[derive(Debug, Args, Builder)] +pub struct ValidateCommand { + /// The path to the recipe. + /// + /// NOTE: In order for this to work, + /// you must be in the root of your + /// bluebuild repository. + pub recipe: PathBuf, + + /// Display all errors that failed + /// validation of the recipe. + #[arg(short, long)] + #[builder(default)] + pub all_errors: bool, + + #[clap(skip)] + recipe_validator: Option, + + #[clap(skip)] + stage_validator: Option, + + #[clap(skip)] + stage_list_validator: Option, + + #[clap(skip)] + module_validator: Option, + + #[clap(skip)] + module_list_validator: Option, +} + +impl BlueBuildCommand for ValidateCommand { + fn try_run(&mut self) -> miette::Result<()> { + let recipe_path_display = self.recipe.display().to_string().bold().italic(); + + if !self.recipe.is_file() { + bail!("File {recipe_path_display} must exist"); + } + + ASYNC_RUNTIME.block_on(self.setup_validators())?; + + if let Err(errors) = self.validate_recipe() { + let errors = errors.into_iter().fold(String::new(), |mut full, err| { + full.push_str(&format!("{err:?}")); + full + }); + + if self.all_errors { + bail!("Recipe {recipe_path_display} failed to validate:\n{errors}"); + } else { + bail!( + help = format!( + "Use `{}` to view more information", + format!("bluebuild validate --all-errors {}", self.recipe.display()).bold(), + ), + "Recipe {recipe_path_display} failed to validate:\n{errors}", + ); + } + } + info!("Recipe {recipe_path_display} is valid"); + + Ok(()) + } +} + +impl ValidateCommand { + async fn setup_validators(&mut self) -> Result<(), Report> { + let (rv, sv, slv, mv, mlv) = tokio::try_join!( + build_validator(RECIPE_V1_SCHEMA_URL), + build_validator(STAGE_V1_SCHEMA_URL), + build_validator(STAGE_LIST_V1_SCHEMA_URL), + build_validator(MODULE_V1_SCHEMA_URL), + build_validator(MODULE_LIST_V1_SCHEMA_URL), + )?; + self.recipe_validator = Some(rv); + self.stage_validator = Some(sv); + self.stage_list_validator = Some(slv); + self.module_validator = Some(mv); + self.module_list_validator = Some(mlv); + Ok(()) + } + + fn validate_file( + &self, + path: &Path, + traversed_files: &[&Path], + single_validator: &SchemaValidator, + list_validator: &SchemaValidator, + ) -> Vec + where + DF: DeserializeOwned + FromFileList, + { + let path_display = path.display().to_string().bold().italic(); + + if traversed_files.contains(&path) { + return vec![miette!( + "{} File {path_display} has already been parsed:\n{traversed_files:?}", + "Circular dependency detected!".bright_red(), + )]; + } + let traversed_files = { + let mut files: Vec<&Path> = Vec::with_capacity(traversed_files.len() + 1); + files.extend_from_slice(traversed_files); + files.push(path); + files + }; + + let file_str = match read_file(path) { + Err(e) => return vec![e], + Ok(f) => f, + }; + + match serde_yaml::from_str::(&file_str) + .into_diagnostic() + .with_context(|| format!("Failed to deserialize file {path_display}")) + { + Ok(instance) => { + trace!("{path_display}:\n{instance}"); + + if instance.get(DF::LIST_KEY).is_some() { + debug!("{path_display} is a multi file file"); + let errors = if self.all_errors { + process_basic_output( + list_validator.validator().apply(&instance).basic(), + &instance, + path, + ) + } else { + list_validator + .validator() + .iter_errors(&instance) + .map(process_err(&self.recipe)) + .collect() + }; + + if errors.is_empty() { + match serde_yaml::from_str::(&file_str).into_diagnostic() { + Err(e) => vec![e], + Ok(file) => file + .get_from_file_paths() + .par_iter() + .map(|file_path| { + self.validate_file::( + file_path, + &traversed_files, + single_validator, + list_validator, + ) + }) + .flatten() + .collect(), + } + } else { + errors + } + } else { + debug!("{path_display} is a single file file"); + if self.all_errors { + process_basic_output( + single_validator.validator().apply(&instance).basic(), + &instance, + path, + ) + } else { + single_validator + .validator() + .iter_errors(&instance) + .map(|err| miette!("{err}")) + .collect() + } + } + } + Err(e) => vec![e], + } + } + + fn validate_recipe(&self) -> Result<(), Vec> { + let recipe_path_display = self.recipe.display().to_string().bold().italic(); + debug!("Validating recipe {recipe_path_display}"); + + let recipe_str = read_file(&self.recipe).map_err(err_vec)?; + let recipe: Value = serde_yaml::from_str(&recipe_str) + .into_diagnostic() + .with_context(|| format!("Failed to deserialize recipe {recipe_path_display}")) + .map_err(err_vec)?; + trace!("{recipe_path_display}:\n{recipe}"); + + let schema_validator = self.recipe_validator.as_ref().unwrap(); + let errors = if self.all_errors { + process_basic_output( + schema_validator.validator().apply(&recipe).basic(), + &recipe, + &self.recipe, + ) + } else { + schema_validator + .validator() + .iter_errors(&recipe) + .map(process_err(&self.recipe)) + .collect() + }; + + if errors.is_empty() { + let recipe: Recipe = serde_yaml::from_str(&recipe_str) + .into_diagnostic() + .with_context(|| { + format!("Unable to convert Value to Recipe for {recipe_path_display}") + }) + .map_err(err_vec)?; + + let mut errors: Vec = Vec::new(); + if let Some(stages) = &recipe.stages_ext { + debug!("Validating stages for recipe {recipe_path_display}"); + + errors.extend( + stages + .get_from_file_paths() + .par_iter() + .map(|stage_path| { + debug!( + "Found 'from-file' reference in {recipe_path_display} going to {}", + stage_path.display().to_string().italic().bold() + ); + self.validate_file::( + stage_path, + &[], + self.stage_validator.as_ref().unwrap(), + self.stage_list_validator.as_ref().unwrap(), + ) + }) + .flatten() + .collect::>(), + ); + } + + debug!("Validating modules for recipe {recipe_path_display}"); + errors.extend( + recipe + .modules_ext + .get_from_file_paths() + .par_iter() + .map(|module_path| { + debug!( + "Found 'from-file' reference in {recipe_path_display} going to {}", + module_path.display().to_string().italic().bold() + ); + self.validate_file::( + module_path, + &[], + self.module_validator.as_ref().unwrap(), + self.module_list_validator.as_ref().unwrap(), + ) + }) + .flatten() + .collect::>(), + ); + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } else { + Err(errors) + } + } +} + +fn err_vec(err: Report) -> Vec { + vec![err] +} + +fn read_file(path: &Path) -> Result { + let mut recipe = String::new(); + BufReader::new( + OpenOptions::new() + .read(true) + .open(path) + .into_diagnostic() + .with_context(|| { + format!( + "Unable to open {}", + path.display().to_string().italic().bold() + ) + })?, + ) + .read_to_string(&mut recipe) + .into_diagnostic()?; + Ok(recipe) +} + +fn process_basic_output(out: BasicOutput<'_>, instance: &Value, path: &Path) -> Vec { + match out { + BasicOutput::Valid(_) => vec![], + BasicOutput::Invalid(errors) => { + let mut collection: IndexMap> = IndexMap::new(); + let errors = { + let mut e = errors.into_iter().collect::>(); + e.sort_by(|e1, e2| { + e1.instance_location() + .as_str() + .cmp(e2.instance_location().as_str()) + }); + e + }; + + for err in errors { + let schema_path = err.keyword_location(); + let instance_path = err.instance_location().to_string(); + let build_err = || { + format!( + "{:?}", + miette!( + "schema_path:'{}'", + schema_path.to_string().italic().dimmed(), + ) + .context(err.error_description().to_string().bold().bright_red()) + ) + }; + + collection + .entry(instance_path) + .and_modify(|errs| { + errs.push(build_err()); + // errs.sort_by(|(path1, _), (path2, _)| path1.cmp(path2)); + }) + .or_insert_with(|| vec![build_err()]); + } + + collection + .into_iter() + .map(|(key, value)| { + let instance = instance.pointer(&key).unwrap(); + + miette!( + "In file {} at '{}':\n\n{}\n{}", + path.display().to_string().bold().italic(), + key.bold().bright_yellow(), + serde_yaml::to_string(instance) + .into_diagnostic() + .and_then(|file| syntax_highlighting::highlight(&file, "yml", None)) + .unwrap_or_else(|_| instance.to_string()), + value.into_iter().collect::() + ) + }) + .collect() + } + } +} + +fn process_err<'a, 'b>(path: &'b Path) -> impl Fn(ValidationError<'a>) -> Report + use<'a, 'b> { + move |ValidationError { + instance, + instance_path, + kind: _, + schema_path: _, + }| { + miette!( + "- Invalid value {} file '{}':\n{}", + if instance_path.as_str().is_empty() { + string!("in root of") + } else { + format!( + "at path '{}' in", + instance_path.as_str().bold().bright_yellow() + ) + }, + path.display().to_string().italic().bold(), + &serde_yaml::to_string(&*instance) + .into_diagnostic() + .and_then(|file| syntax_highlighting::highlight(&file, "yml", None)) + .unwrap_or_else(|_| instance.to_string()) + ) + } +} diff --git a/src/commands/validate/schema_validator.rs b/src/commands/validate/schema_validator.rs new file mode 100644 index 00000000..695385ca --- /dev/null +++ b/src/commands/validate/schema_validator.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; + +use blue_build_process_management::ASYNC_RUNTIME; +use cached::proc_macro::cached; +use colored::Colorize; +use jsonschema::{Retrieve, Uri, Validator}; +use log::{debug, trace}; +use miette::{bail, Context, IntoDiagnostic, Report}; +use serde_json::Value; + +pub const BASE_SCHEMA_URL: &str = "https://schema.blue-build.org"; +pub const RECIPE_V1_SCHEMA_URL: &str = "https://schema.blue-build.org/recipe-v1.json"; +pub const STAGE_V1_SCHEMA_URL: &str = "https://schema.blue-build.org/stage-v1.json"; +pub const STAGE_LIST_V1_SCHEMA_URL: &str = "https://schema.blue-build.org/stage-list-v1.json"; +pub const MODULE_V1_SCHEMA_URL: &str = "https://schema.blue-build.org/module-v1.json"; +pub const MODULE_LIST_V1_SCHEMA_URL: &str = "https://schema.blue-build.org/module-list-v1.json"; + +#[derive(Debug, Clone)] +pub struct SchemaValidator { + schema: Arc, + validator: Arc, +} + +impl SchemaValidator { + pub fn validator(&self) -> Arc { + self.validator.clone() + } + + pub fn schema(&self) -> Arc { + self.schema.clone() + } +} + +pub async fn build_validator(url: &'static str) -> Result { + tokio::spawn(async move { + let schema: Arc = Arc::new( + reqwest::get(url) + .await + .into_diagnostic() + .with_context(|| format!("Failed to get schema at {url}"))? + .json() + .await + .into_diagnostic() + .with_context(|| format!("Failed to get json for schema {url}"))?, + ); + let validator = Arc::new( + tokio::task::spawn_blocking({ + let schema = schema.clone(); + move || { + jsonschema::options() + .with_retriever(ModuleSchemaRetriever) + .build(&schema) + .into_diagnostic() + .with_context(|| format!("Failed to build validator for schema {url}")) + } + }) + .await + .expect("Should join blocking thread")?, + ); + + Ok(SchemaValidator { schema, validator }) + }) + .await + .expect("Should join task") +} + +struct ModuleSchemaRetriever; + +impl Retrieve for ModuleSchemaRetriever { + fn retrieve( + &self, + uri: &Uri<&str>, + ) -> Result> { + Ok(ASYNC_RUNTIME.block_on(cache_retrieve(uri))?) + } +} + +#[cached(result = true, key = "String", convert = r#"{ format!("{uri}") }"#)] +async fn cache_retrieve(uri: &Uri<&str>) -> miette::Result { + let scheme = uri.scheme(); + let path = uri.path(); + + let uri = match scheme.as_str() { + "json-schema" => { + format!("{BASE_SCHEMA_URL}{path}") + } + "https" => uri.to_string(), + scheme => bail!("Unknown scheme {scheme}"), + }; + + debug!("Retrieving schema from {}", uri.bold().italic()); + tokio::spawn(async move { + reqwest::get(&uri) + .await + .into_diagnostic() + .with_context(|| format!("Failed to retrieve schema from {uri}"))? + .json() + .await + .into_diagnostic() + .with_context(|| format!("Failed to parse json from {uri}")) + .inspect(|value| trace!("{}:\n{value}", uri.bold().italic())) + }) + .await + .expect("Should join task") +} diff --git a/src/lib.rs b/src/lib.rs index 0676261c..01b15ffc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,10 @@ #![doc = include_str!("../README.md")] #![allow(clippy::needless_raw_string_hashes)] -shadow_rs::shadow!(shadow); +pub(crate) mod info { + #![allow(clippy::too_long_first_doc_paragraph)] + shadow_rs::shadow!(shadow); +} pub mod commands; pub mod rpm_ostree_status; diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index 01adef73..5a26c9ac 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -17,7 +17,7 @@ RUN \ {%- else if self::config_dir_exists() %} --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ {%- endif %} - {%- if let Some(source) = module.source %} + {%- if let Some(source) = module.get_non_local_source() %} --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ @@ -33,6 +33,7 @@ RUN \ {%- endif %} {%- endfor %} {% endmacro %} + {% macro stage_modules_run(modules_ext, os_version) %} # Module RUNs {%- for module in modules_ext.modules %} @@ -53,7 +54,7 @@ RUN \ {%- else if self::config_dir_exists() %} --mount=type=bind,from=stage-config,src=/config,dst=/tmp/config,rw \ {%- endif %} - {%- if let Some(source) = module.source %} + {%- if let Some(source) = module.get_non_local_source() %} --mount=type=bind,from={{ source }},src=/modules,dst=/tmp/modules,rw \ {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \