From c04af6e48865aced73191349cc6cbb2e0801b0dc Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 21:09:08 -0600 Subject: [PATCH 1/2] wait for the registry deployment to be fully ready before continuing (#2116) ## Description This changes the registry wait command to wait for the full deployment of the registry rather than a pod to avoid race conditions during bootup. ## Related Issue Fixes #N/A ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [x] Test, docs, adr added or updated as needed - [x] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed --- packages/zarf-registry/zarf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index 79255b2538..1fe3fa9286 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -135,7 +135,7 @@ components: after: - wait: cluster: - kind: pod + kind: deployment namespace: zarf name: app=docker-registry - condition: Ready + condition: Available From ff70c6a617cb162038b3cb3e42fe7b29d690657b Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 31 Oct 2023 23:36:55 -0500 Subject: [PATCH 2/2] chore: refactor Zarf Component composition (#2058) ## Description ```go chain, err := composer.NewImportChain(pkg.Components[0], "amd64") if err != nil { panic(err) } fmt.Println(chain.String()) ``` results in: ``` component "k3s" imports "k3s" in packages/distros/k3s, which imports "k3s" in common ``` ## Related Issue Fixes #2037 Fixes https://github.com/defenseunicorns/zarf/issues/1827 Fixes https://github.com/defenseunicorns/zarf/issues/1845 ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [ ] Test, docs, adr added or updated as needed - [x] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed --------- Signed-off-by: razzle Co-authored-by: Wayne Starr Co-authored-by: Wayne Starr --- .../publish-application-packages.yml | 12 +- adr/0021-composable-components.md | 23 + .../100-cli-commands/zarf_package_create.md | 4 +- docs/3-create-a-zarf-package/4-zarf-schema.md | 81 +--- examples/composable-packages/README.md | 23 +- examples/composable-packages/zarf.yaml | 8 +- go.mod | 5 +- go.sum | 6 +- src/cmd/package.go | 10 +- src/config/lang/english.go | 73 ++- src/extensions/bigbang/bigbang.go | 64 ++- src/extensions/bigbang/manifests.go | 15 +- src/internal/packager/validate/validate.go | 58 ++- src/pkg/layout/package_test.go | 2 +- src/pkg/oci/fetch.go | 12 +- src/pkg/oci/pull.go | 32 +- src/pkg/packager/common.go | 4 - src/pkg/packager/components.go | 5 +- src/pkg/packager/compose.go | 419 ++---------------- src/pkg/packager/composer/extensions.go | 14 + src/pkg/packager/composer/list.go | 284 ++++++++++++ src/pkg/packager/composer/list_test.go | 376 ++++++++++++++++ src/pkg/packager/composer/oci.go | 130 ++++++ src/pkg/packager/composer/override.go | 114 +++++ src/pkg/packager/composer/pathfixer.go | 86 ++++ src/pkg/packager/create.go | 64 +-- src/pkg/packager/extensions.go | 10 - src/pkg/packager/variables.go | 28 -- src/pkg/packager/yaml.go | 4 - src/pkg/transform/image.go | 2 +- src/pkg/utils/helpers/misc.go | 74 ---- src/pkg/utils/helpers/slice.go | 95 ++++ src/pkg/utils/helpers/url.go | 4 +- src/test/e2e/09_component_compose_test.go | 184 ++++++++ .../e2e/32_checksum_and_signature_test.go | 2 +- src/test/e2e/50_oci_package_test.go | 2 +- src/test/e2e/51_oci_compose_test.go | 92 +--- .../e2e/52_oci_compose_differential_test.go | 133 ------ src/test/nightly/ecr_publish_test.go | 2 +- .../files/coffee-ipsum.txt | 0 .../files/kustomization.yaml | 0 .../09-composable-packages/files/service.yaml | 17 + .../files/test-values.yaml | 10 + .../sub-package/zarf.yaml | 44 ++ .../packages/09-composable-packages/zarf.yaml | 64 +++ .../51-import-everything/bar/zarf.yaml | 23 - .../flux-overrides-helm-controller.yaml | 17 + .../big-bang-min/zarf.yaml | 26 ++ .../51-import-everything/foo/zarf.yaml | 9 - .../51-import-everything/inception/zarf.yaml | 20 +- .../51-import-everything/oci-import/zarf.yaml | 13 + .../packages/51-import-everything/zarf.yaml | 119 +---- .../packages/52-oci-differential/README.md | 7 - .../packages/52-oci-differential/zarf.yaml | 21 - src/types/component.go | 4 +- src/types/package.go | 1 - src/types/runtime.go | 1 - zarf.schema.json | 25 +- 58 files changed, 1757 insertions(+), 1220 deletions(-) create mode 100644 adr/0021-composable-components.md create mode 100644 src/pkg/packager/composer/extensions.go create mode 100644 src/pkg/packager/composer/list.go create mode 100644 src/pkg/packager/composer/list_test.go create mode 100644 src/pkg/packager/composer/oci.go create mode 100644 src/pkg/packager/composer/override.go create mode 100644 src/pkg/packager/composer/pathfixer.go create mode 100644 src/pkg/utils/helpers/slice.go create mode 100644 src/test/e2e/09_component_compose_test.go delete mode 100644 src/test/e2e/52_oci_compose_differential_test.go rename src/test/packages/{51-import-everything => 09-composable-packages}/files/coffee-ipsum.txt (100%) rename src/test/packages/{51-import-everything => 09-composable-packages}/files/kustomization.yaml (100%) create mode 100644 src/test/packages/09-composable-packages/files/service.yaml create mode 100644 src/test/packages/09-composable-packages/files/test-values.yaml create mode 100644 src/test/packages/09-composable-packages/sub-package/zarf.yaml create mode 100644 src/test/packages/09-composable-packages/zarf.yaml delete mode 100644 src/test/packages/51-import-everything/bar/zarf.yaml create mode 100644 src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml create mode 100644 src/test/packages/51-import-everything/big-bang-min/zarf.yaml delete mode 100644 src/test/packages/51-import-everything/foo/zarf.yaml create mode 100644 src/test/packages/51-import-everything/oci-import/zarf.yaml delete mode 100644 src/test/packages/52-oci-differential/README.md delete mode 100644 src/test/packages/52-oci-differential/zarf.yaml diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index b79a455686..175543b466 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -35,11 +35,15 @@ jobs: - name: Build And Publish Application Packages # Create the dos-games package with the cosign signature, publish to ghcr and copy the tags to allow 'uname -m' to work run: | - zarf package create -o build -a amd64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm - zarf package create -o build -a arm64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + zarf package create -o build -a amd64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + zarf package create -o build -a arm64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm - zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages - zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages + # Publish a the signed dos-games package + zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + + # Publish a skeleton of the dos-games package + zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 diff --git a/adr/0021-composable-components.md b/adr/0021-composable-components.md new file mode 100644 index 0000000000..eb26be6788 --- /dev/null +++ b/adr/0021-composable-components.md @@ -0,0 +1,23 @@ +# 21. Composable Components + +Date: 2023-10-26 + +## Status + +Accepted + +## Context + +Zarf has supports composing components together between packages on `zarf package create` since v0.16.0. This has allowed package creators to make more complex packages from smaller reusable bits. As this functionality grew however there were a few problems that developed: + +1. Import chains did not handle scaling to larger numbers of layers with test coverage usually only covering the first import. +2. When OCI skeletons were added they were largely bolted on after the fact without rethinking how they would impact composability. +3. Component filtering via the `only` filter was not implemented in a central location leading to bugs with create-time filters. + +## Decision + +We decided to separate composability into its own package that represents a composability import chain as a doubly linked list. This allows us to represent the whole chain as it exists relative to the "head" Zarf package (the definition that Zarf was asked to build) to more easily handle packages that are in different locations (such as OCI skeletons in one's cache). We also run the compose functions on all components so that the additional filter logic that is needed for these components can be handled more concisely and built upon (as it might for `flavor` https://github.com/defenseunicorns/zarf/issues/2101). + +## Consequences + +Maintaining the full context within a linked list does use more memory and some operations on it are less efficient than they could be if we one-shotted the compose. This is a decent tradeoff however as most import chains won't be longer than 4 or 5 elements in practice and these structs and operations are relatively small. diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md index 4d7f3f9933..8bd70e500d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md @@ -19,14 +19,14 @@ zarf package create [ DIRECTORY ] [flags] --confirm Confirm package creation without prompting --differential string [beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package -h, --help help for create - -k, --key string Path to private key file for signing packages - --key-pass string Password to the private key file used for signing packages -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting. -o, --output string Specify the output (either a directory or an oci:// URL) for the created Zarf package --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default []) -s, --sbom View SBOM contents after creating the package --sbom-out string Specify an output directory for the SBOMs from the created Zarf package --set stringToString Specify package variables to set on the command line (KEY=value) (default []) + --signing-key string Path to private key file for signing packages + --signing-key-pass string Password to the private key file used for signing packages --skip-sbom Skip generating SBOM for this package ``` diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index b0659ede67..478a863c26 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -491,45 +491,6 @@ must respect the following conditions -
- - OCIImportedComponents - -  -
- - ## build > OCIImportedComponents - -**Description:** Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -
- - Pattern Property .* - -  -
- -:::note -All properties whose name matches the regular expression -```.*``` ([Test](https://regex101.com/?regex=.%2A)) -must respect the following conditions -::: - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
-
-
lastNonBreakingVersion @@ -1002,50 +963,10 @@ Must be one of: | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `combining` | +| **Type** | `object` | | **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | | **Defined in** | #/definitions/ZarfChart | -
- -| One of(Option) | -| ---------------------------------------------------- | -| [url](#components_items_charts_items_oneOf_i0) | -| [localPath](#components_items_charts_items_oneOf_i1) | - -
- -### Property `url` - -**Title:** url - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -#### The following properties are required -* url - -
-
- -### Property `localPath` - -**Title:** localPath - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -#### The following properties are required -* localPath - -
- -
-
name * diff --git a/examples/composable-packages/README.md b/examples/composable-packages/README.md index 764242f40d..657fd03b60 100644 --- a/examples/composable-packages/README.md +++ b/examples/composable-packages/README.md @@ -2,13 +2,13 @@ import ExampleYAML from "@site/src/components/ExampleYAML"; # Composable Packages -This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality. It uses the existing [DOS games](../dos-games/README.md) and [WordPress](../wordpress/README.md) examples by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file. +This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality. It uses the existing [DOS games](../dos-games/README.md) example by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file. The `import` key in Zarf supports two modes to pull in a component: -1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem. This allows you to have a common component that you can reuse across multiple packages *within* a project. +1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem. This allows you to have a common component that you can reuse across multiple packages *within* a project (i.e. within one team/codebase). -2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of components and import them into multiple packages *across* projects. +2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of reusable components and import them into multiple packages *across* projects (i.e. across teams/codebases). :::tip @@ -40,23 +40,6 @@ To view the example in its entirety, select the `Edit this page` link below the ::: -:::note - -Creating this example requires a locally hosted container registry that has the `wordpress` skeleton package published and available. You can do this by running the following commands: - -```bash -docker run -d -p 555:5000 --restart=always --name registry registry:2 -zarf package publish examples/wordpress oci://127.0.0.1:555 --insecure -``` - -You will also need to pass the `--insecure` flag to `zarf package create` to pull from the `http` registry: - -```bash -zarf package create examples/composable-packages/ --insecure -``` - -::: - :::info diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index ab5890fddd..34f499d670 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -24,20 +24,18 @@ components: files: - quake-service.yaml - - name: oci-wordpress-url + - name: oci-games-url # The component logic keys ('required', 'group', and 'default') always override those of the imported package # required: false # the initial value overrides the child component # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component import: # The URL to the skeleton package containing this component's package definition - url: oci://localhost:555/wordpress:16.0.4-skeleton + url: oci://🦄/dos-games:1.0.0-skeleton # Example optional custom name to point to in the imported package (default is to use this component's name) - name: wordpress + name: baseline # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component. actions: onDeploy: before: - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp} - setVariables: - - name: WORDPRESS_BLOG_NAME diff --git a/go.mod b/go.mod index cf09b1dcf8..3f3df240eb 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 github.com/derailed/k9s v0.27.4 - github.com/distribution/distribution v2.8.3+incompatible + github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.6+incompatible github.com/fairwindsops/pluto/v5 v5.18.4 github.com/fatih/color v1.15.0 @@ -186,9 +186,8 @@ require ( github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v24.0.6+incompatible // indirect + github.com/docker/docker v24.0.7+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect diff --git a/go.sum b/go.sum index 31af02f262..848584069a 100644 --- a/go.sum +++ b/go.sum @@ -510,8 +510,6 @@ github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c/go.mod h1:GvWn github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo= -github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= @@ -522,8 +520,8 @@ github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWT github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= diff --git a/src/cmd/package.go b/src/cmd/package.go index e448667b39..14eb104ab2 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -349,11 +349,17 @@ func bindCreateFlags(v *viper.Viper) { createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(common.VPkgCreateSbomOutput), lang.CmdPackageCreateFlagSbomOut) createFlags.BoolVar(&pkgConfig.CreateOpts.SkipSBOM, "skip-sbom", v.GetBool(common.VPkgCreateSkipSbom), lang.CmdPackageCreateFlagSkipSbom) createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(common.VPkgCreateMaxPackageSize), lang.CmdPackageCreateFlagMaxPackageSize) - createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) - createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride) + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPath, "signing-key", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "signing-key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) + + createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagDeprecatedKey) + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagDeprecatedKeyPassword) + createFlags.MarkHidden("output-directory") + createFlags.MarkHidden("key") + createFlags.MarkHidden("key-pass") } func bindDeployFlags(v *viper.Viper) { diff --git a/src/config/lang/english.go b/src/config/lang/english.go index c7ac99ca6f..94380e542f 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -238,19 +238,21 @@ const ( CmdPackageListNoPackageWarn = "Unable to get the packages deployed to the cluster" CmdPackageListUnmarshalErr = "Unable to read all of the packages deployed to the cluster" - CmdPackageCreateFlagConfirm = "Confirm package creation without prompting" - CmdPackageCreateFlagSet = "Specify package variables to set on the command line (KEY=value)" - CmdPackageCreateFlagOutput = "Specify the output (either a directory or an oci:// URL) for the created Zarf package" - CmdPackageCreateFlagSbom = "View SBOM contents after creating the package" - CmdPackageCreateFlagSbomOut = "Specify an output directory for the SBOMs from the created Zarf package" - CmdPackageCreateFlagSkipSbom = "Skip generating SBOM for this package" - CmdPackageCreateFlagMaxPackageSize = "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting." - CmdPackageCreateFlagSigningKey = "Path to private key file for signing packages" - CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages" - CmdPackageCreateFlagDifferential = "[beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package" - CmdPackageCreateFlagRegistryOverride = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)" - CmdPackageCreateCleanPathErr = "Invalid characters in Zarf cache path, defaulting to %s" - CmdPackageCreateErr = "Failed to create package: %s" + CmdPackageCreateFlagConfirm = "Confirm package creation without prompting" + CmdPackageCreateFlagSet = "Specify package variables to set on the command line (KEY=value)" + CmdPackageCreateFlagOutput = "Specify the output (either a directory or an oci:// URL) for the created Zarf package" + CmdPackageCreateFlagSbom = "View SBOM contents after creating the package" + CmdPackageCreateFlagSbomOut = "Specify an output directory for the SBOMs from the created Zarf package" + CmdPackageCreateFlagSkipSbom = "Skip generating SBOM for this package" + CmdPackageCreateFlagMaxPackageSize = "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting." + CmdPackageCreateFlagSigningKey = "Path to private key file for signing packages" + CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages" + CmdPackageCreateFlagDeprecatedKey = "[Deprecated] Path to private key file for signing packages (use --signing-key instead)" + CmdPackageCreateFlagDeprecatedKeyPassword = "[Deprecated] Password to the private key file used for signing packages (use --signing-key-pass instead)" + CmdPackageCreateFlagDifferential = "[beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package" + CmdPackageCreateFlagRegistryOverride = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)" + CmdPackageCreateCleanPathErr = "Invalid characters in Zarf cache path, defaulting to %s" + CmdPackageCreateErr = "Failed to create package: %s" CmdPackageDeployFlagConfirm = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes." CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover." @@ -569,39 +571,36 @@ const ( // src/internal/packager/validate. const ( - PkgValidateTemplateDeprecation = "Package template '%s' is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." - PkgValidateMustBeUppercase = "variable name '%s' must be all uppercase and contain no special characters except _" + PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." + PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" - PkgValidateErrActionVariables = "component %s cannot contain setVariables outside of onDeploy in actions" - PkgValidateErrActionCmdWait = "action %s cannot be both a command and wait action" + PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" + PkgValidateErrActionCmdWait = "action %q cannot be both a command and wait action" PkgValidateErrActionClusterNetwork = "a single wait action must contain only one of cluster or network" PkgValidateErrChart = "invalid chart definition: %w" - PkgValidateErrChartName = "chart %s exceed the maximum length of %d characters" - PkgValidateErrChartNameMissing = "chart %s must include a name" + PkgValidateErrChartName = "chart %q exceed the maximum length of %d characters" + PkgValidateErrChartNameMissing = "chart %q must include a name" PkgValidateErrChartNameNotUnique = "chart name %q is not unique" - PkgValidateErrChartNamespaceMissing = "chart %s must include a namespace" - PkgValidateErrChartURLOrPath = "chart %s must only have a url or localPath" - PkgValidateErrChartVersion = "chart %s must include a chart version" - PkgValidateErrComponentNameNotUnique = "component name '%s' is not unique" - PkgValidateErrComponent = "invalid component: %w" - PkgValidateErrComponentReqDefault = "component %s cannot be both required and default" - PkgValidateErrComponentReqGrouped = "component %s cannot be both required and grouped" - PkgValidateErrComponentYOLO = "component %s incompatible with the online-only package flag (metadata.yolo): %w" + PkgValidateErrChartNamespaceMissing = "chart %q must include a namespace" + PkgValidateErrChartURLOrPath = "chart %q must have either a url or localPath" + PkgValidateErrChartVersion = "chart %q must include a chart version" + PkgValidateErrComponentNameNotUnique = "component name %q is not unique" + PkgValidateErrComponent = "invalid component %q: %w" + PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" + PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" + PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" PkgValidateErrConstant = "invalid package constant: %w" - PkgValidateErrImportPathInvalid = "invalid file path '%s' provided directory must contain a valid zarf.yaml file" - PkgValidateErrImportURLInvalid = "invalid url '%s' provided" - PkgValidateErrImportOptions = "imported package %s must have either a url or a path" - PkgValidateErrImportPathMissing = "imported package %s must include a path" + PkgValidateErrImportDefinition = "invalid imported definition for %s: %s" PkgValidateErrInitNoYOLO = "sorry, you can't YOLO an init package" PkgValidateErrManifest = "invalid manifest definition: %w" - PkgValidateErrManifestFileOrKustomize = "manifest %s must have at least one file or kustomization" - PkgValidateErrManifestNameLength = "manifest %s exceed the maximum length of %d characters" - PkgValidateErrManifestNameMissing = "manifest %s must include a name" + PkgValidateErrManifestFileOrKustomize = "manifest %q must have at least one file or kustomization" + PkgValidateErrManifestNameLength = "manifest %q exceed the maximum length of %d characters" + PkgValidateErrManifestNameMissing = "manifest %q must include a name" PkgValidateErrManifestNameNotUnique = "manifest name %q is not unique" PkgValidateErrName = "invalid package name: %w" - PkgValidateErrPkgConstantName = "constant name '%s' must be all uppercase and contain no special characters except _" - PkgValidateErrPkgConstantPattern = "provided value for constant %q does not match pattern \"%s\"" - PkgValidateErrPkgName = "package name '%s' must be all lowercase and contain no special characters except -" + PkgValidateErrPkgConstantName = "constant name %q must be all uppercase and contain no special characters except _" + PkgValidateErrPkgConstantPattern = "provided value for constant %q does not match pattern %q" + PkgValidateErrPkgName = "package name %q must be all lowercase and contain no special characters except -" PkgValidateErrVariable = "invalid package variable: %w" PkgValidateErrYOLONoArch = "cluster architecture not allowed" PkgValidateErrYOLONoDistro = "cluster distros not allowed" diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index c15fe80d5a..3462017858 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -249,14 +249,14 @@ func Run(YOLO bool, tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (typ // Skeletonize mutates a component so that the valuesFiles can be contained inside a skeleton package func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(valuesFile, filepath.Ext(valuesFile)) + // Get the base file name for this file. + baseName := filepath.Base(valuesFile) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the skeleton name prefix. - skelName := fmt.Sprintf("bb-ext-skeleton-values-%s.yaml", baseName) + skelName := fmt.Sprintf("bb-skel-vals-%d-%s.yaml", valuesIdx, baseName) rel := filepath.Join(layout.TempDir, skelName) dst := filepath.Join(tmpPaths.Base, rel) @@ -269,14 +269,14 @@ func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types. } for fluxPatchFileIdx, fluxPatchFile := range c.Extensions.BigBang.FluxPatchFiles { - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(fluxPatchFile, filepath.Ext(fluxPatchFile)) + // Get the base file name for this file. + baseName := filepath.Base(fluxPatchFile) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the skeleton name prefix. - skelName := fmt.Sprintf("bb-ext-skeleton-flux-patches-%s.yaml", baseName) + skelName := fmt.Sprintf("bb-skel-flux-patch-%d-%s.yaml", fluxPatchFileIdx, baseName) rel := filepath.Join(layout.TempDir, skelName) dst := filepath.Join(tmpPaths.Base, rel) @@ -291,19 +291,37 @@ func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types. return c, nil } -// Compose mutates a component so that the valuesFiles are relative to the parent importing component -func Compose(pathAncestry string, c types.ZarfComponent) types.ZarfComponent { - for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { - parentRel := filepath.Join(pathAncestry, valuesFile) - c.Extensions.BigBang.ValuesFiles[valuesIdx] = parentRel - } +// Compose mutates a component so that its local paths are relative to the provided path +// +// additionally, it will merge any overrides +func Compose(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { + // perform any overrides + if override.Extensions.BigBang != nil { + for valuesIdx, valuesFile := range override.Extensions.BigBang.ValuesFiles { + if helpers.IsURL(valuesFile) { + continue + } - for fluxPatchFileIdx, fluxPatchFile := range c.Extensions.BigBang.FluxPatchFiles { - parentRel := filepath.Join(pathAncestry, fluxPatchFile) - c.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = parentRel - } + fixed := filepath.Join(relativeTo, valuesFile) + override.Extensions.BigBang.ValuesFiles[valuesIdx] = fixed + } + + for fluxPatchFileIdx, fluxPatchFile := range override.Extensions.BigBang.FluxPatchFiles { + if helpers.IsURL(fluxPatchFile) { + continue + } - return c + fixed := filepath.Join(relativeTo, fluxPatchFile) + override.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = fixed + } + + if c.Extensions.BigBang == nil { + c.Extensions.BigBang = override.Extensions.BigBang + } else { + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + } + } } // isValidVersion check if the version is 1.54.0 or greater. @@ -477,8 +495,8 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) } // Loop through the valuesFrom list and create a manifest for each. - for _, path := range cfg.ValuesFiles { - data, err := manifestValuesFile(path) + for valuesIdx, valuesFile := range cfg.ValuesFiles { + data, err := manifestValuesFile(valuesIdx, valuesFile) if err != nil { return manifest, err } diff --git a/src/extensions/bigbang/manifests.go b/src/extensions/bigbang/manifests.go index 6aea42e421..cec481d6cb 100644 --- a/src/extensions/bigbang/manifests.go +++ b/src/extensions/bigbang/manifests.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "github.com/Masterminds/semver/v3" @@ -19,8 +18,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var nonAlphnumeric = regexp.MustCompile("[^a-zA-Z0-9]+") - const bbV1ZarfCredentialsValues = ` registryCredentials: registry: "###ZARF_REGISTRY###" @@ -122,21 +119,21 @@ func manifestGitRepo(cfg *extensions.BigBang) fluxSrcCtrl.GitRepository { } // manifestValuesFile generates a Secret object for the Big Bang umbrella repo. -func manifestValuesFile(path string) (secret corev1.Secret, err error) { +func manifestValuesFile(idx int, path string) (secret corev1.Secret, err error) { // Read the file from the path. file, err := os.ReadFile(path) if err != nil { return secret, err } - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(path, filepath.Ext(path)) + // Get the base file name for this file. + baseName := filepath.Base(path) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the name prefix. - name := fmt.Sprintf("bb-ext-user-values-%s", baseName) + name := fmt.Sprintf("bb-usr-vals-%d-%s", idx, baseName) // Create a secret with the file contents. secret = corev1.Secret{ diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 46336e3995..9a1d57db1a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -6,15 +6,13 @@ package validate import ( "fmt" - "os" "path/filepath" "regexp" "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -60,46 +58,44 @@ func Run(pkg types.ZarfPackage) error { uniqueComponentNames[component.Name] = true if err := validateComponent(pkg, component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponent, err) + return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) } } return nil } -// ImportPackage validates the package trying to be imported. -func ImportPackage(composedComponent *types.ZarfComponent) error { - path := composedComponent.Import.Path - url := composedComponent.Import.URL +// ImportDefinition validates the component trying to be imported. +func ImportDefinition(component *types.ZarfComponent) error { + path := component.Import.Path + url := component.Import.URL - if url == "" { - // ensure path exists - if path == "" { - return fmt.Errorf(lang.PkgValidateErrImportPathMissing, composedComponent.Name) - } + // ensure path or url is provided + if path == "" && url == "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "neither a path nor a URL was provided") + } - // remove zarf.yaml from path if path has zarf.yaml suffix - if strings.HasSuffix(path, layout.ZarfYAML) { - path = strings.Split(path, layout.ZarfYAML)[0] - } + // ensure path and url are not both provided + if path != "" && url != "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "both a path and a URL were provided") + } - // add a forward slash to end of path if it does not have one - if !strings.HasSuffix(path, string(os.PathSeparator)) { - path = filepath.Clean(path) + string(os.PathSeparator) + // validation for path + if url == "" && path != "" { + // ensure path is not an absolute path + if filepath.IsAbs(path) { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "path cannot be an absolute path") } + } - // ensure there is a zarf.yaml in provided path - if utils.InvalidPath(filepath.Join(path, layout.ZarfYAML)) { - return fmt.Errorf(lang.PkgValidateErrImportPathInvalid, composedComponent.Import.Path) - } - } else { - // ensure path is empty - if path != "" { - return fmt.Errorf(lang.PkgValidateErrImportOptions, composedComponent.Name) - } + // validation for url + if url != "" && path == "" { ok := helpers.IsOCIURL(url) if !ok { - return fmt.Errorf(lang.PkgValidateErrImportURLInvalid, composedComponent.Import.URL) + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") + } + if !strings.HasSuffix(url, oci.SkeletonSuffix) { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "OCI import URL must end with -skeleton") } } @@ -303,7 +299,7 @@ func validateChart(chart types.ZarfChart) error { return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) } - // Must only have a url or localPath + // Must have a url or localPath (and not both) count := oneIfNotEmpty(chart.URL) + oneIfNotEmpty(chart.LocalPath) if count != 1 { return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index c009496581..dc98286775 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestPackage_Files(t *testing.T) { +func TestPackageFiles(t *testing.T) { pp := New("test") raw := &PackagePaths{ diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index be7406819a..4306e709e1 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -52,12 +52,20 @@ func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err erro } // FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *OrasRemote) FetchZarfYAML(manifest *ZarfOCIManifest) (pkg types.ZarfPackage, err error) { +func (o *OrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return pkg, err + } return FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) } // FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *OrasRemote) FetchImagesIndex(manifest *ZarfOCIManifest) (index *ocispec.Index, err error) { +func (o *OrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return index, err + } return FetchJSONFile[*ocispec.Index](o.FetchLayer, manifest, ZarfPackageIndexPath) } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index c109f8efd5..c7f33d97f8 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -84,7 +84,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) return nil, err } - pkg, err := o.FetchZarfYAML(root) + pkg, err := o.FetchZarfYAML() if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) if len(images) > 0 { // Add the image index and the oci-layout layers layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex(root) + index, err := o.FetchImagesIndex() if err != nil { return nil, err } @@ -195,11 +195,11 @@ func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersT copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, ©Opts, destinationDir) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir) } // CopyWithProgress copies the given layers from the remote repository to the given store. -func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, copyOpts *oras.CopyOptions, destinationDir string) error { +func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, destinationDir string) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -209,18 +209,20 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } } - copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - nodes, err := content.Successors(ctx, fetcher, desc) - if err != nil { - return nil, err - } - var ret []ocispec.Descriptor - for _, node := range nodes { - if slices.Contains(shas, node.Digest.Encoded()) { - ret = append(ret, node) + if copyOpts.FindSuccessors == nil { + copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + nodes, err := content.Successors(ctx, fetcher, desc) + if err != nil { + return nil, err + } + var ret []ocispec.Descriptor + for _, node := range nodes { + if slices.Contains(shas, node.Digest.Encoded()) { + ret = append(ret, node) + } } + return ret, nil } - return ret, nil } // Create a thread to update a progress bar as we save the package to disk @@ -229,7 +231,7 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.repo.Reference.String()) go utils.RenderProgressBarForLocalDirWrite(destinationDir, estimatedBytes, &wg, doneSaving, "Pulling", successText) - _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), *copyOpts) + _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { return err } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 6d4d3606e4..bc16e78385 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -98,10 +98,6 @@ func New(cfg *types.PackagerConfig, mods ...Modifier) (*Packager, error) { cfg.SetVariableMap = make(map[string]*types.ZarfSetVariable) } - if cfg.Pkg.Build.OCIImportedComponents == nil { - cfg.Pkg.Build.OCIImportedComponents = make(map[string]string) - } - var ( err error pkgr = &Packager{ diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 5101f2c0ee..45b9cf5ae3 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -29,17 +29,18 @@ func (p *Packager) getValidComponents() []types.ZarfComponent { // Break up components into choice groups for _, component := range p.cfg.Pkg.Components { + matchFn := func(a, b string) bool { return a == b } key := component.Group // If not a choice group, then use the component name as the key if key == "" { key = component.Name } else { // Otherwise, add the component name to the choice group list for later validation - choiceComponents = helpers.AppendIfNotExists(choiceComponents, component.Name) + choiceComponents = helpers.MergeSlices(choiceComponents, []string{component.Name}, matchFn) } // Preserve component order - orderedKeys = helpers.AppendIfNotExists(orderedKeys, key) + orderedKeys = helpers.MergeSlices(orderedKeys, []string{key}, matchFn) // Append the component to the list of components in the group componentGroups[key] = append(componentGroups[key], component) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 01aab83748..b390dfb5f4 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -5,18 +5,8 @@ package packager import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" ) @@ -24,397 +14,44 @@ import ( func (p *Packager) composeComponents() error { components := []types.ZarfComponent{} - for _, component := range p.cfg.Pkg.Components { - if component.Import.Path == "" && component.Import.URL == "" { - // Migrate any deprecated component configurations now - migratedComponent, warnings := deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - components = append(components, migratedComponent) - p.warnings = append(p.warnings, warnings...) - } else { - composedComponent, err := p.getComposedComponent(component) - if err != nil { - return fmt.Errorf("unable to compose component %s: %w", component.Name, err) - } - components = append(components, composedComponent) - } - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - p.cfg.Pkg.Components = components - - return nil -} - -// getComposedComponent recursively retrieves a composed Zarf component -// -------------------------------------------------------------------- -// For composed components, we build the tree of components starting at the root and adding children as we go; -// this follows the composite design pattern outlined here: https://en.wikipedia.org/wiki/Composite_pattern -// where 1 component parent is made up of 0...n composite or leaf children. -func (p *Packager) getComposedComponent(parentComponent types.ZarfComponent) (child types.ZarfComponent, err error) { - // Make sure the component we're trying to import can't be accessed. - if err := validate.ImportPackage(&parentComponent); err != nil { - return child, fmt.Errorf("invalid import definition in the %s component: %w", parentComponent.Name, err) - } - - // Keep track of the composed components import path to build nested composed components. - pathAncestry := "" - - // Get the component that we are trying to import. - // NOTE: This function is recursive and will continue getting the children until there are no more 'imported' components left. - child, err = p.getChildComponent(parentComponent, pathAncestry) - if err != nil { - return child, fmt.Errorf("unable to get child component: %w", err) - } - - // Merge the overrides from the child that we just received with the parent we were provided. - p.mergeComponentOverrides(&child, parentComponent) - - return -} - -func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry string) (child types.ZarfComponent, err error) { - // Figure out which component we are actually importing. - // NOTE: Default to the component name if a custom one was not provided. - childComponentName := parent.Import.ComponentName - if childComponentName == "" { - childComponentName = parent.Name - } - - var cachePath string - - subPkgPaths := layout.New(parent.Import.Path) - - if parent.Import.URL != "" { - if !strings.HasSuffix(parent.Import.URL, oci.SkeletonSuffix) { - return child, fmt.Errorf("import URL must be a 'skeleton' package: %s", parent.Import.URL) - } - - // Save all the OCI imported components into our build data - p.cfg.Pkg.Build.OCIImportedComponents[parent.Import.URL] = childComponentName - - skelURL := strings.TrimPrefix(parent.Import.URL, helpers.OCIURLPrefix) - cachePath = filepath.Join(config.GetAbsCachePath(), "oci", skelURL) - err = os.MkdirAll(cachePath, 0755) - if err != nil { - return child, fmt.Errorf("unable to create cache path %s: %w", cachePath, err) - } - - err = p.setOCIRemote(parent.Import.URL) - if err != nil { - return child, err - } - manifest, err := p.remote.FetchRoot() - if err != nil { - return child, err - } - tb := filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", childComponentName)) - fetchedLayers, err := p.remote.PullPackage(cachePath, config.CommonOptions.OCIConcurrency, manifest.Locate(tb)) - if err != nil { - return child, fmt.Errorf("unable to pull skeleton from %s: %w", skelURL, err) - } - cwd, err := os.Getwd() - if err != nil { - return child, fmt.Errorf("unable to get current working directory: %w", err) - } - - rel, err := filepath.Rel(cwd, cachePath) - if err != nil { - return child, fmt.Errorf("unable to get relative path: %w", err) - } - parent.Import.Path = rel - subPkgPaths = layout.New(parent.Import.Path) - subPkgPaths.SetFromLayers(fetchedLayers) - } - - var subPkg types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(pathAncestry, subPkgPaths.ZarfYAML), &subPkg); err != nil { - return child, err - } - - // Merge in child package variables (only if the variable does not exist in parent). - for _, importedVariable := range subPkg.Variables { - p.injectImportedVariable(importedVariable) - } - - // Merge in child package constants (only if the constant does not exist in parent). - for _, importedConstant := range subPkg.Constants { - p.injectImportedConstant(importedConstant) - } - - // Find the child component from the imported package that matches our arch. - for _, component := range subPkg.Components { - if component.Name == childComponentName { - filterArch := component.Only.Cluster.Architecture + pkgVars := p.cfg.Pkg.Variables + pkgConsts := p.cfg.Pkg.Constants - // Override the filter if it is set by the parent component. - if parent.Only.Cluster.Architecture != "" { - filterArch = parent.Only.Cluster.Architecture - } - - // Only add this component if it is valid for the target architecture. - if filterArch == "" || filterArch == p.arch { - child = component - break - } - } - } - - // If we didn't find a child component, bail. - if child.Name == "" { - return child, fmt.Errorf("unable to find the component %s in the imported package", childComponentName) - } - - // If it's OCI, we need to unpack the component tarball - if parent.Import.URL != "" { - parent.Import.Path = filepath.Join(parent.Import.Path, layout.ComponentsDir, child.Name) - if err := subPkgPaths.Components.Unarchive(child); err != nil { - if layout.IsNotLoaded(err) { - // If the tarball doesn't exist (skeleton component had no local resources), we need to create the directory anyways in case there are actions - _, err := subPkgPaths.Components.Create(child) - if err != nil { - return child, fmt.Errorf("unable to create composed component cache path %s: %w", cachePath, err) - } - } else { - return child, fmt.Errorf("unable to unarchive component: %w", err) - } + for _, component := range p.cfg.Pkg.Components { + arch := p.arch + // filter by architecture + if component.Only.Cluster.Architecture != "" && component.Only.Cluster.Architecture != arch { + continue } - } - pathAncestry = filepath.Join(pathAncestry, parent.Import.Path) - // Check if we need to get more of children. - if child.Import.Path != "" { - // Recursively call this function to get the next layer of children. - grandchildComponent, err := p.getChildComponent(child, pathAncestry) + // build the import chain + chain, err := composer.NewImportChain(component, arch) if err != nil { - return child, err + return err } + message.Debugf("%s", chain) - // Merge the grandchild values into the child. - p.mergeComponentOverrides(&grandchildComponent, child) + // migrate any deprecated component configurations now + warnings := chain.Migrate(p.cfg.Pkg.Build) + p.warnings = append(p.warnings, warnings...) - // Set the grandchild as the child component now that we're done with recursively importing. - child = grandchildComponent - } else { - // Fix the filePaths of imported components to be accessible from our current location. - child, err = p.fixComposedFilepaths(pathAncestry, child) + // get the composed component + composed, err := chain.Compose() if err != nil { - return child, fmt.Errorf("unable to fix composed filepaths: %s", err.Error()) + return err } - } - - // Migrate any deprecated component configurations now - var warnings []string - child, warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, child) - p.warnings = append(p.warnings, warnings...) - - return -} + components = append(components, composed) -func (p *Packager) fixComposedFilepaths(pathAncestry string, child types.ZarfComponent) (types.ZarfComponent, error) { - for fileIdx, file := range child.Files { - composed := p.getComposedFilePath(pathAncestry, file.Source) - child.Files[fileIdx].Source = composed + // merge variables and constants + pkgVars = chain.MergeVariables(pkgVars) + pkgConsts = chain.MergeConstants(pkgConsts) } - for chartIdx, chart := range child.Charts { - for valuesIdx, valuesFile := range chart.ValuesFiles { - composed := p.getComposedFilePath(pathAncestry, valuesFile) - child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed - } - if child.Charts[chartIdx].LocalPath != "" { - composed := p.getComposedFilePath(pathAncestry, child.Charts[chartIdx].LocalPath) - child.Charts[chartIdx].LocalPath = composed - } - } - - for manifestIdx, manifest := range child.Manifests { - for fileIdx, file := range manifest.Files { - composed := p.getComposedFilePath(pathAncestry, file) - child.Manifests[manifestIdx].Files[fileIdx] = composed - } - for kustomizeIdx, kustomization := range manifest.Kustomizations { - composed := p.getComposedFilePath(pathAncestry, kustomization) - // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem - abs, _ := filepath.Abs(composed) - invalid := utils.InvalidPath(abs) - if !invalid { - child.Manifests[manifestIdx].Kustomizations[kustomizeIdx] = composed - } - } - } - - for dataInjectionsIdx, dataInjection := range child.DataInjections { - composed := p.getComposedFilePath(pathAncestry, dataInjection.Source) - child.DataInjections[dataInjectionsIdx].Source = composed - } - - var err error - - if child.Actions.OnCreate.OnSuccess, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.OnSuccess); err != nil { - return child, err - } - if child.Actions.OnCreate.OnFailure, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.OnFailure); err != nil { - return child, err - } - if child.Actions.OnCreate.Before, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.Before); err != nil { - return child, err - } - if child.Actions.OnCreate.After, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.After); err != nil { - return child, err - } - - totalActions := len(child.Actions.OnCreate.OnSuccess) + len(child.Actions.OnCreate.OnFailure) + len(child.Actions.OnCreate.Before) + len(child.Actions.OnCreate.After) - - if totalActions > 0 { - composedDefaultDir := p.getComposedFilePath(pathAncestry, child.Actions.OnCreate.Defaults.Dir) - child.Actions.OnCreate.Defaults.Dir = composedDefaultDir - } - - if child.DeprecatedCosignKeyPath != "" { - composed := p.getComposedFilePath(pathAncestry, child.DeprecatedCosignKeyPath) - child.DeprecatedCosignKeyPath = composed - } - - child = p.composeExtensions(pathAncestry, child) - - return child, nil -} - -func (p *Packager) fixComposedActionFilepaths(pathAncestry string, actions []types.ZarfComponentAction) ([]types.ZarfComponentAction, error) { - for actionIdx, action := range actions { - if action.Dir != nil { - composedActionDir := p.getComposedFilePath(pathAncestry, *action.Dir) - actions[actionIdx].Dir = &composedActionDir - } - } - - return actions, nil -} - -// Sets Name, Default, Required and Description to the original components values. -func (p *Packager) mergeComponentOverrides(target *types.ZarfComponent, override types.ZarfComponent) { - target.Name = override.Name - target.Group = override.Group - target.Default = override.Default - target.Required = override.Required - - // Override description if it was provided. - if override.Description != "" { - target.Description = override.Description - } - - // Override cosign key path if it was provided. - if override.DeprecatedCosignKeyPath != "" { - target.DeprecatedCosignKeyPath = override.DeprecatedCosignKeyPath - } - - // Append slices where they exist. - target.DataInjections = append(target.DataInjections, override.DataInjections...) - target.Files = append(target.Files, override.Files...) - target.Images = append(target.Images, override.Images...) - target.Repos = append(target.Repos, override.Repos...) - - // Merge charts with the same name to keep them unique - for _, overrideChart := range override.Charts { - existing := false - for idx := range target.Charts { - if target.Charts[idx].Name == overrideChart.Name { - if overrideChart.Namespace != "" { - target.Charts[idx].Namespace = overrideChart.Namespace - } - if overrideChart.ReleaseName != "" { - target.Charts[idx].ReleaseName = overrideChart.ReleaseName - } - target.Charts[idx].ValuesFiles = append(target.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...) - existing = true - } - } - - if !existing { - target.Charts = append(target.Charts, overrideChart) - } - } - - // Merge manifests with the same name to keep them unique - for _, overrideManifest := range override.Manifests { - existing := false - for idx := range target.Manifests { - if target.Manifests[idx].Name == overrideManifest.Name { - if overrideManifest.Namespace != "" { - target.Manifests[idx].Namespace = overrideManifest.Namespace - } - target.Manifests[idx].Files = append(target.Manifests[idx].Files, overrideManifest.Files...) - target.Manifests[idx].Kustomizations = append(target.Manifests[idx].Kustomizations, overrideManifest.Kustomizations...) - - existing = true - } - } - - if !existing { - target.Manifests = append(target.Manifests, overrideManifest) - } - } - - // Check for nil array - if override.Extensions.BigBang != nil { - if override.Extensions.BigBang.ValuesFiles != nil { - target.Extensions.BigBang.ValuesFiles = append(target.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - target.Extensions.BigBang.FluxPatchFiles = append(target.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) - } - } - - // Merge deprecated scripts for backwards compatibility with older zarf binaries. - target.DeprecatedScripts.Before = append(target.DeprecatedScripts.Before, override.DeprecatedScripts.Before...) - target.DeprecatedScripts.After = append(target.DeprecatedScripts.After, override.DeprecatedScripts.After...) - - if override.DeprecatedScripts.Retry { - target.DeprecatedScripts.Retry = true - } - if override.DeprecatedScripts.ShowOutput { - target.DeprecatedScripts.ShowOutput = true - } - if override.DeprecatedScripts.TimeoutSeconds > 0 { - target.DeprecatedScripts.TimeoutSeconds = override.DeprecatedScripts.TimeoutSeconds - } - - // Merge create actions. - target.Actions.OnCreate.Before = append(target.Actions.OnCreate.Before, override.Actions.OnCreate.Before...) - target.Actions.OnCreate.After = append(target.Actions.OnCreate.After, override.Actions.OnCreate.After...) - target.Actions.OnCreate.OnFailure = append(target.Actions.OnCreate.OnFailure, override.Actions.OnCreate.OnFailure...) - target.Actions.OnCreate.OnSuccess = append(target.Actions.OnCreate.OnSuccess, override.Actions.OnCreate.OnSuccess...) - - // Merge deploy actions. - target.Actions.OnDeploy.Before = append(target.Actions.OnDeploy.Before, override.Actions.OnDeploy.Before...) - target.Actions.OnDeploy.After = append(target.Actions.OnDeploy.After, override.Actions.OnDeploy.After...) - target.Actions.OnDeploy.OnFailure = append(target.Actions.OnDeploy.OnFailure, override.Actions.OnDeploy.OnFailure...) - target.Actions.OnDeploy.OnSuccess = append(target.Actions.OnDeploy.OnSuccess, override.Actions.OnDeploy.OnSuccess...) - - // Merge remove actions. - target.Actions.OnRemove.Before = append(target.Actions.OnRemove.Before, override.Actions.OnRemove.Before...) - target.Actions.OnRemove.After = append(target.Actions.OnRemove.After, override.Actions.OnRemove.After...) - target.Actions.OnRemove.OnFailure = append(target.Actions.OnRemove.OnFailure, override.Actions.OnRemove.OnFailure...) - target.Actions.OnRemove.OnSuccess = append(target.Actions.OnRemove.OnSuccess, override.Actions.OnRemove.OnSuccess...) - - // Merge Only filters. - target.Only.Cluster.Distros = append(target.Only.Cluster.Distros, override.Only.Cluster.Distros...) - if override.Only.Cluster.Architecture != "" { - target.Only.Cluster.Architecture = override.Only.Cluster.Architecture - } - if override.Only.LocalOS != "" { - target.Only.LocalOS = override.Only.LocalOS - } -} + // set the filtered + composed components + p.cfg.Pkg.Components = components -// Prefix file path with importPath if original file path is not a url. -func (p *Packager) getComposedFilePath(prefix string, path string) string { - // Return original if it is a remote file. - if helpers.IsURL(path) { - return path - } + p.cfg.Pkg.Variables = pkgVars + p.cfg.Pkg.Constants = pkgConsts - // Add prefix for local files. - return filepath.Join(prefix, path) + return nil } diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go new file mode 100644 index 0000000000..77ac73c038 --- /dev/null +++ b/src/pkg/packager/composer/extensions.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/types" +) + +func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { + bigbang.Compose(c, override, relativeTo) +} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go new file mode 100644 index 0000000000..0db1aaffb0 --- /dev/null +++ b/src/pkg/packager/composer/list.go @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +// Node is a node in the import chain +type Node struct { + types.ZarfComponent + + vars []types.ZarfPackageVariable + consts []types.ZarfPackageConstant + + relativeToHead string + + prev *Node + next *Node +} + +// ImportName returns the name of the component to import +// +// If the component import has a ComponentName defined, that will be used +// otherwise the name of the component will be used +func (n *Node) ImportName() string { + name := n.ZarfComponent.Name + if n.Import.ComponentName != "" { + name = n.Import.ComponentName + } + return name +} + +// ImportChain is a doubly linked list of component import definitions +type ImportChain struct { + head *Node + tail *Node + + remote *oci.OrasRemote +} + +func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + node := &Node{ + ZarfComponent: c, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, + } + if ic.head == nil { + ic.head = node + ic.tail = node + } else { + p := ic.tail + node.prev = p + p.next = node + ic.tail = node + } +} + +// NewImportChain creates a new import chain from a component +func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { + if arch == "" { + return nil, fmt.Errorf("cannot build import chain: architecture must be provided") + } + + ic := &ImportChain{} + + ic.append(head, ".", nil, nil) + + history := []string{} + + node := ic.head + for node != nil { + isLocal := node.Import.Path != "" + isRemote := node.Import.URL != "" + + if !isLocal && !isRemote { + // This is the end of the import chain, + // as the current node/component is not importing anything + return ic, nil + } + + // TODO: stuff like this should also happen in linting + if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { + return ic, err + } + + // ensure that remote components are not importing other remote components + if node.prev != nil && node.prev.Import.URL != "" && isRemote { + return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") + } + // ensure that remote components are not importing local components + if node.prev != nil && node.prev.Import.URL != "" && isLocal { + return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") + } + + var pkg types.ZarfPackage + + if isLocal { + history = append(history, node.Import.Path) + relativeToHead := filepath.Join(history...) + + // prevent circular imports (including self-imports) + // this is O(n^2) but the import chain should be small + prev := node + for prev != nil { + if prev.relativeToHead == relativeToHead { + return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) + } + prev = prev.prev + } + + // this assumes the composed package is following the zarf layout + if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { + return ic, err + } + } else if isRemote { + remote, err := ic.getRemote(node.Import.URL) + if err != nil { + return ic, err + } + pkg, err = remote.FetchZarfYAML() + if err != nil { + return ic, err + } + } + + name := node.ImportName() + + found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { + matchesName := c.Name == name + satisfiesArch := c.Only.Cluster.Architecture == "" || c.Only.Cluster.Architecture == arch + return matchesName && satisfiesArch + }) + + if len(found) == 0 { + if isLocal { + return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) + } else if isRemote { + return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) + } + } else if len(found) > 1 { + if isLocal { + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) + } else if isRemote { + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) + } + } + + ic.append(found[0], filepath.Join(history...), pkg.Variables, pkg.Constants) + node = node.next + } + return ic, nil +} + +// String returns a string representation of the import chain +func (ic *ImportChain) String() string { + if ic.head.next == nil { + return fmt.Sprintf("component %q imports nothing", ic.head.Name) + } + + s := strings.Builder{} + + name := ic.head.ImportName() + + if ic.head.Import.Path != "" { + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.Path)) + } else { + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.URL)) + } + + node := ic.head.next + for node != ic.tail { + name := node.ImportName() + s.WriteString(", which imports ") + if node.Import.Path != "" { + s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.Path)) + } else { + s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.URL)) + } + + node = node.next + } + + return s.String() +} + +// Migrate performs migrations on the import chain +func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { + node := ic.head + for node != nil { + migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) + node.ZarfComponent = migrated + warnings = append(warnings, w...) + node = node.next + } + if len(warnings) > 0 { + final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.head.Name) + warnings = append(warnings, final) + } + return warnings +} + +// Compose merges the import chain into a single component +// fixing paths, overriding metadata, etc +func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { + composed = ic.tail.ZarfComponent + + if ic.tail.prev == nil { + // only had one component in the import chain + return composed, nil + } + + if err := ic.fetchOCISkeleton(); err != nil { + return composed, err + } + + // start with an empty component to compose into + composed = types.ZarfComponent{} + + // start overriding with the tail node + node := ic.tail + for node != nil { + fixPaths(&node.ZarfComponent, node.relativeToHead) + + // perform overrides here + overrideMetadata(&composed, node.ZarfComponent) + overrideDeprecated(&composed, node.ZarfComponent) + overrideResources(&composed, node.ZarfComponent) + overrideActions(&composed, node.ZarfComponent) + + composeExtensions(&composed, node.ZarfComponent, node.relativeToHead) + + node = node.prev + } + + return composed, nil +} + +// MergeVariables merges variables from the import chain +func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (merged []types.ZarfPackageVariable) { + exists := func(v1 types.ZarfPackageVariable, v2 types.ZarfPackageVariable) bool { + return v1.Name == v2.Name + } + + merged = helpers.MergeSlices(existing, merged, exists) + node := ic.head + for node != nil { + // merge the vars + merged = helpers.MergeSlices(node.vars, merged, exists) + node = node.next + } + return merged +} + +// MergeConstants merges constants from the import chain +func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (merged []types.ZarfPackageConstant) { + exists := func(c1 types.ZarfPackageConstant, c2 types.ZarfPackageConstant) bool { + return c1.Name == c2.Name + } + + merged = helpers.MergeSlices(existing, merged, exists) + node := ic.head + for node != nil { + // merge the consts + merged = helpers.MergeSlices(node.consts, merged, exists) + node = node.next + } + return merged +} diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go new file mode 100644 index 0000000000..6d250f6e44 --- /dev/null +++ b/src/pkg/packager/composer/list_test.go @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/defenseunicorns/zarf/src/types/extensions" + "github.com/stretchr/testify/require" +) + +func TestNewImportChain(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + head types.ZarfComponent + arch string + expectedErrorMessage string + } + + testCases := []testCase{ + { + name: "No Architecture", + head: types.ZarfComponent{}, + expectedErrorMessage: "architecture must be provided", + }, + { + name: "Circular Import", + head: types.ZarfComponent{ + Import: types.ZarfComponentImport{ + Path: ".", + }, + }, + arch: "amd64", + expectedErrorMessage: "detected circular import chain", + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + _, err := NewImportChain(testCase.head, testCase.arch) + require.Contains(t, err.Error(), testCase.expectedErrorMessage) + }) + } +} + +func TestCompose(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + ic *ImportChain + returnError bool + expectedComposed types.ZarfComponent + expectedErrorMessage string + } + + firstDirectory := "hello" + secondDirectory := "world" + finalDirectory := filepath.Join(firstDirectory, secondDirectory) + + finalDirectoryActionDefault := filepath.Join(firstDirectory, secondDirectory, "today-dc") + secondDirectoryActionDefault := filepath.Join(firstDirectory, "world-dc") + firstDirectoryActionDefault := "hello-dc" + + testCases := []testCase{ + { + name: "Single Component", + ic: createChainFromSlice([]types.ZarfComponent{ + { + Name: "no-import", + }, + }), + returnError: false, + expectedComposed: types.ZarfComponent{ + Name: "no-import", + }, + }, + { + name: "Multiple Components", + ic: createChainFromSlice([]types.ZarfComponent{ + createDummyComponent("hello", firstDirectory, "hello"), + createDummyComponent("world", secondDirectory, "world"), + createDummyComponent("today", "", "hello"), + }), + returnError: false, + expectedComposed: types.ZarfComponent{ + Name: "import-hello", + // Files should always be appended with corrected directories + Files: []types.ZarfFile{ + {Source: fmt.Sprintf("%s%stoday.txt", finalDirectory, string(os.PathSeparator))}, + {Source: fmt.Sprintf("%s%sworld.txt", firstDirectory, string(os.PathSeparator))}, + {Source: "hello.txt"}, + }, + // Charts should be merged if names match and appended if not with corrected directories + Charts: []types.ZarfChart{ + { + Name: "hello", + LocalPath: fmt.Sprintf("%s%schart", finalDirectory, string(os.PathSeparator)), + ValuesFiles: []string{ + fmt.Sprintf("%s%svalues.yaml", finalDirectory, string(os.PathSeparator)), + "values.yaml", + }, + }, + { + Name: "world", + LocalPath: fmt.Sprintf("%s%schart", firstDirectory, string(os.PathSeparator)), + ValuesFiles: []string{ + fmt.Sprintf("%s%svalues.yaml", firstDirectory, string(os.PathSeparator)), + }, + }, + }, + // Manifests should be merged if names match and appended if not with corrected directories + Manifests: []types.ZarfManifest{ + { + Name: "hello", + Files: []string{ + fmt.Sprintf("%s%smanifest.yaml", finalDirectory, string(os.PathSeparator)), + "manifest.yaml", + }, + }, + { + Name: "world", + Files: []string{ + fmt.Sprintf("%s%smanifest.yaml", firstDirectory, string(os.PathSeparator)), + }, + }, + }, + // DataInjections should always be appended with corrected directories + DataInjections: []types.ZarfDataInjection{ + {Source: fmt.Sprintf("%s%stoday", finalDirectory, string(os.PathSeparator))}, + {Source: fmt.Sprintf("%s%sworld", firstDirectory, string(os.PathSeparator))}, + {Source: "hello"}, + }, + Actions: types.ZarfComponentActions{ + // OnCreate actions should be appended with corrected directories that properly handle default directories + OnCreate: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dc", + }, + Before: []types.ZarfComponentAction{ + {Cmd: "today-bc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-bc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-bc", Dir: &firstDirectoryActionDefault}, + }, + After: []types.ZarfComponentAction{ + {Cmd: "today-ac", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-ac", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-ac", Dir: &firstDirectoryActionDefault}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: "today-sc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-sc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-sc", Dir: &firstDirectoryActionDefault}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: "today-fc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-fc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-fc", Dir: &firstDirectoryActionDefault}, + }, + }, + // OnDeploy actions should be appended without corrected directories + OnDeploy: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dd", + }, + Before: []types.ZarfComponentAction{ + {Cmd: "today-bd"}, + {Cmd: "world-bd"}, + {Cmd: "hello-bd"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: "today-ad"}, + {Cmd: "world-ad"}, + {Cmd: "hello-ad"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: "today-sd"}, + {Cmd: "world-sd"}, + {Cmd: "hello-sd"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: "today-fd"}, + {Cmd: "world-fd"}, + {Cmd: "hello-fd"}, + }, + }, + // OnRemove actions should be appended without corrected directories + OnRemove: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dr", + }, + Before: []types.ZarfComponentAction{ + {Cmd: "today-br"}, + {Cmd: "world-br"}, + {Cmd: "hello-br"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: "today-ar"}, + {Cmd: "world-ar"}, + {Cmd: "hello-ar"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: "today-sr"}, + {Cmd: "world-sr"}, + {Cmd: "hello-sr"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: "today-fr"}, + {Cmd: "world-fr"}, + {Cmd: "hello-fr"}, + }, + }, + }, + // Extensions should be appended with corrected directories + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + fmt.Sprintf("%s%svalues.yaml", finalDirectory, string(os.PathSeparator)), + fmt.Sprintf("%s%svalues.yaml", firstDirectory, string(os.PathSeparator)), + "values.yaml", + }, + FluxPatchFiles: []string{ + fmt.Sprintf("%s%spatch.yaml", finalDirectory, string(os.PathSeparator)), + fmt.Sprintf("%s%spatch.yaml", firstDirectory, string(os.PathSeparator)), + "patch.yaml", + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + composed, err := testCase.ic.Compose() + if testCase.returnError { + require.Contains(t, err.Error(), testCase.expectedErrorMessage) + } else { + require.EqualValues(t, testCase.expectedComposed, composed) + } + }) + } +} + +func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { + ic = &ImportChain{} + + if len(components) == 0 { + return ic + } + + ic.append(components[0], ".", nil, nil) + history := []string{} + + for idx := 1; idx < len(components); idx++ { + history = append(history, components[idx-1].Import.Path) + ic.append(components[idx], filepath.Join(history...), nil, nil) + } + + return ic +} + +func createDummyComponent(name, importDir, subName string) types.ZarfComponent { + return types.ZarfComponent{ + Name: fmt.Sprintf("import-%s", name), + Import: types.ZarfComponentImport{ + Path: importDir, + }, + Files: []types.ZarfFile{ + { + Source: fmt.Sprintf("%s.txt", name), + }, + }, + Charts: []types.ZarfChart{ + { + Name: subName, + LocalPath: "chart", + ValuesFiles: []string{ + "values.yaml", + }, + }, + }, + Manifests: []types.ZarfManifest{ + { + Name: subName, + Files: []string{ + "manifest.yaml", + }, + }, + }, + DataInjections: []types.ZarfDataInjection{ + { + Source: name, + }, + }, + Actions: types.ZarfComponentActions{ + OnCreate: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dc", + }, + Before: []types.ZarfComponentAction{ + {Cmd: name + "-bc"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: name + "-ac"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: name + "-sc"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: name + "-fc"}, + }, + }, + OnDeploy: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dd", + }, + Before: []types.ZarfComponentAction{ + {Cmd: name + "-bd"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: name + "-ad"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: name + "-sd"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: name + "-fd"}, + }, + }, + OnRemove: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dr", + }, + Before: []types.ZarfComponentAction{ + {Cmd: name + "-br"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: name + "-ar"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: name + "-sr"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: name + "-fr"}, + }, + }, + }, + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + "values.yaml", + }, + FluxPatchFiles: []string{ + "patch.yaml", + }, + }, + }, + } +} diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go new file mode 100644 index 0000000000..5a7b4637ed --- /dev/null +++ b/src/pkg/packager/composer/oci.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + ocistore "oras.land/oras-go/v2/content/oci" +) + +func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { + if ic.remote != nil { + return ic.remote, nil + } + var err error + ic.remote, err = oci.NewOrasRemote(url) + if err != nil { + return nil, err + } + return ic.remote, nil +} + +// ContainsOCIImport returns true if the import chain contains a remote import +func (ic *ImportChain) ContainsOCIImport() bool { + // only the 2nd to last node may have a remote import + return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" +} + +func (ic *ImportChain) fetchOCISkeleton() error { + if !ic.ContainsOCIImport() { + return nil + } + node := ic.tail.prev + remote, err := ic.getRemote(node.Import.URL) + if err != nil { + return err + } + + manifest, err := remote.FetchRoot() + if err != nil { + return err + } + + name := node.ImportName() + + componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) + + cache := filepath.Join(config.GetAbsCachePath(), "oci") + if err := utils.CreateDirectory(cache, 0700); err != nil { + return err + } + + var tb, dir string + + // if there is not a tarball to fetch, create a directory named based upon + // the import url and the component name + if oci.IsEmptyDescriptor(componentDesc) { + h := sha256.New() + h.Write([]byte(node.Import.URL + name)) + id := fmt.Sprintf("%x", h.Sum(nil)) + + dir = filepath.Join(cache, "dirs", id) + + message.Debug("creating empty directory for remote component:", filepath.Join("", "oci", "dirs", id)) + } else { + tb = filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) + dir = filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) + + store, err := ocistore.New(cache) + if err != nil { + return err + } + + ctx := context.TODO() + // ensure the tarball is in the cache + exists, err := store.Exists(ctx, componentDesc) + if err != nil { + return err + } else if !exists { + copyOpts := remote.CopyOpts + // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. + copyOpts.FindSuccessors = content.Successors + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { + return err + } + } + } + + if err := utils.CreateDirectory(dir, 0700); err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + rel, err := filepath.Rel(cwd, dir) + if err != nil { + return err + } + // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache + // contrary to the other nodes, which are based upon the previous node + ic.tail.relativeToHead = rel + + if oci.IsEmptyDescriptor(componentDesc) { + // nothing was fetched, nothing to extract + return nil + } + + tu := archiver.Tar{ + OverwriteExisting: true, + // removes // from the paths + StripComponents: 1, + } + return tu.Unarchive(tb, dir) +} diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go new file mode 100644 index 0000000000..b45426dc96 --- /dev/null +++ b/src/pkg/packager/composer/override.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) { + c.Name = override.Name + c.Default = override.Default + c.Required = override.Required + + // Override description if it was provided. + if override.Description != "" { + c.Description = override.Description + } +} + +func overrideDeprecated(c *types.ZarfComponent, override types.ZarfComponent) { + // Override cosign key path if it was provided. + if override.DeprecatedCosignKeyPath != "" { + c.DeprecatedCosignKeyPath = override.DeprecatedCosignKeyPath + } + + c.Group = override.Group + + // Merge deprecated scripts for backwards compatibility with older zarf binaries. + c.DeprecatedScripts.Before = append(c.DeprecatedScripts.Before, override.DeprecatedScripts.Before...) + c.DeprecatedScripts.After = append(c.DeprecatedScripts.After, override.DeprecatedScripts.After...) + + if override.DeprecatedScripts.Retry { + c.DeprecatedScripts.Retry = true + } + if override.DeprecatedScripts.ShowOutput { + c.DeprecatedScripts.ShowOutput = true + } + if override.DeprecatedScripts.TimeoutSeconds > 0 { + c.DeprecatedScripts.TimeoutSeconds = override.DeprecatedScripts.TimeoutSeconds + } +} + +func overrideActions(c *types.ZarfComponent, override types.ZarfComponent) { + // Merge create actions. + c.Actions.OnCreate.Defaults = override.Actions.OnCreate.Defaults + c.Actions.OnCreate.Before = append(c.Actions.OnCreate.Before, override.Actions.OnCreate.Before...) + c.Actions.OnCreate.After = append(c.Actions.OnCreate.After, override.Actions.OnCreate.After...) + c.Actions.OnCreate.OnFailure = append(c.Actions.OnCreate.OnFailure, override.Actions.OnCreate.OnFailure...) + c.Actions.OnCreate.OnSuccess = append(c.Actions.OnCreate.OnSuccess, override.Actions.OnCreate.OnSuccess...) + + // Merge deploy actions. + c.Actions.OnDeploy.Defaults = override.Actions.OnDeploy.Defaults + c.Actions.OnDeploy.Before = append(c.Actions.OnDeploy.Before, override.Actions.OnDeploy.Before...) + c.Actions.OnDeploy.After = append(c.Actions.OnDeploy.After, override.Actions.OnDeploy.After...) + c.Actions.OnDeploy.OnFailure = append(c.Actions.OnDeploy.OnFailure, override.Actions.OnDeploy.OnFailure...) + c.Actions.OnDeploy.OnSuccess = append(c.Actions.OnDeploy.OnSuccess, override.Actions.OnDeploy.OnSuccess...) + + // Merge remove actions. + c.Actions.OnRemove.Defaults = override.Actions.OnRemove.Defaults + c.Actions.OnRemove.Before = append(c.Actions.OnRemove.Before, override.Actions.OnRemove.Before...) + c.Actions.OnRemove.After = append(c.Actions.OnRemove.After, override.Actions.OnRemove.After...) + c.Actions.OnRemove.OnFailure = append(c.Actions.OnRemove.OnFailure, override.Actions.OnRemove.OnFailure...) + c.Actions.OnRemove.OnSuccess = append(c.Actions.OnRemove.OnSuccess, override.Actions.OnRemove.OnSuccess...) +} + +func overrideResources(c *types.ZarfComponent, override types.ZarfComponent) { + c.DataInjections = append(c.DataInjections, override.DataInjections...) + c.Files = append(c.Files, override.Files...) + c.Images = append(c.Images, override.Images...) + c.Repos = append(c.Repos, override.Repos...) + + // Merge charts with the same name to keep them unique + for _, overrideChart := range override.Charts { + existing := false + for idx := range c.Charts { + if c.Charts[idx].Name == overrideChart.Name { + if overrideChart.Namespace != "" { + c.Charts[idx].Namespace = overrideChart.Namespace + } + if overrideChart.ReleaseName != "" { + c.Charts[idx].ReleaseName = overrideChart.ReleaseName + } + c.Charts[idx].ValuesFiles = append(c.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...) + existing = true + } + } + + if !existing { + c.Charts = append(c.Charts, overrideChart) + } + } + + // Merge manifests with the same name to keep them unique + for _, overrideManifest := range override.Manifests { + existing := false + for idx := range c.Manifests { + if c.Manifests[idx].Name == overrideManifest.Name { + if overrideManifest.Namespace != "" { + c.Manifests[idx].Namespace = overrideManifest.Namespace + } + c.Manifests[idx].Files = append(c.Manifests[idx].Files, overrideManifest.Files...) + c.Manifests[idx].Kustomizations = append(c.Manifests[idx].Kustomizations, overrideManifest.Kustomizations...) + + existing = true + } + } + + if !existing { + c.Manifests = append(c.Manifests, overrideManifest) + } + } +} diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go new file mode 100644 index 0000000000..b52c2c3257 --- /dev/null +++ b/src/pkg/packager/composer/pathfixer.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +func makePathRelativeTo(path, relativeTo string) string { + if helpers.IsURL(path) { + return path + } + + return filepath.Join(relativeTo, path) +} + +func fixPaths(child *types.ZarfComponent, relativeToHead string) { + for fileIdx, file := range child.Files { + composed := makePathRelativeTo(file.Source, relativeToHead) + child.Files[fileIdx].Source = composed + } + + for chartIdx, chart := range child.Charts { + for valuesIdx, valuesFile := range chart.ValuesFiles { + composed := makePathRelativeTo(valuesFile, relativeToHead) + child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed + } + if child.Charts[chartIdx].LocalPath != "" { + composed := makePathRelativeTo(chart.LocalPath, relativeToHead) + child.Charts[chartIdx].LocalPath = composed + } + } + + for manifestIdx, manifest := range child.Manifests { + for fileIdx, file := range manifest.Files { + composed := makePathRelativeTo(file, relativeToHead) + child.Manifests[manifestIdx].Files[fileIdx] = composed + } + for kustomizeIdx, kustomization := range manifest.Kustomizations { + composed := makePathRelativeTo(kustomization, relativeToHead) + // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem + abs, _ := filepath.Abs(composed) + invalid := utils.InvalidPath(abs) + if !invalid { + child.Manifests[manifestIdx].Kustomizations[kustomizeIdx] = composed + } + } + } + + for dataInjectionsIdx, dataInjection := range child.DataInjections { + composed := makePathRelativeTo(dataInjection.Source, relativeToHead) + child.DataInjections[dataInjectionsIdx].Source = composed + } + + defaultDir := child.Actions.OnCreate.Defaults.Dir + child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHead) + child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHead) + child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHead) + child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHead) + + // deprecated + if child.DeprecatedCosignKeyPath != "" { + composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHead) + child.DeprecatedCosignKeyPath = composed + } +} + +// fixActionPaths takes a slice of actions and mutates the Dir to be relative to the head node +func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHead string) []types.ZarfComponentAction { + for actionIdx, action := range actions { + var composed string + if action.Dir != nil { + composed = makePathRelativeTo(*action.Dir, relativeToHead) + } else { + composed = makePathRelativeTo(defaultDir, relativeToHead) + } + actions[actionIdx].Dir = &composed + } + return actions +} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 4f1f5854b0..2e09168c02 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -58,11 +58,6 @@ func (p *Packager) Create() (err error) { p.cfg.Pkg.Metadata.Version = config.CLIVersion } - // Before we compose the components (and render the imported OCI components), we need to remove any components that are not needed for a differential build - if err := p.removeDifferentialComponentsFromPackage(); err != nil { - return err - } - // Compose components into a single zarf.yaml file if err := p.composeComponents(); err != nil { return err @@ -338,6 +333,21 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkel p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" } + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + if isSkeleton { + component.Actions.OnCreate.Defaults.Dir = "" + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + } + onCreate := component.Actions.OnCreate if !isSkeleton { if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { @@ -633,11 +643,7 @@ func (p *Packager) loadDifferentialData() error { if err != nil { return err } - manifest, err := p.remote.FetchRoot() - if err != nil { - return err - } - pkg, err := p.remote.FetchZarfYAML(manifest) + pkg, err := p.remote.FetchZarfYAML() if err != nil { return err } @@ -671,44 +677,6 @@ func (p *Packager) loadDifferentialData() error { p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents = differentialZarfConfig.Build.OCIImportedComponents - - return nil -} - -// removeDifferentialComponentsFromPackage will remove unchanged OCI imported components from a differential package creation -func (p *Packager) removeDifferentialComponentsFromPackage() error { - // Remove components that were imported and already built into the reference package - if len(p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents) > 0 { - componentsToRemove := []int{} - - for idx, component := range p.cfg.Pkg.Components { - // if the component is imported from an OCI package and everything is the same, don't include this package - if helpers.IsOCIURL(component.Import.URL) { - if _, alsoExists := p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents[component.Import.URL]; alsoExists { - - // If the component spec is not empty, we will still include it in the differential package - // NOTE: We are ignoring fields that are not relevant to the differential build - if component.IsEmpty([]string{"Name", "Required", "Description", "Default", "Import"}) { - componentsToRemove = append(componentsToRemove, idx) - } - } - } - } - - // Remove the components that are already included (via OCI Import) in the reference package - if len(componentsToRemove) > 0 { - for i, componentIndex := range componentsToRemove { - indexToRemove := componentIndex - i - componentToRemove := p.cfg.Pkg.Components[indexToRemove] - - // If we are removing a component, add it to the build metadata and remove it from the list of OCI components for this package - p.cfg.Pkg.Build.DifferentialMissing = append(p.cfg.Pkg.Build.DifferentialMissing, componentToRemove.Name) - - p.cfg.Pkg.Components = append(p.cfg.Pkg.Components[:indexToRemove], p.cfg.Pkg.Components[indexToRemove+1:]...) - } - } - } return nil } diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index 3591e54a80..b5cc0ab17d 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -39,16 +39,6 @@ func (p *Packager) processExtensions() (err error) { return nil } -// Mutate any local files to be relative to the parent -func (p *Packager) composeExtensions(pathAncestry string, component types.ZarfComponent) types.ZarfComponent { - // Big Bang - if component.Extensions.BigBang != nil { - component = bigbang.Compose(pathAncestry, component) - } - - return component -} - // Check for any extensions in use and skeletonize their local files. func (p *Packager) skeletonizeExtensions() (err error) { components := []types.ZarfComponent{} diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 620dd71f4c..f8f9dd20b4 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -126,34 +126,6 @@ func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoI } } -// injectImportedVariable determines if an imported package variable exists in the active config and adds it if not. -func (p *Packager) injectImportedVariable(importedVariable types.ZarfPackageVariable) { - presentInActive := false - for _, configVariable := range p.cfg.Pkg.Variables { - if configVariable.Name == importedVariable.Name { - presentInActive = true - } - } - - if !presentInActive { - p.cfg.Pkg.Variables = append(p.cfg.Pkg.Variables, importedVariable) - } -} - -// injectImportedConstant determines if an imported package constant exists in the active config and adds it if not. -func (p *Packager) injectImportedConstant(importedConstant types.ZarfPackageConstant) { - presentInActive := false - for _, configVariable := range p.cfg.Pkg.Constants { - if configVariable.Name == importedConstant.Name { - presentInActive = true - } - } - - if !presentInActive { - p.cfg.Pkg.Constants = append(p.cfg.Pkg.Constants, importedConstant) - } -} - // findComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads func (p *Packager) findComponentTemplatesAndReload() error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 32c04ebb60..e7c991d3d8 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -29,10 +29,6 @@ func (p *Packager) readZarfYAML(path string) error { p.warnings = append(p.warnings, warning) } - if p.cfg.Pkg.Build.OCIImportedComponents == nil { - p.cfg.Pkg.Build.OCIImportedComponents = make(map[string]string) - } - if len(p.cfg.Pkg.Build.Migrations) > 0 { for idx, component := range p.cfg.Pkg.Components { // Handle component configuration deprecations diff --git a/src/pkg/transform/image.go b/src/pkg/transform/image.go index 2d511c1233..7cff153c34 100644 --- a/src/pkg/transform/image.go +++ b/src/pkg/transform/image.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/distribution/distribution/reference" + "github.com/distribution/reference" ) // Image represents a config for an OCI image. diff --git a/src/pkg/utils/helpers/misc.go b/src/pkg/utils/helpers/misc.go index fc74aca9b7..6031e21262 100644 --- a/src/pkg/utils/helpers/misc.go +++ b/src/pkg/utils/helpers/misc.go @@ -8,60 +8,9 @@ import ( "fmt" "reflect" "regexp" - "strings" "time" ) -// Unique returns a new slice with only unique elements. -func Unique[T comparable](s []T) (r []T) { - exists := make(map[T]bool) - for _, str := range s { - if _, ok := exists[str]; !ok { - exists[str] = true - r = append(r, str) - } - } - return r -} - -// Reverse returns a new slice with the elements in reverse order. -func Reverse[T any](s []T) (r []T) { - for i := len(s) - 1; i >= 0; i-- { - r = append(r, s[i]) - } - return r -} - -// Filter returns a new slice with only the elements that pass the test. -func Filter[T any](ss []T, test func(T) bool) (r []T) { - for _, s := range ss { - if test(s) { - r = append(r, s) - } - } - return r -} - -// Find returns the first element that passes the test. -func Find[T any](ss []T, test func(T) bool) (r T) { - for _, s := range ss { - if test(s) { - return s - } - } - return r -} - -// RemoveMatches removes the given element from the slice that matches the test. -func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { - for _, s := range ss { - if !test(s) { - r = append(r, s) - } - } - return r -} - // Retry will retry a function until it succeeds or the timeout is reached, timeout == retries * delay. func Retry(fn func() error, retries int, delay time.Duration) (err error) { for r := 0; r < retries; r++ { @@ -180,26 +129,3 @@ func MergeNonZero[T any](original T, overrides T) T { } return originalValue.Interface().(T) } - -// StringToSlice converts a comma-separated string to a slice of lowercase strings. -func StringToSlice(s string) []string { - if s != "" { - split := strings.Split(s, ",") - for idx, element := range split { - split[idx] = strings.ToLower(strings.TrimSpace(element)) - } - return split - } - - return []string{} -} - -// AppendIfNotExists appends a string to a slice of strings if it is not present already on the slice. -func AppendIfNotExists(slice []string, item string) []string { - for _, s := range slice { - if s == item { - return slice - } - } - return append(slice, item) -} diff --git a/src/pkg/utils/helpers/slice.go b/src/pkg/utils/helpers/slice.go new file mode 100644 index 0000000000..12b662b157 --- /dev/null +++ b/src/pkg/utils/helpers/slice.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import "strings" + +// Unique returns a new slice with only unique elements. +func Unique[T comparable](s []T) (r []T) { + exists := make(map[T]bool) + for _, str := range s { + if _, ok := exists[str]; !ok { + exists[str] = true + r = append(r, str) + } + } + return r +} + +// Reverse returns a new slice with the elements in reverse order. +func Reverse[T any](s []T) (r []T) { + for i := len(s) - 1; i >= 0; i-- { + r = append(r, s[i]) + } + return r +} + +// Filter returns a new slice with only the elements that pass the test. +func Filter[T any](ss []T, test func(T) bool) (r []T) { + for _, s := range ss { + if test(s) { + r = append(r, s) + } + } + return r +} + +// Find returns the first element that passes the test. +func Find[T any](ss []T, test func(T) bool) (r T) { + for _, s := range ss { + if test(s) { + return s + } + } + return r +} + +// RemoveMatches removes the given element from the slice that matches the test. +func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { + for _, s := range ss { + if !test(s) { + r = append(r, s) + } + } + return r +} + +// StringToSlice converts a comma-separated string to a slice of lowercase strings. +func StringToSlice(s string) []string { + if s != "" { + split := strings.Split(s, ",") + for idx, element := range split { + split[idx] = strings.ToLower(strings.TrimSpace(element)) + } + return split + } + + return []string{} +} + +// EqualFunc defines a type for a function that determines equality between two elements of type T. +type EqualFunc[T any] func(a, b T) bool + +// MergeSlices merges two slices, s1 and s2, and returns a new slice containing all elements from s1 +// and only those elements from s2 that do not exist in s1 based on the provided equal function. +func MergeSlices[T any](s1, s2 []T, equals EqualFunc[T]) []T { + merged := make([]T, 0, len(s1)+len(s2)) + merged = append(merged, s1...) + + for _, v2 := range s2 { + exists := false + for _, v1 := range s1 { + if equals(v1, v2) { + exists = true + break + } + } + if !exists { + merged = append(merged, v2) + } + } + + return merged +} diff --git a/src/pkg/utils/helpers/url.go b/src/pkg/utils/helpers/url.go index d2df69f01f..d98ceb9f5b 100644 --- a/src/pkg/utils/helpers/url.go +++ b/src/pkg/utils/helpers/url.go @@ -12,8 +12,6 @@ import ( "path" "regexp" "strconv" - - "github.com/defenseunicorns/zarf/src/config/lang" ) // Nonstandard URL schemes or prefixes @@ -55,7 +53,7 @@ func DoHostnamesMatch(url1 string, url2 string) (bool, error) { // ExtractBasePathFromURL returns filename from URL string func ExtractBasePathFromURL(urlStr string) (string, error) { if !IsURL(urlStr) { - return "", fmt.Errorf(lang.PkgValidateErrImportURLInvalid, urlStr) + return "", fmt.Errorf("%s is not a valid URL", urlStr) } parsedURL, err := url.Parse(urlStr) if err != nil { diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go new file mode 100644 index 0000000000..7afb5ac0c3 --- /dev/null +++ b/src/test/e2e/09_component_compose_test.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package test provides e2e tests for Zarf. +package test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type CompositionSuite struct { + suite.Suite + *require.Assertions +} + +var ( + composeExample = filepath.Join("examples", "composable-packages") + composeExamplePath string + composeTest = filepath.Join("src", "test", "packages", "09-composable-packages") + composeTestPath string +) + +func (suite *CompositionSuite) SetupSuite() { + suite.Assertions = require.New(suite.T()) + + // Setup the package paths after e2e has been initialized + composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch)) + composeTestPath = filepath.Join("build", fmt.Sprintf("zarf-package-test-compose-package-%s.tar.zst", e2e.Arch)) + +} + +func (suite *CompositionSuite) TearDownSuite() { + err := os.RemoveAll(composeExamplePath) + suite.NoError(err) + err = os.RemoveAll(composeTestPath) + suite.NoError(err) +} + +func (suite *CompositionSuite) Test_0_ComposabilityExample() { + suite.T().Log("E2E: Package Compose Example") + + _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") + suite.NoError(err) + + // Ensure that common names merge + suite.Contains(stdErr, ` + manifests: + - name: multi-games + namespace: dos-games + files: + - ../dos-games/manifests/deployment.yaml + - ../dos-games/manifests/service.yaml + - quake-service.yaml`) + + // Ensure that the action was appended + suite.Contains(stdErr, ` + - defenseunicorns/zarf-game:multi-tile-dark + actions: + onDeploy: + before: + - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp}`) +} + +func (suite *CompositionSuite) Test_1_FullComposability() { + suite.T().Log("E2E: Full Package Compose") + + _, stdErr, err := e2e.Zarf("package", "create", composeTest, "-o", "build", "--insecure", "--no-color", "--confirm") + suite.NoError(err) + + // Ensure that names merge and that composition is added appropriately + + // Check metadata + suite.Contains(stdErr, ` +- name: test-compose-package + description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true +`) + + // Check files + suite.Contains(stdErr, ` + files: + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt +`) + + // Check charts + suite.Contains(stdErr, ` + charts: + - name: podinfo-compose + releaseName: podinfo-override + url: oci://ghcr.io/stefanprodan/charts/podinfo + version: 6.4.0 + namespace: podinfo-override + valuesFiles: + - files/test-values.yaml + - files/test-values.yaml + - name: podinfo-compose-two + releaseName: podinfo-compose-two + url: oci://ghcr.io/stefanprodan/charts/podinfo + version: 6.4.0 + namespace: podinfo-compose-two + valuesFiles: + - files/test-values.yaml +`) + + // Check manifests + suite.Contains(stdErr, ` + manifests: + - name: connect-service + namespace: podinfo-override + files: + - files/service.yaml + - files/service.yaml + kustomizations: + - files + - files + - name: connect-service-two + namespace: podinfo-compose-two + files: + - files/service.yaml + kustomizations: + - files +`) + + // Check images + repos + suite.Contains(stdErr, ` + images: + - ghcr.io/stefanprodan/podinfo:6.4.0 + - ghcr.io/stefanprodan/podinfo:6.4.1 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons +`) + + // Check dataInjections + suite.Contains(stdErr, ` + dataInjections: + - source: files + target: + namespace: podinfo-compose + selector: app.kubernetes.io/name=podinfo-compose + container: podinfo + path: /home/app/service.yaml + - source: files + target: + namespace: podinfo-compose + selector: app.kubernetes.io/name=podinfo-compose + container: podinfo + path: /home/app/service.yaml +`) + + // Check actions + suite.Contains(stdErr, ` + actions: + onCreate: + before: + - dir: sub-package + cmd: ls + - dir: . + cmd: ls + onDeploy: + after: + - cmd: cat coffee-ipsum.txt + - wait: + cluster: + kind: deployment + name: podinfo-compose-two + namespace: podinfo-compose-two + condition: available`) +} + +func TestCompositionSuite(t *testing.T) { + e2e.SetupWithCluster(t) + + suite.Run(t, new(CompositionSuite)) +} diff --git a/src/test/e2e/32_checksum_and_signature_test.go b/src/test/e2e/32_checksum_and_signature_test.go index a4f443ff83..08f7f2cdde 100644 --- a/src/test/e2e/32_checksum_and_signature_test.go +++ b/src/test/e2e/32_checksum_and_signature_test.go @@ -17,7 +17,7 @@ func TestChecksumAndSignature(t *testing.T) { testPackageDirPath := "examples/dos-games" pkgName := fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch) - privateKeyFlag := "--key=src/test/packages/zarf-test.prv-key" + privateKeyFlag := "--signing-key=src/test/packages/zarf-test.prv-key" publicKeyFlag := "--key=src/test/packages/zarf-test.pub" stdOut, stdErr, err := e2e.Zarf("package", "create", testPackageDirPath, privateKeyFlag, "--confirm") diff --git a/src/test/e2e/50_oci_package_test.go b/src/test/e2e/50_oci_package_test.go index 888b9e09e4..b698ba6207 100644 --- a/src/test/e2e/50_oci_package_test.go +++ b/src/test/e2e/50_oci_package_test.go @@ -196,7 +196,7 @@ func (suite *RegistryClientTestSuite) Test_5_Copy() { } } -func TestRegistryClientTestSuite(t *testing.T) { +func TestRegistryClientSuite(t *testing.T) { e2e.SetupWithCluster(t) suite.Run(t, new(RegistryClientTestSuite)) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index ba87329bf4..e0cae59427 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" @@ -30,52 +29,27 @@ type SkeletonSuite struct { } var ( - composeExample = filepath.Join("examples", "composable-packages") - composeExamplePath string importEverything = filepath.Join("src", "test", "packages", "51-import-everything") importEverythingPath string importception = filepath.Join("src", "test", "packages", "51-import-everything", "inception") importceptionPath string - everythingExternal = filepath.Join("src", "test", "packages", "everything-external") - absNoCode = filepath.Join("/", "tmp", "nocode") ) func (suite *SkeletonSuite) SetupSuite() { suite.Assertions = require.New(suite.T()) - err := os.MkdirAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts"), 0755) - suite.NoError(err) - err = utils.CreatePathAndCopy(filepath.Join("examples", "helm-charts", "chart"), filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - suite.NoError(err) - suite.DirExists(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - - err = utils.CreatePathAndCopy(importEverything, everythingExternal) - suite.NoError(err) - suite.DirExists(everythingExternal) - - err = exec.CmdWithPrint("git", "clone", "https://github.com/kelseyhightower/nocode", absNoCode) - suite.NoError(err) - suite.DirExists(absNoCode) e2e.SetupDockerRegistry(suite.T(), 555) suite.Reference.Registry = "localhost:555" // Setup the package paths after e2e has been initialized - composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch)) importEverythingPath = filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)) importceptionPath = filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)) } func (suite *SkeletonSuite) TearDownSuite() { e2e.TeardownRegistry(suite.T(), 555) - err := os.RemoveAll(everythingExternal) - suite.NoError(err) - err = os.RemoveAll(absNoCode) - suite.NoError(err) - err = os.RemoveAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - suite.NoError(err) - err = os.RemoveAll("files") - suite.NoError(err) - err = os.RemoveAll(composeExamplePath) + + err := os.RemoveAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) suite.NoError(err) err = os.RemoveAll(importEverythingPath) suite.NoError(err) @@ -87,17 +61,12 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.T().Log("E2E: Skeleton Package Publish oci://") ref := suite.Reference.String() - wordpress := filepath.Join("examples", "wordpress") - _, stdErr, err := e2e.Zarf("package", "publish", wordpress, "oci://"+ref, "--insecure") - suite.NoError(err) - suite.Contains(stdErr, "Published "+ref) - helmCharts := filepath.Join("examples", "helm-charts") - _, stdErr, err = e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure") + _, stdErr, err := e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) - bigBang := filepath.Join("examples", "big-bang") + bigBang := filepath.Join("src", "test", "packages", "51-import-everything", "big-bang-min") _, stdErr, err = e2e.Zarf("package", "publish", bigBang, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -115,45 +84,11 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1-skeleton", "-o", "build", "--insecure") suite.NoError(err) - _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-example:2.12.0-skeleton", "-o", "build", "--insecure") - suite.NoError(err) -} - -func (suite *SkeletonSuite) Test_1_Compose_Example() { - suite.T().Log("E2E: Skeleton Package Compose oci://") - - _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0-skeleton", "-o", "build", "--insecure") suite.NoError(err) - - // Ensure that common names merge - suite.Contains(stdErr, ` - manifests: - - name: multi-games - namespace: dos-games - files: - - ../dos-games/manifests/deployment.yaml - - ../dos-games/manifests/service.yaml - - quake-service.yaml`) - - // Ensure that the action was appended - suite.Contains(stdErr, ` - - docker.io/bitnami/wordpress:6.2.0-debian-11-r18 - actions: - onDeploy: - before: - - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp} - setVariables: - - name: WORDPRESS_BLOG_NAME`) - - // Ensure that the variables were merged - suite.Contains(stdErr, ` -- name: WORDPRESS_BLOG_NAME - description: The blog name that is used for the WordPress admin account - default: The Zarf Blog - prompt: true`) } -func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { +func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") @@ -167,15 +102,10 @@ func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { targets := []string{ "import-component-local == import-component-local", - "import-component-local-relative == import-component-local-relative", - "import-component-wordpress == import-component-wordpress", "import-component-oci == import-component-oci", + "import-big-bang == import-big-bang", "file-imports == file-imports", - "import-helm-local == import-helm-local", - "import-helm-local-relative == import-helm-local-relative", - "import-helm-oci == import-helm-oci", - "import-repos == import-repos", - "import-images == import-images", + "local-chart-import == local-chart-import", } for _, target := range targets { @@ -183,15 +113,15 @@ func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { } } -func (suite *SkeletonSuite) Test_3_FilePaths() { - suite.T().Log("E2E: Skeleton Package File Paths") +func (suite *SkeletonSuite) Test_2_FilePaths() { + suite.T().Log("E2E: Skeleton + Package File Paths") pkgTars := []string{ filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-import-everything-skeleton-0.0.1.tar.zst"), filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst"), - filepath.Join("build", "zarf-package-big-bang-example-skeleton-2.12.0.tar.zst"), + filepath.Join("build", "zarf-package-big-bang-min-skeleton-2.10.0.tar.zst"), } for _, pkgTar := range pkgTars { diff --git a/src/test/e2e/52_oci_compose_differential_test.go b/src/test/e2e/52_oci_compose_differential_test.go deleted file mode 100644 index 78f6c815a8..0000000000 --- a/src/test/e2e/52_oci_compose_differential_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package test provides e2e tests for Zarf. -package test - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "oras.land/oras-go/v2/registry" -) - -// OCIDifferentialSuite validates that OCI imported components get handled correctly when performing a `zarf package create --differential` -type OCIDifferentialSuite struct { - suite.Suite - *require.Assertions - Reference registry.Reference - tmpdir string -} - -var ( - differentialPackageName = "" - normalPackageName = "" - examplePackagePath = filepath.Join("examples", "helm-charts") - anotherPackagePath = filepath.Join("src", "test", "packages", "52-oci-differential") -) - -func (suite *OCIDifferentialSuite) SetupSuite() { - suite.tmpdir = suite.T().TempDir() - suite.Assertions = require.New(suite.T()) - - differentialPackageName = fmt.Sprintf("zarf-package-podinfo-with-oci-flux-%s-v0.24.0-differential-v0.25.0.tar.zst", e2e.Arch) - normalPackageName = fmt.Sprintf("zarf-package-podinfo-with-oci-flux-%s-v0.24.0.tar.zst", e2e.Arch) - - e2e.SetupDockerRegistry(suite.T(), 555) - suite.Reference.Registry = "localhost:555" -} - -func (suite *OCIDifferentialSuite) TearDownSuite() { - _, _, err := exec.Cmd("docker", "rm", "-f", "registry") - suite.NoError(err) -} - -func (suite *OCIDifferentialSuite) Test_0_Create_Differential_OCI() { - suite.T().Log("E2E: Test Differential Packages w/ OCI Imports") - - // publish one of the example packages to the registry - stdOut, stdErr, err := e2e.Zarf("package", "publish", examplePackagePath, "oci://"+suite.Reference.String(), "--insecure") - suite.NoError(err, stdOut, stdErr) - - // build the package that we are going to publish - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--insecure", "--set=PACKAGE_VERSION=v0.24.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // publish the package that we just built - normalPackagePath := filepath.Join(suite.tmpdir, normalPackageName) - stdOut, stdErr, err = e2e.Zarf("package", "publish", normalPackagePath, "oci://"+suite.Reference.String(), "--insecure") - suite.NoError(err, stdOut, stdErr) - - // Build without differential - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--insecure", "--set=PACKAGE_VERSION=v0.25.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // Extract and load the zarf.yaml config for the normally built package - err = archiver.Extract(filepath.Join(suite.tmpdir, normalPackageName), "zarf.yaml", suite.tmpdir) - suite.NoError(err, "unable to extract zarf.yaml from the differential git package") - var normalZarfConfig types.ZarfPackage - err = utils.ReadYaml(filepath.Join(suite.tmpdir, "zarf.yaml"), &normalZarfConfig) - suite.NoError(err, "unable to read zarf.yaml from the differential git package") - os.Remove(filepath.Join(suite.tmpdir, "zarf.yaml")) - - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--differential", "oci://"+suite.Reference.String()+"/podinfo-with-oci-flux:v0.24.0-amd64", "--insecure", "--set=PACKAGE_VERSION=v0.25.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // Extract and load the zarf.yaml config for the differentially built package - err = archiver.Extract(filepath.Join(suite.tmpdir, differentialPackageName), "zarf.yaml", suite.tmpdir) - suite.NoError(err, "unable to extract zarf.yaml from the differential git package") - var differentialZarfConfig types.ZarfPackage - err = utils.ReadYaml(filepath.Join(suite.tmpdir, "zarf.yaml"), &differentialZarfConfig) - suite.NoError(err, "unable to read zarf.yaml from the differential git package") - - /* Perform a bunch of asserts around the non-differential package */ - // Check the metadata and build data for the normal package - suite.Equal(normalZarfConfig.Metadata.Version, "v0.24.0") - suite.False(normalZarfConfig.Build.Differential) - suite.Len(normalZarfConfig.Build.OCIImportedComponents, 1) - suite.Equal(normalZarfConfig.Build.OCIImportedComponents["oci://127.0.0.1:555/helm-charts:0.0.1-skeleton"], "demo-helm-oci-chart") - - // Check the component data for the normal package - suite.Len(normalZarfConfig.Components, 3) - suite.Equal(normalZarfConfig.Components[0].Name, "demo-helm-oci-chart") - suite.Equal(normalZarfConfig.Components[0].Charts[0].URL, "oci://ghcr.io/stefanprodan/charts/podinfo") - suite.Equal(normalZarfConfig.Components[0].Images[0], "ghcr.io/stefanprodan/podinfo:6.4.0") - suite.Len(normalZarfConfig.Components[1].Images, 2) - suite.Len(normalZarfConfig.Components[1].Repos, 2) - suite.Len(normalZarfConfig.Components[2].Images, 1) - suite.Len(normalZarfConfig.Components[2].Repos, 3) - - /* Perform a bunch of asserts around the differential package */ - // Check the metadata and build data for the differential package - suite.Equal(differentialZarfConfig.Metadata.Version, "v0.25.0") - suite.True(differentialZarfConfig.Build.Differential) - suite.Len(differentialZarfConfig.Build.DifferentialMissing, 1) - suite.Equal(differentialZarfConfig.Build.DifferentialMissing[0], "demo-helm-oci-chart") - suite.Len(differentialZarfConfig.Build.OCIImportedComponents, 0) - - // Check the component data for the differential package - suite.Len(differentialZarfConfig.Components, 2) - suite.Equal(differentialZarfConfig.Components[0].Name, "versioned-assets") - suite.Len(differentialZarfConfig.Components[0].Images, 1) - suite.Equal(differentialZarfConfig.Components[0].Images[0], "ghcr.io/defenseunicorns/zarf/agent:v0.25.0") - suite.Len(differentialZarfConfig.Components[0].Repos, 1) - suite.Equal(differentialZarfConfig.Components[0].Repos[0], "https://github.com/defenseunicorns/zarf.git@refs/tags/v0.25.0") - suite.Len(differentialZarfConfig.Components[1].Images, 1) - suite.Len(differentialZarfConfig.Components[1].Repos, 3) - suite.Equal(differentialZarfConfig.Components[1].Images[0], "ghcr.io/stefanprodan/podinfo:latest") - suite.Equal(differentialZarfConfig.Components[1].Repos[0], "https://github.com/stefanprodan/podinfo.git") -} - -func TestOCIDifferentialSuite(t *testing.T) { - e2e.SetupWithCluster(t) - - suite.Run(t, new(OCIDifferentialSuite)) -} diff --git a/src/test/nightly/ecr_publish_test.go b/src/test/nightly/ecr_publish_test.go index 8995c66095..aff3bf2aef 100644 --- a/src/test/nightly/ecr_publish_test.go +++ b/src/test/nightly/ecr_publish_test.go @@ -51,7 +51,7 @@ func TestECRPublishing(t *testing.T) { keyFlag := fmt.Sprintf("--key=%s", "./src/test/packages/zarf-test.pub") // Build the package with our test signature - stdOut, stdErr, err := e2e.Zarf("package", "create", "examples/helm-charts", "--key=./src/test/packages/zarf-test.prv-key", "--confirm", fmt.Sprintf("-o=%s", tmpDir)) + stdOut, stdErr, err := e2e.Zarf("package", "create", "examples/helm-charts", "--signing-key=./src/test/packages/zarf-test.prv-key", "--confirm", fmt.Sprintf("-o=%s", tmpDir)) require.NoError(t, err, stdOut, stdErr) require.FileExists(t, testPackageLocation) diff --git a/src/test/packages/51-import-everything/files/coffee-ipsum.txt b/src/test/packages/09-composable-packages/files/coffee-ipsum.txt similarity index 100% rename from src/test/packages/51-import-everything/files/coffee-ipsum.txt rename to src/test/packages/09-composable-packages/files/coffee-ipsum.txt diff --git a/src/test/packages/51-import-everything/files/kustomization.yaml b/src/test/packages/09-composable-packages/files/kustomization.yaml similarity index 100% rename from src/test/packages/51-import-everything/files/kustomization.yaml rename to src/test/packages/09-composable-packages/files/kustomization.yaml diff --git a/src/test/packages/09-composable-packages/files/service.yaml b/src/test/packages/09-composable-packages/files/service.yaml new file mode 100644 index 0000000000..9f25e96acd --- /dev/null +++ b/src/test/packages/09-composable-packages/files/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: podinfo + annotations: + zarf.dev/connect-description: Access Podinfo + labels: + # Enables "zarf connect podinfo" + zarf.dev/connect-name: podinfo +spec: + selector: + app.kubernetes.io/name: podinfo-compose + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: 9898 diff --git a/src/test/packages/09-composable-packages/files/test-values.yaml b/src/test/packages/09-composable-packages/files/test-values.yaml new file mode 100644 index 0000000000..b574e0dec9 --- /dev/null +++ b/src/test/packages/09-composable-packages/files/test-values.yaml @@ -0,0 +1,10 @@ +podAnnotations: + zarf.dev/dataInjections: ###ZARF_DATA_INJECTION_MARKER### + +resources: + limits: + requests: + cpu: 1m + memory: 16Mi + # Add some ephemeral storage for data injections + ephemeral-storage: 16Mi diff --git a/src/test/packages/09-composable-packages/sub-package/zarf.yaml b/src/test/packages/09-composable-packages/sub-package/zarf.yaml new file mode 100644 index 0000000000..b4ebd4511c --- /dev/null +++ b/src/test/packages/09-composable-packages/sub-package/zarf.yaml @@ -0,0 +1,44 @@ +kind: ZarfPackageConfig +metadata: + name: test-compose-sub-package + description: Deploy podinfo using a Helm OCI chart + version: 6.4.0 + +components: + - name: test-compose-sub-package + charts: + - name: podinfo-compose + releaseName: podinfo-compose + version: 6.4.0 + namespace: podinfo-compose + url: oci://ghcr.io/stefanprodan/charts/podinfo + valuesFiles: + - ../files/test-values.yaml + manifests: + - name: connect-service + namespace: podinfo-compose + files: + - ../files/service.yaml + kustomizations: + - ../files/ + images: + - ghcr.io/stefanprodan/podinfo:6.4.0 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + files: + - source: ../files/coffee-ipsum.txt + target: coffee-ipsum.txt + dataInjections: + - source: ../files + target: + selector: app.kubernetes.io/name=podinfo-compose + namespace: podinfo-compose + container: podinfo + path: /home/app/service.yaml + actions: + onCreate: + before: + - cmd: ls + onDeploy: + after: + - cmd: cat coffee-ipsum.txt diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml new file mode 100644 index 0000000000..21e40e5df2 --- /dev/null +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -0,0 +1,64 @@ +kind: ZarfPackageConfig +metadata: + name: test-compose-package + description: A contrived example for podinfo using many Zarf primitives for compose testing + +components: + - name: test-compose-package + description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true + import: + path: sub-package + name: test-compose-sub-package + charts: + - name: podinfo-compose + releaseName: podinfo-override + namespace: podinfo-override + valuesFiles: + - files/test-values.yaml + - name: podinfo-compose-two + releaseName: podinfo-compose-two + version: 6.4.0 + namespace: podinfo-compose-two + url: oci://ghcr.io/stefanprodan/charts/podinfo + valuesFiles: + - files/test-values.yaml + manifests: + - name: connect-service + namespace: podinfo-override + files: + - files/service.yaml + kustomizations: + - files/ + - name: connect-service-two + namespace: podinfo-compose-two + files: + - files/service.yaml + kustomizations: + - files/ + images: + - ghcr.io/stefanprodan/podinfo:6.4.1 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons + files: + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt + dataInjections: + - source: files + target: + selector: app.kubernetes.io/name=podinfo-compose + namespace: podinfo-compose + container: podinfo + path: /home/app/service.yaml + actions: + onCreate: + before: + - cmd: ls + onDeploy: + after: + - wait: + cluster: + kind: deployment + name: podinfo-compose-two + namespace: podinfo-compose-two + condition: available diff --git a/src/test/packages/51-import-everything/bar/zarf.yaml b/src/test/packages/51-import-everything/bar/zarf.yaml deleted file mode 100644 index 8df24ef40a..0000000000 --- a/src/test/packages/51-import-everything/bar/zarf.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: bar - version: 0.0.1 - -components: - - name: baz - required: true - files: - - source: ../files/coffee-ipsum.txt - target: files/coffee-ipsum.txt - actions: - onRemove: - after: - - cmd: rm files/coffee-ipsum.txt - onCreate: - before: - - cmd: cat files/coffee-ipsum.txt - dir: ../ - after: - - cmd: cat ../files/coffee-ipsum.txt - onSuccess: - - cmd: cat ../files/coffee-ipsum.txt diff --git a/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml b/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml new file mode 100644 index 0000000000..d5e68feaee --- /dev/null +++ b/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helm-controller + namespace: flux-system +spec: + template: + spec: + containers: + - name: manager + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 64Mi diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml new file mode 100644 index 0000000000..6fd483c848 --- /dev/null +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -0,0 +1,26 @@ +kind: ZarfPackageConfig +metadata: + name: big-bang-min + description: A minimal Big Bang package for use in testing + version: 2.10.0 + url: https://p1.dso.mil/products/big-bang + # Big Bang / Iron Bank are only amd64 + architecture: amd64 + +variables: + - name: DOMAIN + default: bigbang.dev + prompt: false + +components: + - name: bigbang + required: true + extensions: + bigbang: + repo: https://github.com/DoD-Platform-One/big-bang.git + version: 2.10.0 + skipFlux: true + fluxPatchFiles: + - flux-overrides-helm-controller.yaml + valuesFiles: + - ../../../../extensions/bigbang/test/package/disable-all-bb2.yaml diff --git a/src/test/packages/51-import-everything/foo/zarf.yaml b/src/test/packages/51-import-everything/foo/zarf.yaml deleted file mode 100644 index ed203cf430..0000000000 --- a/src/test/packages/51-import-everything/foo/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: foo - version: 0.0.1 - -components: - - name: baz - import: - path: ../bar diff --git a/src/test/packages/51-import-everything/inception/zarf.yaml b/src/test/packages/51-import-everything/inception/zarf.yaml index 032125ac10..297f4b8b41 100644 --- a/src/test/packages/51-import-everything/inception/zarf.yaml +++ b/src/test/packages/51-import-everything/inception/zarf.yaml @@ -5,37 +5,27 @@ metadata: version: 0.0.1 components: - - name: import-component-local-relative + - name: import-component-local required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-component-wordpress + - name: import-component-oci required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: file-imports - required: true - import: - url: oci://localhost:555/import-everything:0.0.1-skeleton - - - name: import-helm-local + - name: import-big-bang required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-helm-local-relative - required: true - import: - url: oci://localhost:555/import-everything:0.0.1-skeleton - - - name: import-helm-oci + - name: file-imports required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-images + - name: local-chart-import required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton diff --git a/src/test/packages/51-import-everything/oci-import/zarf.yaml b/src/test/packages/51-import-everything/oci-import/zarf.yaml new file mode 100644 index 0000000000..1173eb6e5d --- /dev/null +++ b/src/test/packages/51-import-everything/oci-import/zarf.yaml @@ -0,0 +1,13 @@ +kind: ZarfPackageConfig +metadata: + name: import-oci + description: Test OCI import of helm charts + version: 0.0.1 + +components: + - name: import-component-oci + description: "import-component-oci == ###ZARF_COMPONENT_NAME###" + required: false + import: + name: demo-helm-local-chart + url: oci://localhost:555/helm-charts:0.0.1-skeleton diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index e24239ceca..5807c1431e 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -5,50 +5,39 @@ metadata: version: 0.0.1 components: + # Test every simple primitive that Zarf has through a local import - name: import-component-local description: "import-component-local == ###ZARF_COMPONENT_NAME###" required: false import: - path: foo - name: baz + path: ../09-composable-packages + name: test-compose-package - - name: import-component-local-relative - description: "import-component-local-relative == ###ZARF_COMPONENT_NAME###" - required: false - import: - path: ../../../../examples/dos-games - name: baseline - manifests: - - name: override - kustomizations: - - files - - - name: import-component-wordpress - description: "import-component-wordpress == ###ZARF_COMPONENT_NAME###" + # Test nested local to oci imports + - name: import-component-oci + description: "import-component-oci == ###ZARF_COMPONENT_NAME###" required: false import: - path: ../../../../examples/wordpress - name: wordpress + name: import-component-oci + path: oci-import - - name: import-component-oci - description: "import-component-oci == ###ZARF_COMPONENT_NAME###" + # Test big bang extension files + - name: import-big-bang + description: "import-big-bang == ###ZARF_COMPONENT_NAME###" required: false import: - # name is optional, if not provided the name of the component will be used to import - name: demo-helm-local-chart - url: oci://localhost:555/helm-charts:0.0.1-skeleton + name: bigbang + url: oci://localhost:555/big-bang-min:2.10.0-skeleton + # Test file imports including cosignKeyPath - name: file-imports - description: "file-imports == ###ZARF_COMPONENT_NAME###" + description: "file-imports == ###ZARF_COMPONENT_NAME###" required: false cosignKeyPath: ../../../../cosign.pub files: - # Import of a file within the current directory - - source: files/coffee-ipsum.txt - target: files/coffee-ipsum.txt - # Import of a file from a relative path external to the current directory - - source: ../everything-external/files/coffee-ipsum.txt - target: files/latte.txt + # Import of a local file + - source: ../09-composable-packages/files/coffee-ipsum.txt + target: ../09-composable-packages/coffee-ipsum.txt # Import of a file from a URL - source: https://raw.githubusercontent.com/defenseunicorns/zarf/main/README.md target: files/zarf-readme.md @@ -56,24 +45,22 @@ components: onDeploy: after: - cmd: test -f files/coffee-ipsum.txt - - cmd: test -f files/latte.txt - cmd: test -f files/zarf-readme.md onRemove: before: - cmd: rm files/coffee-ipsum.txt - - cmd: rm files/latte.txt - cmd: rm files/zarf-readme.md after: - cmd: test ! -f files/coffee-ipsum.txt - - cmd: test ! -f files/latte.txt - cmd: test ! -f files/zarf-readme.md - - name: import-helm-local - description: "import-helm-local == ###ZARF_COMPONENT_NAME###" + # Test local charts (for skeletons) + - name: local-chart-import + description: "local-chart-import == ###ZARF_COMPONENT_NAME###" required: false charts: - name: podinfo - localPath: charts/local + localPath: ../../../../examples/helm-charts/chart namespace: local-chart version: 6.4.0 images: @@ -87,65 +74,3 @@ components: name: podinfo namespace: local-chart condition: available - - - name: import-helm-local-relative - description: "import-helm-local-relative == ###ZARF_COMPONENT_NAME###" - required: false - charts: - - name: podinfo - localPath: ../everything-external/charts/local - namespace: local-chart-relative - version: 6.4.0 - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 - actions: - onDeploy: - after: - - wait: - cluster: - kind: deployment - name: podinfo - namespace: local-chart-relative - condition: available - - - name: import-helm-oci - description: "import-helm-oci == ###ZARF_COMPONENT_NAME###" - required: false - charts: - - name: oci-demo - version: 6.4.0 - namespace: podinfo - url: oci://ghcr.io/stefanprodan/charts/podinfo - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 - dataInjections: - - source: files/coffee-ipsum.txt - target: - namespace: podinfo - selector: app.kubernetes.io/name=oci-demo-podinfo - container: podinfo - path: /files/coffee-ipsum.txt - actions: - onDeploy: - after: - - wait: - cluster: - kind: pod - name: app.kubernetes.io/name=oci-demo-podinfo - namespace: podinfo - condition: ready - - - name: import-repos - description: "import-repos == ###ZARF_COMPONENT_NAME###" - required: false - repos: - # Import a full repo via HTTPS - - https://github.com/kelseyhightower/nocode.git - # Import a full repo via file:// + absolute path - - file:///tmp/nocode - - - name: import-images - description: "import-images == ###ZARF_COMPONENT_NAME###" - required: false - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 diff --git a/src/test/packages/52-oci-differential/README.md b/src/test/packages/52-oci-differential/README.md deleted file mode 100644 index 0da88acc22..0000000000 --- a/src/test/packages/52-oci-differential/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# OCI Differential - -This test package is used to test the functionality of creating a Zarf Package while using the `--differential` flag where one of the components in the package we're creating is a component that has been imported from an OCI registry. - -This test package demonstrates that OCI imported components will not be included during differential package creation and that the proper build metadata will be added to the finalized package to ensure users of the package know which OCI imported components were not included. - -This test also includes components for more standard differential package creation to make sure all of that expected functionality remains the same when there are also OCI imported components in the package. diff --git a/src/test/packages/52-oci-differential/zarf.yaml b/src/test/packages/52-oci-differential/zarf.yaml deleted file mode 100644 index 75e194f5e6..0000000000 --- a/src/test/packages/52-oci-differential/zarf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: podinfo-with-oci-flux - description: Deploy flux and then podinfo via flux - version: "###ZARF_PKG_TMPL_PACKAGE_VERSION###" - -components: - - name: demo-helm-oci-chart - required: true - import: - url: oci://127.0.0.1:555/helm-charts:0.0.1-skeleton - - - name: versioned-assets - import: - path: ../../packages/08-differential-package - name: versioned-assets - - - name: generalized-assets - import: - path: ../../packages/08-differential-package - name: generalized-assets diff --git a/src/types/component.go b/src/types/component.go index 89b8e00111..df5f37c836 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -91,12 +91,12 @@ type ZarfFile struct { type ZarfChart struct { Name string `json:"name" jsonschema:"description=The name of the chart to deploy; this should be the name of the chart as it is installed in the helm repo"` ReleaseName string `json:"releaseName,omitempty" jsonschema:"description=The name of the release to create; defaults to the name of the chart"` - URL string `json:"url,omitempty" jsonschema:"oneof_required=url,example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"` + URL string `json:"url,omitempty" jsonschema:"example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"` Version string `json:"version,omitempty" jsonschema:"description=The version of the chart to deploy; for git-based charts this is also the tag of the git repo"` Namespace string `json:"namespace" jsonschema:"description=The namespace to deploy the chart to"` ValuesFiles []string `json:"valuesFiles,omitempty" jsonschema:"description=List of local values file paths or remote URLs to include in the package; these will be merged together"` GitPath string `json:"gitPath,omitempty" jsonschema:"description=The path to the chart in the repo if using a git repo instead of a helm repo,example=charts/your-chart"` - LocalPath string `json:"localPath,omitempty" jsonschema:"oneof_required=localPath,description=The path to the chart folder"` + LocalPath string `json:"localPath,omitempty" jsonschema:"description=The path to the chart folder"` NoWait bool `json:"noWait,omitempty" jsonschema:"description=Whether to not wait for chart resources to be ready before continuing"` } diff --git a/src/types/package.go b/src/types/package.go index b089081a82..6b44fe8a16 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -52,7 +52,6 @@ type ZarfBuildData struct { Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` - OCIImportedComponents map[string]string `json:"OCIImportedComponents,omitempty" jsonschema:"description=Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names"` LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` } diff --git a/src/types/runtime.go b/src/types/runtime.go index 65c347b882..2a29484f19 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -127,5 +127,4 @@ type DifferentialData struct { DifferentialPackageVersion string DifferentialImages map[string]bool DifferentialRepos map[string]bool - DifferentialOCIComponents map[string]string } diff --git a/zarf.schema.json b/zarf.schema.json index 483c37ea02..f2ff08fd64 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -132,15 +132,6 @@ "type": "array", "description": "List of components that were not included in this package due to differential packaging" }, - "OCIImportedComponents": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object", - "description": "Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names" - }, "lastNonBreakingVersion": { "type": "string", "description": "The minimum version of Zarf that does not have breaking package structure changes" @@ -204,21 +195,7 @@ } }, "additionalProperties": false, - "type": "object", - "oneOf": [ - { - "required": [ - "url" - ], - "title": "url" - }, - { - "required": [ - "localPath" - ], - "title": "localPath" - } - ] + "type": "object" }, "ZarfComponent": { "required": [