From 26e1d6ddbd0b24f0923a3905f86cdcdfa95baa84 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 10 Jan 2024 18:44:58 -0600 Subject: [PATCH 001/172] Initial commit to split cdToBaseDir into a new creator package This introduces the Creator interface to be used as the source of truth for package create operations. --- src/pkg/packager/create.go | 5 ++++- src/pkg/packager/create_stages.go | 13 ------------- src/pkg/packager/creator/cd.go | 24 ++++++++++++++++++++++++ src/pkg/packager/creator/new.go | 14 ++++++++++++++ src/pkg/packager/creator/normal.go | 16 ++++++++++++++++ src/pkg/packager/creator/skeleton.go | 16 ++++++++++++++++ src/pkg/packager/dev.go | 5 ++++- src/pkg/packager/publish.go | 4 +++- 8 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 src/pkg/packager/creator/cd.go create mode 100644 src/pkg/packager/creator/new.go create mode 100644 src/pkg/packager/creator/normal.go create mode 100644 src/pkg/packager/creator/skeleton.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 02f13b240b..14d86a12ee 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -20,7 +21,9 @@ func (p *Packager) Create() (err error) { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + c := creator.New(&p.cfg.CreateOpts) + + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2ae7f06ad9..e21a2343af 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -32,19 +32,6 @@ import ( "github.com/mholt/archiver/v3" ) -func (p *Packager) cdToBaseDir(base string, cwd string) error { - if err := os.Chdir(base); err != nil { - return fmt.Errorf("unable to access directory %q: %w", base, err) - } - message.Note(fmt.Sprintf("Using build directory %s", base)) - - // differentials are relative to the current working directory - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} - func (p *Packager) load() error { if err := p.readZarfYAML(layout.ZarfYAML); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) diff --git a/src/pkg/packager/creator/cd.go b/src/pkg/packager/creator/cd.go new file mode 100644 index 0000000000..65689cd476 --- /dev/null +++ b/src/pkg/packager/creator/cd.go @@ -0,0 +1,24 @@ +package creator + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// cdToBaseDir changes the current working directory to the specified base directory. +func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + if err := os.Chdir(createOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) + } + message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) + + // differentials are relative to the current working directory + if createOpts.DifferentialData.DifferentialPackagePath != "" { + createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go new file mode 100644 index 0000000000..7fbe2ca1d6 --- /dev/null +++ b/src/pkg/packager/creator/new.go @@ -0,0 +1,14 @@ +package creator + +import "github.com/defenseunicorns/zarf/src/types" + +type Creator interface { + CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error +} + +func New(createOpts *types.ZarfCreateOptions) Creator { + if createOpts.IsSkeleton { + return &SkeletonCreator{} + } + return &PackageCreator{} +} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go new file mode 100644 index 0000000000..e13d2aa4da --- /dev/null +++ b/src/pkg/packager/creator/normal.go @@ -0,0 +1,16 @@ +package creator + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + // verify that PackageCreator implements Creator + _ Creator = (*PackageCreator)(nil) +) + +type PackageCreator struct{} + +func (p *PackageCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + return cdToBaseDir(createOpts, cwd) +} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go new file mode 100644 index 0000000000..813e41ee46 --- /dev/null +++ b/src/pkg/packager/creator/skeleton.go @@ -0,0 +1,16 @@ +package creator + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + // verify that SkeletonCreator implements Creator + _ Creator = (*SkeletonCreator)(nil) +) + +type SkeletonCreator struct{} + +func (p *SkeletonCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + return cdToBaseDir(createOpts, cwd) +} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b4e1744d46..66ce238fb9 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/types" ) @@ -24,7 +25,9 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + c := creator.New(&p.cfg.CreateOpts) + + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6022eeada2..05dcbc4a28 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -88,7 +89,8 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + c := creator.New(&p.cfg.CreateOpts) + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } if err := p.load(); err != nil { From f26e88f8e6ef2f40b1fa154893cd6cb10c87823d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 10 Jan 2024 19:03:39 -0600 Subject: [PATCH 002/172] Fix lint warnings --- src/pkg/packager/creator/cd.go | 4 ++++ src/pkg/packager/creator/new.go | 6 ++++++ src/pkg/packager/creator/normal.go | 6 ++++++ src/pkg/packager/creator/skeleton.go | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/src/pkg/packager/creator/cd.go b/src/pkg/packager/creator/cd.go index 65689cd476..c5c91d7488 100644 --- a/src/pkg/packager/creator/cd.go +++ b/src/pkg/packager/creator/cd.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. package creator import ( diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 7fbe2ca1d6..41f8d18833 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -1,11 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. package creator import "github.com/defenseunicorns/zarf/src/types" +// Creator is an interface for creating Zarf packages. type Creator interface { CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error } +// New returns a new Creator based on the provided create options. func New(createOpts *types.ZarfCreateOptions) Creator { if createOpts.IsSkeleton { return &SkeletonCreator{} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e13d2aa4da..7016ce5c28 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. package creator import ( @@ -9,8 +13,10 @@ var ( _ Creator = (*PackageCreator)(nil) ) +// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct{} +// CdToBaseDir changes the current working directory to the specified base directory. func (p *PackageCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { return cdToBaseDir(createOpts, cwd) } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 813e41ee46..c84f8ac3aa 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. package creator import ( @@ -9,8 +13,10 @@ var ( _ Creator = (*SkeletonCreator)(nil) ) +// SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct{} +// CdToBaseDir changes the current working directory to the specified base directory. func (p *SkeletonCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { return cdToBaseDir(createOpts, cwd) } From 83a3cf2fd8f80edb912f1f7fccbead7e01221955 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 11:46:41 -0600 Subject: [PATCH 003/172] Remove deprecated migrations logic from readZarfYAML deprecated.MigrateComponent is called in composeComponents so we should be able to remove this duplicate logic --- src/pkg/packager/yaml.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index ba38a31bf1..ff4442b0e8 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -18,8 +18,6 @@ import ( // readZarfYAML reads a Zarf YAML file. func (p *Packager) readZarfYAML(path string) error { - var warnings []string - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { return err } @@ -29,14 +27,6 @@ func (p *Packager) readZarfYAML(path string) error { p.warnings = append(p.warnings, warning) } - if len(p.cfg.Pkg.Build.Migrations) > 0 { - for idx, component := range p.cfg.Pkg.Components { - // Handle component configuration deprecations - p.cfg.Pkg.Components[idx], warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - p.warnings = append(p.warnings, warnings...) - } - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) return nil From f8e65694918b2aa1c99e777c644551187be38f02 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 12:39:46 -0600 Subject: [PATCH 004/172] Decouple running deprecated migrations and adding package warnings from readZarfYAML The goal here is separation of concerns --- src/pkg/packager/deploy.go | 2 ++ src/pkg/packager/migrate.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/pkg/packager/migrate.go diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index a4a2408431..0e8c6e5c99 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -48,6 +48,8 @@ func (p *Packager) Deploy() (err error) { return err } + p.runMigrations() + if err := p.validateLastNonBreakingVersion(); err != nil { return err } diff --git a/src/pkg/packager/migrate.go b/src/pkg/packager/migrate.go new file mode 100644 index 0000000000..1fd91d4ff6 --- /dev/null +++ b/src/pkg/packager/migrate.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + +func (p *Packager) runMigrations() { + var warnings []string + + if len(p.cfg.Pkg.Build.Migrations) > 0 { + for idx, component := range p.cfg.Pkg.Components { + // Handle component configuration deprecations + p.cfg.Pkg.Components[idx], warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, component) + p.warnings = append(p.warnings, warnings...) + } + } +} From 99cb14e1e30afd8b59a8ab49258fb08df3009dba Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 13:30:05 -0600 Subject: [PATCH 005/172] Move legacy layout warning logic to runMigrations Goal here is separation of concerns --- src/pkg/packager/migrate.go | 5 +++++ src/pkg/packager/yaml.go | 7 ------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/migrate.go b/src/pkg/packager/migrate.go index 1fd91d4ff6..f0e652a0a8 100644 --- a/src/pkg/packager/migrate.go +++ b/src/pkg/packager/migrate.go @@ -9,6 +9,11 @@ import "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" func (p *Packager) runMigrations() { var warnings []string + if p.layout.IsLegacyLayout() { + warning := "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0" + p.warnings = append(p.warnings, warning) + } + if len(p.cfg.Pkg.Build.Migrations) > 0 { for idx, component := range p.cfg.Pkg.Components { // Handle component configuration deprecations diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index ff4442b0e8..913f17c07c 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -21,14 +21,7 @@ func (p *Packager) readZarfYAML(path string) error { if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { return err } - - if p.layout.IsLegacyLayout() { - warning := "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0" - p.warnings = append(p.warnings, warning) - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - return nil } From b5420c0e1a15945eae528988e90b15668d91b55d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 16:32:35 -0600 Subject: [PATCH 006/172] Extract set package arch logic to new receiver method Use utils.ReadYaml in place of readZarfYAML --- src/pkg/packager/common.go | 26 +++++++++++++++----------- src/pkg/packager/create_stages.go | 3 ++- src/pkg/packager/deploy.go | 4 ++-- src/pkg/packager/inspect.go | 4 +++- src/pkg/packager/mirror.go | 4 +++- src/pkg/packager/prepare.go | 3 ++- src/pkg/packager/publish.go | 3 ++- src/pkg/packager/remove.go | 4 +++- src/pkg/packager/yaml.go | 9 --------- 9 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index e1cbfb0a20..4c603867f7 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -151,17 +151,6 @@ func NewOrDie(config *types.PackagerConfig, mods ...Modifier) *Packager { return pkgr } -// setTempDirectory sets the temp directory for the packager. -func (p *Packager) setTempDirectory(path string) error { - dir, err := utils.MakeTempDir(path) - if err != nil { - return fmt.Errorf("unable to create package temp paths: %w", err) - } - - p.layout = layout.New(dir) - return nil -} - // GetInitPackageName returns the formatted name of the init package. func GetInitPackageName(arch string) string { if arch == "" { @@ -200,6 +189,21 @@ func (p *Packager) ClearTempPaths() { _ = os.RemoveAll(layout.SBOMDir) } +// setTempDirectory sets the temp directory for the packager. +func (p *Packager) setTempDirectory(path string) error { + dir, err := utils.MakeTempDir(path) + if err != nil { + return fmt.Errorf("unable to create package temp paths: %w", err) + } + + p.layout = layout.New(dir) + return nil +} + +func (p *Packager) setArch() { + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) +} + // connectToCluster attempts to connect to a cluster if a connection is not already established func (p *Packager) connectToCluster(timeout time.Duration) (err error) { if p.isConnectedToCluster() { diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index e21a2343af..b7a2b45039 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -33,12 +33,13 @@ import ( ) func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } + p.setArch() // Compose components into a single zarf.yaml file if err := p.composeComponents(); err != nil { diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 0e8c6e5c99..e7902acdb3 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -44,10 +44,10 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - + p.setArch() p.runMigrations() if err := p.validateLastNonBreakingVersion(); err != nil { diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 993c9ffeb0..2fa154499c 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -6,6 +6,7 @@ package packager import ( "github.com/defenseunicorns/zarf/src/internal/packager/sbom" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -17,9 +18,10 @@ func (p *Packager) Inspect() (err error) { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.setArch() utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 689ddcfbaa..1d1b06cd2d 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,7 +9,9 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -21,7 +23,7 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 3bdde325d9..60b6b32b52 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -47,9 +47,10 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } + p.setArch() if err := p.composeComponents(); err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 05dcbc4a28..512ac52130 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -103,7 +104,7 @@ func (p *Packager) Publish() (err error) { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } } diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 2aaee0c3af..a09952ff8e 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -14,8 +14,10 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -38,7 +40,7 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } p.filterComponents() diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 913f17c07c..5a1b4fd462 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -16,15 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// readZarfYAML reads a Zarf YAML file. -func (p *Packager) readZarfYAML(path string) error { - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - return nil -} - // filterComponents removes components not matching the current OS if filterByOS is set. func (p *Packager) filterComponents() { // Filter each component to only compatible platforms. From 80fd2ecdca102dd4efbbc965e2ca45b5d6db03a0 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 21:03:35 -0600 Subject: [PATCH 007/172] Revert "Extract set package arch logic to new receiver method" This reverts commit b5420c0e1a15945eae528988e90b15668d91b55d. --- src/pkg/packager/common.go | 26 +++++++++++--------------- src/pkg/packager/create_stages.go | 3 +-- src/pkg/packager/deploy.go | 4 ++-- src/pkg/packager/inspect.go | 4 +--- src/pkg/packager/mirror.go | 4 +--- src/pkg/packager/prepare.go | 3 +-- src/pkg/packager/publish.go | 3 +-- src/pkg/packager/remove.go | 4 +--- src/pkg/packager/yaml.go | 9 +++++++++ 9 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 4c603867f7..e1cbfb0a20 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -151,6 +151,17 @@ func NewOrDie(config *types.PackagerConfig, mods ...Modifier) *Packager { return pkgr } +// setTempDirectory sets the temp directory for the packager. +func (p *Packager) setTempDirectory(path string) error { + dir, err := utils.MakeTempDir(path) + if err != nil { + return fmt.Errorf("unable to create package temp paths: %w", err) + } + + p.layout = layout.New(dir) + return nil +} + // GetInitPackageName returns the formatted name of the init package. func GetInitPackageName(arch string) string { if arch == "" { @@ -189,21 +200,6 @@ func (p *Packager) ClearTempPaths() { _ = os.RemoveAll(layout.SBOMDir) } -// setTempDirectory sets the temp directory for the packager. -func (p *Packager) setTempDirectory(path string) error { - dir, err := utils.MakeTempDir(path) - if err != nil { - return fmt.Errorf("unable to create package temp paths: %w", err) - } - - p.layout = layout.New(dir) - return nil -} - -func (p *Packager) setArch() { - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) -} - // connectToCluster attempts to connect to a cluster if a connection is not already established func (p *Packager) connectToCluster(timeout time.Duration) (err error) { if p.isConnectedToCluster() { diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index b7a2b45039..e21a2343af 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -33,13 +33,12 @@ import ( ) func (p *Packager) load() error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err := p.readZarfYAML(layout.ZarfYAML); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } - p.setArch() // Compose components into a single zarf.yaml file if err := p.composeComponents(); err != nil { diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index e7902acdb3..0e8c6e5c99 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -44,10 +44,10 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.setArch() + p.runMigrations() if err := p.validateLastNonBreakingVersion(); err != nil { diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 2fa154499c..993c9ffeb0 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -6,7 +6,6 @@ package packager import ( "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -18,10 +17,9 @@ func (p *Packager) Inspect() (err error) { return err } - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.setArch() utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 1d1b06cd2d..689ddcfbaa 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,9 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -23,7 +21,7 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 60b6b32b52..3bdde325d9 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -47,10 +47,9 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(layout.ZarfYAML); err != nil { return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } - p.setArch() if err := p.composeComponents(); err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 512ac52130..05dcbc4a28 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -104,7 +103,7 @@ func (p *Packager) Publish() (err error) { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } } diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index a09952ff8e..2aaee0c3af 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -14,10 +14,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -40,7 +38,7 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err = utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } p.filterComponents() diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 5a1b4fd462..913f17c07c 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -16,6 +16,15 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// readZarfYAML reads a Zarf YAML file. +func (p *Packager) readZarfYAML(path string) error { + if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { + return err + } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + return nil +} + // filterComponents removes components not matching the current OS if filterByOS is set. func (p *Packager) filterComponents() { // Filter each component to only compatible platforms. From 8e9790bd4c8274296e3002e328e575422a70747f Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 21:25:37 -0600 Subject: [PATCH 008/172] Split reading zarf.yaml and setting arch apart --- src/pkg/packager/create_stages.go | 6 ++++-- src/pkg/packager/deploy.go | 3 ++- src/pkg/packager/inspect.go | 4 +++- src/pkg/packager/mirror.go | 4 +++- src/pkg/packager/prepare.go | 6 +++--- src/pkg/packager/publish.go | 3 ++- src/pkg/packager/remove.go | 5 ++++- src/pkg/packager/yaml.go | 9 --------- 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index e21a2343af..bc437f2b37 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -33,9 +33,11 @@ import ( ) func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 0e8c6e5c99..4158159a28 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -44,9 +44,10 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.runMigrations() diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 993c9ffeb0..525b5c0196 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -5,6 +5,7 @@ package packager import ( + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -17,9 +18,10 @@ func (p *Packager) Inspect() (err error) { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 689ddcfbaa..524daba13d 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -21,9 +22,10 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.stageSBOMViewFiles(); err != nil { return err diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 3bdde325d9..fcce9510af 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -16,7 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/k8s" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -47,9 +46,10 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return nil, err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.composeComponents(); err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 05dcbc4a28..e22095ed03 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -103,9 +103,10 @@ func (p *Packager) Publish() (err error) { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) } // Get a reference to the registry for this package diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 2aaee0c3af..d552943252 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -38,9 +39,11 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 913f17c07c..5a1b4fd462 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -16,15 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// readZarfYAML reads a Zarf YAML file. -func (p *Packager) readZarfYAML(path string) error { - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - return nil -} - // filterComponents removes components not matching the current OS if filterByOS is set. func (p *Packager) filterComponents() { // Filter each component to only compatible platforms. From fbda210ed29970af0b244aa58117df4af009d4bb Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 21:30:07 -0600 Subject: [PATCH 009/172] Revert "Split reading zarf.yaml and setting arch apart" This reverts commit 8e9790bd4c8274296e3002e328e575422a70747f. --- src/pkg/packager/create_stages.go | 6 ++---- src/pkg/packager/deploy.go | 3 +-- src/pkg/packager/inspect.go | 4 +--- src/pkg/packager/mirror.go | 4 +--- src/pkg/packager/prepare.go | 6 +++--- src/pkg/packager/publish.go | 3 +-- src/pkg/packager/remove.go | 5 +---- src/pkg/packager/yaml.go | 9 +++++++++ 8 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index bc437f2b37..e21a2343af 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -33,11 +33,9 @@ import ( ) func (p *Packager) load() error { - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return err + if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 4158159a28..0e8c6e5c99 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -44,10 +44,9 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.runMigrations() diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 525b5c0196..993c9ffeb0 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -5,7 +5,6 @@ package packager import ( - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -18,10 +17,9 @@ func (p *Packager) Inspect() (err error) { return err } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 524daba13d..689ddcfbaa 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -10,7 +10,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -22,10 +21,9 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.stageSBOMViewFiles(); err != nil { return err diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index fcce9510af..3bdde325d9 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/k8s" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -46,10 +47,9 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return nil, err + if err = p.readZarfYAML(layout.ZarfYAML); err != nil { + return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.composeComponents(); err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index e22095ed03..05dcbc4a28 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -103,10 +103,9 @@ func (p *Packager) Publish() (err error) { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) } // Get a reference to the registry for this package diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index d552943252..2aaee0c3af 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -16,7 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -39,11 +38,9 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 5a1b4fd462..913f17c07c 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -16,6 +16,15 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// readZarfYAML reads a Zarf YAML file. +func (p *Packager) readZarfYAML(path string) error { + if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { + return err + } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + return nil +} + // filterComponents removes components not matching the current OS if filterByOS is set. func (p *Packager) filterComponents() { // Filter each component to only compatible platforms. From cef36605d9eeca29cce177888182e4b767da6e92 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 11 Jan 2024 21:41:50 -0600 Subject: [PATCH 010/172] Split reading zarf.yaml and setting arch apart Ensure ReadYaml is passed the correct path --- src/pkg/packager/create_stages.go | 6 ++++-- src/pkg/packager/deploy.go | 3 ++- src/pkg/packager/inspect.go | 4 +++- src/pkg/packager/mirror.go | 4 +++- src/pkg/packager/prepare.go | 5 +++-- src/pkg/packager/publish.go | 3 ++- src/pkg/packager/remove.go | 5 ++++- src/pkg/packager/yaml.go | 9 --------- 8 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index e21a2343af..62c0a68683 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -33,9 +33,11 @@ import ( ) func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 0e8c6e5c99..4158159a28 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -44,9 +44,10 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.runMigrations() diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 993c9ffeb0..525b5c0196 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -5,6 +5,7 @@ package packager import ( + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -17,9 +18,10 @@ func (p *Packager) Inspect() (err error) { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 689ddcfbaa..524daba13d 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -21,9 +22,10 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.stageSBOMViewFiles(); err != nil { return err diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 3bdde325d9..c3fb5e7543 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -47,9 +47,10 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return nil, err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.composeComponents(); err != nil { return nil, err diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 05dcbc4a28..e22095ed03 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -103,9 +103,10 @@ func (p *Packager) Publish() (err error) { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) } // Get a reference to the registry for this package diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 2aaee0c3af..d552943252 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -38,9 +39,11 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 913f17c07c..5a1b4fd462 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -16,15 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// readZarfYAML reads a Zarf YAML file. -func (p *Packager) readZarfYAML(path string) error { - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - return nil -} - // filterComponents removes components not matching the current OS if filterByOS is set. func (p *Packager) filterComponents() { // Filter each component to only compatible platforms. From d7b8eed37fe6b476c47f577efa7157ee94e711b7 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 00:05:03 -0600 Subject: [PATCH 011/172] Add a Loader interface for loading package definitions on create "Normal" packages and skeleton packages require different loading processes, so using an interface allows us to have two separate implementations for them. This makes the load logic easier to reason about and easier to maintain. Loader is defined in package packager because most of the operations require direct access to the top-level Packager struct fields. --- src/pkg/packager/create.go | 3 +- src/pkg/packager/create_stages.go | 54 --------------- src/pkg/packager/dev.go | 3 +- src/pkg/packager/load.go | 109 ++++++++++++++++++++++++++++++ src/pkg/packager/publish.go | 3 +- 5 files changed, 115 insertions(+), 57 deletions(-) create mode 100644 src/pkg/packager/load.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 14d86a12ee..e72c4d2a60 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,7 +27,8 @@ func (p *Packager) Create() (err error) { return err } - if err := p.load(); err != nil { + loader := NewLoader(&p.cfg.CreateOpts) + if err := loader.LoadPackageDefinition(p); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 62c0a68683..3ca20a1764 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -5,7 +5,6 @@ package packager import ( - "errors" "fmt" "os" "path/filepath" @@ -32,59 +31,6 @@ import ( "github.com/mholt/archiver/v3" ) -func (p *Packager) load() error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { - return err - } - - if p.cfg.CreateOpts.IsSkeleton { - return nil - } - - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { - return err - } - - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - - return nil -} - func (p *Packager) assemble() error { componentSBOMs := map[string]*layout.ComponentSBOM{} var imageList []transform.Image diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 66ce238fb9..bfc6c9f2e4 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -31,7 +31,8 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.load(); err != nil { + loader := NewLoader(&p.cfg.CreateOpts) + if err := loader.LoadPackageDefinition(p); err != nil { return err } diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go new file mode 100644 index 0000000000..dbaf25e815 --- /dev/null +++ b/src/pkg/packager/load.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "errors" + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + // verify that SkeletonLoader implements Loader + _ Loader = (*SkeletonLoader)(nil) + + // verify that PackageLoader implements Loader + _ Loader = (*PackageLoader)(nil) +) + +// Loader is an interface for loading and configuring package definitions when creating Zarf packages. +type Loader interface { + LoadPackageDefinition(*Packager) error +} + +// NewLoader returns a new Loader based on the provided create options. +func NewLoader(createOpts *types.ZarfCreateOptions) Loader { + if createOpts.IsSkeleton { + return &SkeletonLoader{} + } + return &PackageLoader{} +} + +type SkeletonLoader struct{} + +func (sl *SkeletonLoader) LoadPackageDefinition(pkgr *Packager) error { + if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { + return err + } + + pkgr.arch = config.GetArch(pkgr.cfg.Pkg.Metadata.Architecture, pkgr.cfg.Pkg.Build.Architecture) + + if pkgr.isInitConfig() { + pkgr.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + if err := pkgr.composeComponents(); err != nil { + return err + } + + return nil +} + +type PackageLoader struct{} + +func (pl *PackageLoader) LoadPackageDefinition(pkgr *Packager) error { + if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { + return err + } + pkgr.arch = config.GetArch(pkgr.cfg.Pkg.Metadata.Architecture, pkgr.cfg.Pkg.Build.Architecture) + + if pkgr.isInitConfig() { + pkgr.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + if err := pkgr.composeComponents(); err != nil { + return err + } + + // After components are composed, template the active package. + if err := pkgr.fillActiveTemplate(); err != nil { + return fmt.Errorf("unable to fill values in template: %s", err.Error()) + } + + // After templates are filled process any create extensions + if err := pkgr.processExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + // Load the images and repos from the 'reference' package + if err := pkgr.loadDifferentialData(); err != nil { + return err + } + // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building + // If the package versions are the same return an error + if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pkgr.cfg.Pkg.Metadata.Version { + return errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pkgr.cfg.Pkg.Metadata.Version == "" { + return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + } + + // Handle any potential differential images/repos before going forward + if err := pkgr.removeCopiesFromDifferentialPackage(); err != nil { + return err + } + } + + return nil +} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index e22095ed03..a5f066b12b 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -93,7 +93,8 @@ func (p *Packager) Publish() (err error) { if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := p.load(); err != nil { + loader := NewLoader(&p.cfg.CreateOpts) + if err := loader.LoadPackageDefinition(p); err != nil { return err } if err := p.assembleSkeleton(); err != nil { From 902d966cd72ac06850f89bf2d1e5ed9273ed0eb6 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 00:16:48 -0600 Subject: [PATCH 012/172] Add comments to load code to fix linter warnings --- src/pkg/packager/load.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index dbaf25e815..80c7253475 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -23,7 +23,7 @@ var ( _ Loader = (*PackageLoader)(nil) ) -// Loader is an interface for loading and configuring package definitions when creating Zarf packages. +// Loader is an interface for loading and configuring package definitions during package create. type Loader interface { LoadPackageDefinition(*Packager) error } @@ -36,8 +36,10 @@ func NewLoader(createOpts *types.ZarfCreateOptions) Loader { return &PackageLoader{} } +// SkeletonLoader is used to load and configure skeleton Zarf packages during package create. type SkeletonLoader struct{} +// LoadPackageDefinition loads and configures skeleton Zarf packages during package create. func (sl *SkeletonLoader) LoadPackageDefinition(pkgr *Packager) error { if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { return err @@ -57,8 +59,10 @@ func (sl *SkeletonLoader) LoadPackageDefinition(pkgr *Packager) error { return nil } +// PackageLoader is used to load and configure normal (not skeleton) Zarf packages during package create. type PackageLoader struct{} +// LoadPackageDefinition loads and configures normal (not skeleton) Zarf packages during package create. func (pl *PackageLoader) LoadPackageDefinition(pkgr *Packager) error { if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { return err From 4e56132f72417cceab0f9cfd71d1d9773a957cdb Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 01:05:28 -0600 Subject: [PATCH 013/172] Move differential functions out of create_stages.go to differential.go --- src/pkg/packager/create_stages.go | 120 ----------------- src/pkg/packager/differential.go | 142 ++++++++++++++++++++ src/test/e2e/08_create_differential_test.go | 4 +- 3 files changed, 144 insertions(+), 122 deletions(-) create mode 100644 src/pkg/packager/differential.go diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 3ca20a1764..2380d1d91a 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -27,7 +27,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" "github.com/mholt/archiver/v3" ) @@ -576,122 +575,3 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Calculate the checksum of the checksum file return utils.GetSHA256OfFile(checksumsFilePath) } - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil -} diff --git a/src/pkg/packager/differential.go b/src/pkg/packager/differential.go new file mode 100644 index 0000000000..995f738655 --- /dev/null +++ b/src/pkg/packager/differential.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying zarf packages. +package packager + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mholt/archiver/v3" +) + +// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package +func (p *Packager) loadDifferentialData() error { + // Save the fact that this is a differential build into the build data of the package + p.cfg.Pkg.Build.Differential = true + + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { + remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + pkg, err := remote.FetchZarfYAML() + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return err + } + } else { + if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return nil +} + +// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package +func (p *Packager) removeCopiesFromDifferentialPackage() error { + // If a differential build was not requested, continue on as normal + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { + return nil + } + + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range p.cfg.Pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + p.cfg.Pkg.Components[idx] = component + } + + return nil +} diff --git a/src/test/e2e/08_create_differential_test.go b/src/test/e2e/08_create_differential_test.go index 5d7db9fedc..458e67a9b5 100644 --- a/src/test/e2e/08_create_differential_test.go +++ b/src/test/e2e/08_create_differential_test.go @@ -23,8 +23,8 @@ func TestCreateDifferential(t *testing.T) { cachePath := filepath.Join(tmpdir, ".cache-location") packagePath := "src/test/packages/08-differential-package" - packageName := "zarf-package-differential-package-amd64-v0.25.0.tar.zst" - differentialPackageName := "zarf-package-differential-package-amd64-v0.25.0-differential-v0.26.0.tar.zst" + packageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0.tar.zst", e2e.Arch) + differentialPackageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0-differential-v0.26.0.tar.zst", e2e.Arch) differentialFlag := fmt.Sprintf("--differential=%s", packageName) // Build the package a first time From c5e83df102e2936b93f0f736bf5e1af0c82394f2 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 02:00:39 -0600 Subject: [PATCH 014/172] Move Creator interface to packager package Create Assembler interface Creator interface embeds Loader and Assembler interfaces to allow various create methods to be accessed through a single interface. Create operations are (and should be) closely coupled to Packager, so it makes sense to have create functions in the packager package. Modularity, separation of concerns, library usage, etc can be achieved through the use of these interfaces while remaining in package packager. --- src/pkg/packager/assemble.go | 151 +++++++++++++++++++++++++++ src/pkg/packager/create.go | 53 ++++++++-- src/pkg/packager/create_stages.go | 120 --------------------- src/pkg/packager/creator/cd.go | 28 ----- src/pkg/packager/creator/new.go | 20 ---- src/pkg/packager/creator/normal.go | 22 ---- src/pkg/packager/creator/skeleton.go | 22 ---- src/pkg/packager/dev.go | 12 +-- src/pkg/packager/load.go | 9 -- src/pkg/packager/publish.go | 10 +- 10 files changed, 206 insertions(+), 241 deletions(-) create mode 100644 src/pkg/packager/assemble.go delete mode 100644 src/pkg/packager/creator/cd.go delete mode 100644 src/pkg/packager/creator/new.go delete mode 100644 src/pkg/packager/creator/normal.go delete mode 100644 src/pkg/packager/creator/skeleton.go diff --git a/src/pkg/packager/assemble.go b/src/pkg/packager/assemble.go new file mode 100644 index 0000000000..3390340e2f --- /dev/null +++ b/src/pkg/packager/assemble.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying zarf packages. +package packager + +import ( + "fmt" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/images" + "github.com/defenseunicorns/zarf/src/internal/packager/sbom" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" +) + +var ( + // verify that SkeletonAssembler implements Assembler + _ Assembler = (*SkeletonAssembler)(nil) + + // verify that PackageAssembler implements Assembler + _ Assembler = (*PackageAssembler)(nil) +) + +type Assembler interface { + Assemble(*Packager) error +} + +type SkeletonAssembler struct{} + +func (sa *SkeletonAssembler) Assemble(p *Packager) error { + if err := p.skeletonizeExtensions(); err != nil { + return err + } + for _, warning := range p.warnings { + message.Warn(warning) + } + for idx, component := range p.cfg.Pkg.Components { + if err := p.addComponent(idx, component); err != nil { + return err + } + + if err := p.layout.Components.Archive(component, false); err != nil { + return err + } + } + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return p.writeYaml() +} + +type PackageAssembler struct{} + +func (pa *PackageAssembler) Assemble(p *Packager) error { + componentSBOMs := map[string]*layout.ComponentSBOM{} + var imageList []transform.Image + for idx, component := range p.cfg.Pkg.Components { + onCreate := component.Actions.OnCreate + onFailure := func() { + if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + if err := p.addComponent(idx, component); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !p.cfg.CreateOpts.SkipSBOM { + componentSBOM, err := p.getFilesToSBOM(component) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + p.layout = p.layout.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: p.layout.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if p.cfg.CreateOpts.SkipSBOM { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + p.layout = p.layout.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index e72c4d2a60..63499efcc2 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -7,12 +7,38 @@ package packager import ( "fmt" "os" + "path/filepath" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" ) +// Creator is an interface for creating Zarf packages. +type Creator interface { + Loader + Assembler +} + +// New returns a new Creator based on the provided create options. +func NewCreator(createOpts *types.ZarfCreateOptions) Creator { + if createOpts.IsSkeleton { + return &SkeletonCreator{} + } + return &PackageCreator{} +} + +type SkeletonCreator struct { + *SkeletonLoader + *SkeletonAssembler +} + +type PackageCreator struct { + *PackageLoader + *PackageAssembler +} + // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { @@ -21,14 +47,13 @@ func (p *Packager) Create() (err error) { return err } - c := creator.New(&p.cfg.CreateOpts) - - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - loader := NewLoader(&p.cfg.CreateOpts) - if err := loader.LoadPackageDefinition(p); err != nil { + c := NewCreator(&p.cfg.CreateOpts) + + if err := c.LoadPackageDefinition(p); err != nil { return err } @@ -41,7 +66,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := p.assemble(); err != nil { + if err := c.Assemble(p); err != nil { return err } @@ -52,3 +77,17 @@ func (p *Packager) Create() (err error) { return p.output() } + +// cdToBaseDir changes the current working directory to the specified base directory. +func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + if err := os.Chdir(createOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) + } + message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) + + // differentials are relative to the current working directory + if createOpts.DifferentialData.DifferentialPackagePath != "" { + createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2380d1d91a..8f5f5f7d67 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -11,142 +11,22 @@ import ( "slices" "strconv" "strings" - "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ) -func (p *Packager) assemble() error { - componentSBOMs := map[string]*layout.ComponentSBOM{} - var imageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - if err := p.addComponent(idx, component); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if !p.cfg.CreateOpts.SkipSBOM { - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - return fmt.Errorf("unable to create component SBOM: %w", err) - } - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - imageList = append(imageList, refInfo) - } - } - - imageList = helpers.Unique(imageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - var err error - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - return nil -} - -func (p *Packager) assembleSkeleton() error { - if err := p.skeletonizeExtensions(); err != nil { - return err - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range p.cfg.Pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} - // output assumes it is running from cwd, not the build directory func (p *Packager) output() error { // Process the component directories into compressed tarballs diff --git a/src/pkg/packager/creator/cd.go b/src/pkg/packager/creator/cd.go deleted file mode 100644 index c5c91d7488..0000000000 --- a/src/pkg/packager/creator/cd.go +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" -) - -// cdToBaseDir changes the current working directory to the specified base directory. -func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - if err := os.Chdir(createOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) - } - message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) - - // differentials are relative to the current working directory - if createOpts.DifferentialData.DifferentialPackagePath != "" { - createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go deleted file mode 100644 index 41f8d18833..0000000000 --- a/src/pkg/packager/creator/new.go +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import "github.com/defenseunicorns/zarf/src/types" - -// Creator is an interface for creating Zarf packages. -type Creator interface { - CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error -} - -// New returns a new Creator based on the provided create options. -func New(createOpts *types.ZarfCreateOptions) Creator { - if createOpts.IsSkeleton { - return &SkeletonCreator{} - } - return &PackageCreator{} -} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go deleted file mode 100644 index 7016ce5c28..0000000000 --- a/src/pkg/packager/creator/normal.go +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "github.com/defenseunicorns/zarf/src/types" -) - -var ( - // verify that PackageCreator implements Creator - _ Creator = (*PackageCreator)(nil) -) - -// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. -type PackageCreator struct{} - -// CdToBaseDir changes the current working directory to the specified base directory. -func (p *PackageCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - return cdToBaseDir(createOpts, cwd) -} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go deleted file mode 100644 index c84f8ac3aa..0000000000 --- a/src/pkg/packager/creator/skeleton.go +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "github.com/defenseunicorns/zarf/src/types" -) - -var ( - // verify that SkeletonCreator implements Creator - _ Creator = (*SkeletonCreator)(nil) -) - -// SkeletonCreator provides methods for creating skeleton Zarf packages. -type SkeletonCreator struct{} - -// CdToBaseDir changes the current working directory to the specified base directory. -func (p *SkeletonCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - return cdToBaseDir(createOpts, cwd) -} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index bfc6c9f2e4..fba271a1e5 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/types" ) @@ -25,14 +24,13 @@ func (p *Packager) DevDeploy() error { return err } - c := creator.New(&p.cfg.CreateOpts) - - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - loader := NewLoader(&p.cfg.CreateOpts) - if err := loader.LoadPackageDefinition(p); err != nil { + c := NewCreator(&p.cfg.CreateOpts) + + if err := c.LoadPackageDefinition(p); err != nil { return err } @@ -58,7 +56,7 @@ func (p *Packager) DevDeploy() error { } } - if err := p.assemble(); err != nil { + if err := c.Assemble(p); err != nil { return err } diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index 80c7253475..ae2537c55e 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -12,7 +12,6 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" ) var ( @@ -28,14 +27,6 @@ type Loader interface { LoadPackageDefinition(*Packager) error } -// NewLoader returns a new Loader based on the provided create options. -func NewLoader(createOpts *types.ZarfCreateOptions) Loader { - if createOpts.IsSkeleton { - return &SkeletonLoader{} - } - return &PackageLoader{} -} - // SkeletonLoader is used to load and configure skeleton Zarf packages during package create. type SkeletonLoader struct{} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index a5f066b12b..22d722d7c3 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -89,15 +88,14 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - c := creator.New(&p.cfg.CreateOpts) - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - loader := NewLoader(&p.cfg.CreateOpts) - if err := loader.LoadPackageDefinition(p); err != nil { + c := NewCreator(&p.cfg.CreateOpts) + if err := c.LoadPackageDefinition(p); err != nil { return err } - if err := p.assembleSkeleton(); err != nil { + if err := c.Assemble(p); err != nil { return err } } else { From b5d37cba9fc282a1940876de0ecf71c6649dbe62 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 02:16:09 -0600 Subject: [PATCH 015/172] Add comments to fix linter warnings --- src/pkg/packager/assemble.go | 5 +++++ src/pkg/packager/create.go | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/assemble.go b/src/pkg/packager/assemble.go index 3390340e2f..98e7bf21ed 100644 --- a/src/pkg/packager/assemble.go +++ b/src/pkg/packager/assemble.go @@ -25,12 +25,15 @@ var ( _ Assembler = (*PackageAssembler)(nil) ) +// Assembler is an interface for assembling package assets (components, images, SBOMSs, etc) during package create. type Assembler interface { Assemble(*Packager) error } +// SkeletonAssembler is used to assemble assets for skeleton Zarf packages during package create. type SkeletonAssembler struct{} +// Assemble assembles assets for skeleton Zarf packages during package create. func (sa *SkeletonAssembler) Assemble(p *Packager) error { if err := p.skeletonizeExtensions(); err != nil { return err @@ -56,8 +59,10 @@ func (sa *SkeletonAssembler) Assemble(p *Packager) error { return p.writeYaml() } +// PackageAssembler is used to assemble assets for normal (not skeleton) Zarf packages during package create. type PackageAssembler struct{} +// Assemble assembles assets for normal (not skeleton) Zarf packages during package create. func (pa *PackageAssembler) Assemble(p *Packager) error { componentSBOMs := map[string]*layout.ComponentSBOM{} var imageList []transform.Image diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 63499efcc2..b53baf1e07 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -21,7 +21,7 @@ type Creator interface { Assembler } -// New returns a new Creator based on the provided create options. +// NewCreator returns a new Creator based on the provided create options. func NewCreator(createOpts *types.ZarfCreateOptions) Creator { if createOpts.IsSkeleton { return &SkeletonCreator{} @@ -78,7 +78,6 @@ func (p *Packager) Create() (err error) { return p.output() } -// cdToBaseDir changes the current working directory to the specified base directory. func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { if err := os.Chdir(createOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) From 47dc136c01d76c9205311092b83102dbe67cacfe Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 10:43:07 -0600 Subject: [PATCH 016/172] Add comments to SkeletonCreator and PackageCreator structs --- src/pkg/packager/create.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index b53baf1e07..f4dcb6cdfe 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -29,11 +29,13 @@ func NewCreator(createOpts *types.ZarfCreateOptions) Creator { return &PackageCreator{} } +// SkeletonAssembler is used to create skeleton Zarf packages. type SkeletonCreator struct { *SkeletonLoader *SkeletonAssembler } +// PackageCreator is used to create normal (not skeleton) Zarf packages. type PackageCreator struct { *PackageLoader *PackageAssembler From f2b2f038b36556591a8bd39331e76c2e2e5cf926 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 10:51:08 -0600 Subject: [PATCH 017/172] Improve comments for differential methods --- src/pkg/packager/differential.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/differential.go b/src/pkg/packager/differential.go index 995f738655..36e6361c8d 100644 --- a/src/pkg/packager/differential.go +++ b/src/pkg/packager/differential.go @@ -22,7 +22,9 @@ import ( "github.com/mholt/archiver/v3" ) -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package +// loadDifferentialData extracts the Zarf config of a designated 'reference' package used for building a differential package. +// +// It creates a list of all images and repositories that are in the reference package. func (p *Packager) loadDifferentialData() error { // Save the fact that this is a differential build into the build data of the package p.cfg.Pkg.Build.Differential = true @@ -74,7 +76,11 @@ func (p *Packager) loadDifferentialData() error { return nil } -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package +// removeCopiesFromDifferentialPackage removes any images and repositories that are already in the reference package from the new package. +// +// For each component in the new package, it checks if any images or repositories are duplicates from the reference package. +// +// Duplicate images and repositories are excluded, and the component lists are updated accordingly. func (p *Packager) removeCopiesFromDifferentialPackage() error { // If a differential build was not requested, continue on as normal if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { From f94c7a0024b1cf4e3fc04e372b8107dbd6287c8d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 10:58:15 -0600 Subject: [PATCH 018/172] Remove redundant err != nil check --- src/pkg/packager/load.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index ae2537c55e..02768ad053 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -43,11 +43,7 @@ func (sl *SkeletonLoader) LoadPackageDefinition(pkgr *Packager) error { } // Compose components into a single zarf.yaml file - if err := pkgr.composeComponents(); err != nil { - return err - } - - return nil + return pkgr.composeComponents() } // PackageLoader is used to load and configure normal (not skeleton) Zarf packages during package create. From d84ccd4f6da41238171b652ff35dd077a7e74c04 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 11:30:28 -0600 Subject: [PATCH 019/172] Do some cleanup for differential code --- src/config/lang/english.go | 9 +++++---- src/pkg/packager/load.go | 19 +++++++------------ src/test/e2e/08_create_differential_test.go | 5 +++-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 6a666474d7..b256d47c75 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -618,12 +618,13 @@ const ( AgentErrUnableTransform = "unable to transform the provided request; see zarf http proxy logs for more details" ) -// src/internal/packager/create +// Package create const ( - PkgCreateErrDifferentialSameVersion = "unable to create a differential package with the same version as the package you are using as a reference; the package version must be incremented" + PkgCreateErrDifferentialSameVersion = "unable to create differential package. Please ensure the differential package version and reference package version are not the same. The package version must be incremented." + PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure the package version is set." ) -// src/internal/packager/deploy. +// Package deploy const ( PkgDeployErrMultipleComponentsSameGroup = "You cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag." PkgDeployErrNoDefaultOrSelection = "You must make a selection from %q with the --components flag as there is no default in their group." @@ -631,7 +632,7 @@ const ( PkgDeployErrComponentSelectionCanceled = "Component selection canceled: %s" ) -// src/internal/packager/validate. +// Package validate const ( PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index 02768ad053..08bc7dfd52 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -75,25 +75,20 @@ func (pl *PackageLoader) LoadPackageDefinition(pkgr *Packager) error { return err } - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + // If we are building a differential package, remove duplicate repos and images. if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package if err := pkgr.loadDifferentialData(); err != nil { return err } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pkgr.cfg.Pkg.Metadata.Version { + versionsMatch := pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pkgr.cfg.Pkg.Metadata.Version + if versionsMatch { return errors.New(lang.PkgCreateErrDifferentialSameVersion) } - if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pkgr.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := pkgr.removeCopiesFromDifferentialPackage(); err != nil { - return err + noVersionSet := pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pkgr.cfg.Pkg.Metadata.Version == "" + if noVersionSet { + return errors.New(lang.PkgCreateErrDifferentialNoVersion) } + return pkgr.removeCopiesFromDifferentialPackage() } return nil diff --git a/src/test/e2e/08_create_differential_test.go b/src/test/e2e/08_create_differential_test.go index 458e67a9b5..1121f3514b 100644 --- a/src/test/e2e/08_create_differential_test.go +++ b/src/test/e2e/08_create_differential_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -16,7 +17,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestCreateDifferential creates several differential packages and ensures the already built images and repos and not included in the new package +// TestCreateDifferential creates several differential packages and ensures the reference package images and repos are not included in the new package. func TestCreateDifferential(t *testing.T) { t.Log("E2E: Test Differential Package Behavior") tmpdir := t.TempDir() @@ -35,7 +36,7 @@ func TestCreateDifferential(t *testing.T) { // Build the differential package without changing the version _, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.25.0", differentialFlag, "--confirm") require.Error(t, err, "zarf package create should have errored when a differential package was being created without updating the package version number") - require.Contains(t, stdErr, "unable to create a differential package with the same version") + require.Contains(t, e2e.StripMessageFormatting(stdErr), lang.PkgCreateErrDifferentialSameVersion) // Build the differential package stdOut, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.26.0", differentialFlag, "--confirm") From ff5e3ea66c112791d384b08fcf63ae5dba81c52c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 11:35:38 -0600 Subject: [PATCH 020/172] Fix SkeletonCreator comment --- src/pkg/packager/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index f4dcb6cdfe..1a0fc3ce53 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -29,7 +29,7 @@ func NewCreator(createOpts *types.ZarfCreateOptions) Creator { return &PackageCreator{} } -// SkeletonAssembler is used to create skeleton Zarf packages. +// SkeletonCreator is used to create skeleton Zarf packages. type SkeletonCreator struct { *SkeletonLoader *SkeletonAssembler From c2daf04852a3967db6bd631d4c02c7eb3659c650 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 11:41:14 -0600 Subject: [PATCH 021/172] Cleanup for consistency --- src/pkg/packager/assemble.go | 4 ++-- src/pkg/packager/load.go | 38 ++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/assemble.go b/src/pkg/packager/assemble.go index 98e7bf21ed..9d1a4bf084 100644 --- a/src/pkg/packager/assemble.go +++ b/src/pkg/packager/assemble.go @@ -34,7 +34,7 @@ type Assembler interface { type SkeletonAssembler struct{} // Assemble assembles assets for skeleton Zarf packages during package create. -func (sa *SkeletonAssembler) Assemble(p *Packager) error { +func (*SkeletonAssembler) Assemble(p *Packager) error { if err := p.skeletonizeExtensions(); err != nil { return err } @@ -63,7 +63,7 @@ func (sa *SkeletonAssembler) Assemble(p *Packager) error { type PackageAssembler struct{} // Assemble assembles assets for normal (not skeleton) Zarf packages during package create. -func (pa *PackageAssembler) Assemble(p *Packager) error { +func (*PackageAssembler) Assemble(p *Packager) error { componentSBOMs := map[string]*layout.ComponentSBOM{} var imageList []transform.Image for idx, component := range p.cfg.Pkg.Components { diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index 08bc7dfd52..4b95cd393f 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -31,64 +31,64 @@ type Loader interface { type SkeletonLoader struct{} // LoadPackageDefinition loads and configures skeleton Zarf packages during package create. -func (sl *SkeletonLoader) LoadPackageDefinition(pkgr *Packager) error { - if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { +func (*SkeletonLoader) LoadPackageDefinition(p *Packager) error { + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - pkgr.arch = config.GetArch(pkgr.cfg.Pkg.Metadata.Architecture, pkgr.cfg.Pkg.Build.Architecture) + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if pkgr.isInitConfig() { - pkgr.cfg.Pkg.Metadata.Version = config.CLIVersion + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion } // Compose components into a single zarf.yaml file - return pkgr.composeComponents() + return p.composeComponents() } // PackageLoader is used to load and configure normal (not skeleton) Zarf packages during package create. type PackageLoader struct{} // LoadPackageDefinition loads and configures normal (not skeleton) Zarf packages during package create. -func (pl *PackageLoader) LoadPackageDefinition(pkgr *Packager) error { - if err := utils.ReadYaml(layout.ZarfYAML, &pkgr.cfg.Pkg); err != nil { +func (*PackageLoader) LoadPackageDefinition(p *Packager) error { + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - pkgr.arch = config.GetArch(pkgr.cfg.Pkg.Metadata.Architecture, pkgr.cfg.Pkg.Build.Architecture) + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if pkgr.isInitConfig() { - pkgr.cfg.Pkg.Metadata.Version = config.CLIVersion + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion } // Compose components into a single zarf.yaml file - if err := pkgr.composeComponents(); err != nil { + if err := p.composeComponents(); err != nil { return err } // After components are composed, template the active package. - if err := pkgr.fillActiveTemplate(); err != nil { + if err := p.fillActiveTemplate(); err != nil { return fmt.Errorf("unable to fill values in template: %s", err.Error()) } // After templates are filled process any create extensions - if err := pkgr.processExtensions(); err != nil { + if err := p.processExtensions(); err != nil { return err } // If we are building a differential package, remove duplicate repos and images. - if pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - if err := pkgr.loadDifferentialData(); err != nil { + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + if err := p.loadDifferentialData(); err != nil { return err } - versionsMatch := pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pkgr.cfg.Pkg.Metadata.Version + versionsMatch := p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version if versionsMatch { return errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := pkgr.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pkgr.cfg.Pkg.Metadata.Version == "" + noVersionSet := p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" if noVersionSet { return errors.New(lang.PkgCreateErrDifferentialNoVersion) } - return pkgr.removeCopiesFromDifferentialPackage() + return p.removeCopiesFromDifferentialPackage() } return nil From 4400e108b8f2e6bd87e7add91690a0d877269d8d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 13:08:13 -0600 Subject: [PATCH 022/172] Remove nested interfaces to reduce unnecessary complexity Add CdToBaseDir to Creator interface so that it is grouped together with other create-specific operations. --- src/pkg/packager/assemble.go | 73 +++++++++++++----------------------- src/pkg/packager/create.go | 43 +++++++-------------- src/pkg/packager/creator.go | 36 ++++++++++++++++++ src/pkg/packager/dev.go | 7 ++-- src/pkg/packager/load.go | 53 +++++++++----------------- src/pkg/packager/publish.go | 4 +- 6 files changed, 98 insertions(+), 118 deletions(-) create mode 100644 src/pkg/packager/creator.go diff --git a/src/pkg/packager/assemble.go b/src/pkg/packager/assemble.go index 9d1a4bf084..671cb0f209 100644 --- a/src/pkg/packager/assemble.go +++ b/src/pkg/packager/assemble.go @@ -17,53 +17,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) -var ( - // verify that SkeletonAssembler implements Assembler - _ Assembler = (*SkeletonAssembler)(nil) - - // verify that PackageAssembler implements Assembler - _ Assembler = (*PackageAssembler)(nil) -) - -// Assembler is an interface for assembling package assets (components, images, SBOMSs, etc) during package create. -type Assembler interface { - Assemble(*Packager) error -} - -// SkeletonAssembler is used to assemble assets for skeleton Zarf packages during package create. -type SkeletonAssembler struct{} - -// Assemble assembles assets for skeleton Zarf packages during package create. -func (*SkeletonAssembler) Assemble(p *Packager) error { - if err := p.skeletonizeExtensions(); err != nil { - return err - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range p.cfg.Pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} - -// PackageAssembler is used to assemble assets for normal (not skeleton) Zarf packages during package create. -type PackageAssembler struct{} - // Assemble assembles assets for normal (not skeleton) Zarf packages during package create. -func (*PackageAssembler) Assemble(p *Packager) error { +func (*PackageCreator) Assemble(p *Packager) error { componentSBOMs := map[string]*layout.ComponentSBOM{} var imageList []transform.Image for idx, component := range p.cfg.Pkg.Components { @@ -154,3 +109,29 @@ func (*PackageAssembler) Assemble(p *Packager) error { return nil } + +// Assemble assembles assets for skeleton Zarf packages during package create. +func (*SkeletonCreator) Assemble(p *Packager) error { + if err := p.skeletonizeExtensions(); err != nil { + return err + } + for _, warning := range p.warnings { + message.Warn(warning) + } + for idx, component := range p.cfg.Pkg.Components { + if err := p.addComponent(idx, component); err != nil { + return err + } + + if err := p.layout.Components.Archive(component, false); err != nil { + return err + } + } + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return p.writeYaml() +} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 1a0fc3ce53..42aeae90de 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -15,32 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// Creator is an interface for creating Zarf packages. -type Creator interface { - Loader - Assembler -} - -// NewCreator returns a new Creator based on the provided create options. -func NewCreator(createOpts *types.ZarfCreateOptions) Creator { - if createOpts.IsSkeleton { - return &SkeletonCreator{} - } - return &PackageCreator{} -} - -// SkeletonCreator is used to create skeleton Zarf packages. -type SkeletonCreator struct { - *SkeletonLoader - *SkeletonAssembler -} - -// PackageCreator is used to create normal (not skeleton) Zarf packages. -type PackageCreator struct { - *PackageLoader - *PackageAssembler -} - // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { @@ -49,12 +23,11 @@ func (p *Packager) Create() (err error) { return err } - if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { - return err - } - c := NewCreator(&p.cfg.CreateOpts) + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + return err + } if err := c.LoadPackageDefinition(p); err != nil { return err } @@ -80,6 +53,16 @@ func (p *Packager) Create() (err error) { return p.output() } +// CdToBaseDir changes to the specified base directory during package create. +func (*PackageCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + return cdToBaseDir(createOpts, cwd) +} + +// CdToBaseDir changes to the specified base directory during package create. +func (*SkeletonCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + return cdToBaseDir(createOpts, cwd) +} + func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { if err := os.Chdir(createOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) diff --git a/src/pkg/packager/creator.go b/src/pkg/packager/creator.go new file mode 100644 index 0000000000..d10b03b5de --- /dev/null +++ b/src/pkg/packager/creator.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import "github.com/defenseunicorns/zarf/src/types" + +var ( + // verify that PackageCreator implements Creator + _ Creator = (*PackageCreator)(nil) + + // verify that SkeletonCreator implements Creator + _ Creator = (*SkeletonCreator)(nil) +) + +// Creator is an interface for creating Zarf packages. +type Creator interface { + CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error + LoadPackageDefinition(*Packager) error + Assemble(*Packager) error +} + +// NewCreator returns a new Creator based on the provided create options. +func NewCreator(createOpts *types.ZarfCreateOptions) Creator { + if createOpts.IsSkeleton { + return &SkeletonCreator{} + } + return &PackageCreator{} +} + +// PackageCreator is used to create normal (not skeleton) Zarf packages. +type PackageCreator struct{} + +// SkeletonCreator is used to create skeleton Zarf packages. +type SkeletonCreator struct{} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index fba271a1e5..77d6190151 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -24,12 +24,11 @@ func (p *Packager) DevDeploy() error { return err } - if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { - return err - } - c := NewCreator(&p.cfg.CreateOpts) + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + return err + } if err := c.LoadPackageDefinition(p); err != nil { return err } diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index 4b95cd393f..1787b605a5 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -14,43 +14,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" ) -var ( - // verify that SkeletonLoader implements Loader - _ Loader = (*SkeletonLoader)(nil) - - // verify that PackageLoader implements Loader - _ Loader = (*PackageLoader)(nil) -) - -// Loader is an interface for loading and configuring package definitions during package create. -type Loader interface { - LoadPackageDefinition(*Packager) error -} - -// SkeletonLoader is used to load and configure skeleton Zarf packages during package create. -type SkeletonLoader struct{} - -// LoadPackageDefinition loads and configures skeleton Zarf packages during package create. -func (*SkeletonLoader) LoadPackageDefinition(p *Packager) error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return err - } - - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - return p.composeComponents() -} - -// PackageLoader is used to load and configure normal (not skeleton) Zarf packages during package create. -type PackageLoader struct{} - // LoadPackageDefinition loads and configures normal (not skeleton) Zarf packages during package create. -func (*PackageLoader) LoadPackageDefinition(p *Packager) error { +func (*PackageCreator) LoadPackageDefinition(p *Packager) error { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } @@ -93,3 +58,19 @@ func (*PackageLoader) LoadPackageDefinition(p *Packager) error { return nil } + +// LoadPackageDefinition loads and configures skeleton Zarf packages during package create. +func (*SkeletonCreator) LoadPackageDefinition(p *Packager) error { + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return err + } + + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + return p.composeComponents() +} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 22d722d7c3..49f7b6f9ec 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -88,10 +88,10 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - if err := cdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + c := NewCreator(&p.cfg.CreateOpts) + if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - c := NewCreator(&p.cfg.CreateOpts) if err := c.LoadPackageDefinition(p); err != nil { return err } From df8f4a712c8b9e7cf7a2342304fba079d06b0017 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 13:57:50 -0600 Subject: [PATCH 023/172] Do not set p.arch in operations that do not need to --- src/pkg/packager/inspect.go | 2 -- src/pkg/packager/load.go | 1 - src/pkg/packager/mirror.go | 2 -- src/pkg/packager/publish.go | 1 - src/pkg/packager/remove.go | 1 - 5 files changed, 7 deletions(-) diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 525b5c0196..e666434eb3 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -5,7 +5,6 @@ package packager import ( - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -21,7 +20,6 @@ func (p *Packager) Inspect() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) utils.ColorPrintYAML(p.cfg.Pkg, nil, false) diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go index 1787b605a5..735361936a 100644 --- a/src/pkg/packager/load.go +++ b/src/pkg/packager/load.go @@ -64,7 +64,6 @@ func (*SkeletonCreator) LoadPackageDefinition(p *Packager) error { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if p.isInitConfig() { diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 524daba13d..ea210a5846 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -25,8 +25,6 @@ func (p *Packager) Mirror() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if err := p.stageSBOMViewFiles(); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 49f7b6f9ec..a04a6fe15c 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -105,7 +105,6 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) } // Get a reference to the registry for this package diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index d552943252..9f2297255e 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -42,7 +42,6 @@ func (p *Packager) Remove() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name From 8e6e5b5b3811eca7f66cce715f82d95136cbc84a Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 12 Jan 2024 14:14:16 -0600 Subject: [PATCH 024/172] p.arch needs to be set for remove --- src/pkg/packager/remove.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 9f2297255e..d552943252 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -42,6 +42,7 @@ func (p *Packager) Remove() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name From 460f0783ab77c44a4f1caeff801e1b3986e16cef Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 16 Jan 2024 11:37:12 -0600 Subject: [PATCH 025/172] Embed the Creator interface to encapsulate create logic in Packager Adds a new modifier WithCreator() that is intended to be passed in as an argument when instantiating a new Packager instance. --- src/cmd/dev.go | 2 +- src/cmd/package.go | 4 ++-- src/pkg/packager/common.go | 10 ++++++++++ src/pkg/packager/create.go | 8 +++----- src/pkg/packager/dev.go | 8 +++----- src/pkg/packager/publish.go | 7 +++---- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 63426692ea..4cb50fa960 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -50,7 +50,7 @@ var devDeployCmd = &cobra.Command{ v.GetStringMapString(common.VPkgDeploySet), pkgConfig.PkgOpts.SetVariables, strings.ToUpper) // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig) + pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) defer pkgClient.ClearTempPaths() // Create the package diff --git a/src/cmd/package.go b/src/cmd/package.go index 1ccd3e3061..a6e48ee2f7 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -54,7 +54,7 @@ var packageCreateCmd = &cobra.Command{ v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig) + pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) defer pkgClient.ClearTempPaths() // Create the package @@ -221,7 +221,7 @@ var packagePublishCmd = &cobra.Command{ pkgConfig.PublishOpts.PackageDestination = ref.String() // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig) + pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) defer pkgClient.ClearTempPaths() // Publish the package diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index e1cbfb0a20..cbf19e5054 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -46,6 +46,7 @@ type Packager struct { sbomViewFiles []string source sources.PackageSource generation int + creator Creator } // Zarf Packager Variables. @@ -83,6 +84,15 @@ func WithTemp(base string) Modifier { } } +// WithCreator sets the creator for the packager. +// +// Creator is an interface that provides methods for creating Zarf packages. +func WithCreator() Modifier { + return func(p *Packager) { + p.creator = NewCreator(&p.cfg.CreateOpts) + } +} + /* New creates a new package instance with the provided config. diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 42aeae90de..22cb916bb9 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -23,12 +23,10 @@ func (p *Packager) Create() (err error) { return err } - c := NewCreator(&p.cfg.CreateOpts) - - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := c.LoadPackageDefinition(p); err != nil { + if err := p.creator.LoadPackageDefinition(p); err != nil { return err } @@ -41,7 +39,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(p); err != nil { + if err := p.creator.Assemble(p); err != nil { return err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 77d6190151..3561fadd7d 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -24,12 +24,10 @@ func (p *Packager) DevDeploy() error { return err } - c := NewCreator(&p.cfg.CreateOpts) - - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := c.LoadPackageDefinition(p); err != nil { + if err := p.creator.LoadPackageDefinition(p); err != nil { return err } @@ -55,7 +53,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(p); err != nil { + if err := p.creator.Assemble(p); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index a04a6fe15c..3590ecde9f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -88,14 +88,13 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - c := NewCreator(&p.cfg.CreateOpts) - if err := c.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := c.LoadPackageDefinition(p); err != nil { + if err := p.creator.LoadPackageDefinition(p); err != nil { return err } - if err := c.Assemble(p); err != nil { + if err := p.creator.Assemble(p); err != nil { return err } } else { From 9becd358eee61b4e4cd60b6760fc1d06e640a3f4 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 16 Jan 2024 15:15:04 -0600 Subject: [PATCH 026/172] Undo creator changes --- src/cmd/dev.go | 2 +- src/cmd/package.go | 4 +- src/pkg/packager/assemble.go | 137 --------- src/pkg/packager/common.go | 10 - src/pkg/packager/create.go | 33 +-- src/pkg/packager/create_stages.go | 305 ++++++++++++++++++++ src/pkg/packager/creator.go | 36 --- src/pkg/packager/dev.go | 7 +- src/pkg/packager/differential.go | 148 ---------- src/pkg/packager/load.go | 75 ----- src/pkg/packager/publish.go | 8 +- src/test/e2e/08_create_differential_test.go | 9 +- 12 files changed, 324 insertions(+), 450 deletions(-) delete mode 100644 src/pkg/packager/assemble.go delete mode 100644 src/pkg/packager/creator.go delete mode 100644 src/pkg/packager/differential.go delete mode 100644 src/pkg/packager/load.go diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 4cb50fa960..63426692ea 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -50,7 +50,7 @@ var devDeployCmd = &cobra.Command{ v.GetStringMapString(common.VPkgDeploySet), pkgConfig.PkgOpts.SetVariables, strings.ToUpper) // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) + pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() // Create the package diff --git a/src/cmd/package.go b/src/cmd/package.go index a6e48ee2f7..1ccd3e3061 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -54,7 +54,7 @@ var packageCreateCmd = &cobra.Command{ v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) + pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() // Create the package @@ -221,7 +221,7 @@ var packagePublishCmd = &cobra.Command{ pkgConfig.PublishOpts.PackageDestination = ref.String() // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig, packager.WithCreator()) + pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() // Publish the package diff --git a/src/pkg/packager/assemble.go b/src/pkg/packager/assemble.go deleted file mode 100644 index 671cb0f209..0000000000 --- a/src/pkg/packager/assemble.go +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying zarf packages. -package packager - -import ( - "fmt" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" -) - -// Assemble assembles assets for normal (not skeleton) Zarf packages during package create. -func (*PackageCreator) Assemble(p *Packager) error { - componentSBOMs := map[string]*layout.ComponentSBOM{} - var imageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - if err := p.addComponent(idx, component); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if !p.cfg.CreateOpts.SkipSBOM { - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - return fmt.Errorf("unable to create component SBOM: %w", err) - } - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - imageList = append(imageList, refInfo) - } - } - - imageList = helpers.Unique(imageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - var err error - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - return nil -} - -// Assemble assembles assets for skeleton Zarf packages during package create. -func (*SkeletonCreator) Assemble(p *Packager) error { - if err := p.skeletonizeExtensions(); err != nil { - return err - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range p.cfg.Pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index cbf19e5054..e1cbfb0a20 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -46,7 +46,6 @@ type Packager struct { sbomViewFiles []string source sources.PackageSource generation int - creator Creator } // Zarf Packager Variables. @@ -84,15 +83,6 @@ func WithTemp(base string) Modifier { } } -// WithCreator sets the creator for the packager. -// -// Creator is an interface that provides methods for creating Zarf packages. -func WithCreator() Modifier { - return func(p *Packager) { - p.creator = NewCreator(&p.cfg.CreateOpts) - } -} - /* New creates a new package instance with the provided config. diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 22cb916bb9..02f13b240b 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -7,12 +7,9 @@ package packager import ( "fmt" "os" - "path/filepath" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -23,10 +20,11 @@ func (p *Packager) Create() (err error) { return err } - if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { return err } - if err := p.creator.LoadPackageDefinition(p); err != nil { + + if err := p.load(); err != nil { return err } @@ -39,7 +37,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := p.creator.Assemble(p); err != nil { + if err := p.assemble(); err != nil { return err } @@ -50,26 +48,3 @@ func (p *Packager) Create() (err error) { return p.output() } - -// CdToBaseDir changes to the specified base directory during package create. -func (*PackageCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - return cdToBaseDir(createOpts, cwd) -} - -// CdToBaseDir changes to the specified base directory during package create. -func (*SkeletonCreator) CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - return cdToBaseDir(createOpts, cwd) -} - -func cdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - if err := os.Chdir(createOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) - } - message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) - - // differentials are relative to the current working directory - if createOpts.DifferentialData.DifferentialPackagePath != "" { - createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 8f5f5f7d67..2ae7f06ad9 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -5,28 +5,214 @@ package packager import ( + "errors" "fmt" "os" "path/filepath" "slices" "strconv" "strings" + "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/images" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" "github.com/mholt/archiver/v3" ) +func (p *Packager) cdToBaseDir(base string, cwd string) error { + if err := os.Chdir(base); err != nil { + return fmt.Errorf("unable to access directory %q: %w", base, err) + } + message.Note(fmt.Sprintf("Using build directory %s", base)) + + // differentials are relative to the current working directory + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} + +func (p *Packager) load() error { + if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + if err := p.composeComponents(); err != nil { + return err + } + + if p.cfg.CreateOpts.IsSkeleton { + return nil + } + + // After components are composed, template the active package. + if err := p.fillActiveTemplate(); err != nil { + return fmt.Errorf("unable to fill values in template: %s", err.Error()) + } + + // After templates are filled process any create extensions + if err := p.processExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + // Load the images and repos from the 'reference' package + if err := p.loadDifferentialData(); err != nil { + return err + } + // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building + // If the package versions are the same return an error + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { + return errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { + return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + } + + // Handle any potential differential images/repos before going forward + if err := p.removeCopiesFromDifferentialPackage(); err != nil { + return err + } + } + + return nil +} + +func (p *Packager) assemble() error { + componentSBOMs := map[string]*layout.ComponentSBOM{} + var imageList []transform.Image + for idx, component := range p.cfg.Pkg.Components { + onCreate := component.Actions.OnCreate + onFailure := func() { + if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + if err := p.addComponent(idx, component); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !p.cfg.CreateOpts.SkipSBOM { + componentSBOM, err := p.getFilesToSBOM(component) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + p.layout = p.layout.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: p.layout.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if p.cfg.CreateOpts.SkipSBOM { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + p.layout = p.layout.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} + +func (p *Packager) assembleSkeleton() error { + if err := p.skeletonizeExtensions(); err != nil { + return err + } + for _, warning := range p.warnings { + message.Warn(warning) + } + for idx, component := range p.cfg.Pkg.Components { + if err := p.addComponent(idx, component); err != nil { + return err + } + + if err := p.layout.Components.Archive(component, false); err != nil { + return err + } + } + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return p.writeYaml() +} + // output assumes it is running from cwd, not the build directory func (p *Packager) output() error { // Process the component directories into compressed tarballs @@ -455,3 +641,122 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Calculate the checksum of the checksum file return utils.GetSHA256OfFile(checksumsFilePath) } + +// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package +func (p *Packager) loadDifferentialData() error { + // Save the fact that this is a differential build into the build data of the package + p.cfg.Pkg.Build.Differential = true + + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { + remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + pkg, err := remote.FetchZarfYAML() + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return err + } + } else { + if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return nil +} + +// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package +func (p *Packager) removeCopiesFromDifferentialPackage() error { + // If a differential build was not requested, continue on as normal + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { + return nil + } + + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range p.cfg.Pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + p.cfg.Pkg.Components[idx] = component + } + + return nil +} diff --git a/src/pkg/packager/creator.go b/src/pkg/packager/creator.go deleted file mode 100644 index d10b03b5de..0000000000 --- a/src/pkg/packager/creator.go +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import "github.com/defenseunicorns/zarf/src/types" - -var ( - // verify that PackageCreator implements Creator - _ Creator = (*PackageCreator)(nil) - - // verify that SkeletonCreator implements Creator - _ Creator = (*SkeletonCreator)(nil) -) - -// Creator is an interface for creating Zarf packages. -type Creator interface { - CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error - LoadPackageDefinition(*Packager) error - Assemble(*Packager) error -} - -// NewCreator returns a new Creator based on the provided create options. -func NewCreator(createOpts *types.ZarfCreateOptions) Creator { - if createOpts.IsSkeleton { - return &SkeletonCreator{} - } - return &PackageCreator{} -} - -// PackageCreator is used to create normal (not skeleton) Zarf packages. -type PackageCreator struct{} - -// SkeletonCreator is used to create skeleton Zarf packages. -type SkeletonCreator struct{} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 3561fadd7d..b4e1744d46 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -24,10 +24,11 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { return err } - if err := p.creator.LoadPackageDefinition(p); err != nil { + + if err := p.load(); err != nil { return err } @@ -53,7 +54,7 @@ func (p *Packager) DevDeploy() error { } } - if err := p.creator.Assemble(p); err != nil { + if err := p.assemble(); err != nil { return err } diff --git a/src/pkg/packager/differential.go b/src/pkg/packager/differential.go deleted file mode 100644 index 36e6361c8d..0000000000 --- a/src/pkg/packager/differential.go +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying zarf packages. -package packager - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" -) - -// loadDifferentialData extracts the Zarf config of a designated 'reference' package used for building a differential package. -// -// It creates a list of all images and repositories that are in the reference package. -func (p *Packager) loadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage removes any images and repositories that are already in the reference package from the new package. -// -// For each component in the new package, it checks if any images or repositories are duplicates from the reference package. -// -// Duplicate images and repositories are excluded, and the component lists are updated accordingly. -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil -} diff --git a/src/pkg/packager/load.go b/src/pkg/packager/load.go deleted file mode 100644 index 735361936a..0000000000 --- a/src/pkg/packager/load.go +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "errors" - "fmt" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/utils" -) - -// LoadPackageDefinition loads and configures normal (not skeleton) Zarf packages during package create. -func (*PackageCreator) LoadPackageDefinition(p *Packager) error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { - return err - } - - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { - return err - } - - // If we are building a differential package, remove duplicate repos and images. - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - if err := p.loadDifferentialData(); err != nil { - return err - } - versionsMatch := p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version - if versionsMatch { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - noVersionSet := p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" - if noVersionSet { - return errors.New(lang.PkgCreateErrDifferentialNoVersion) - } - return p.removeCopiesFromDifferentialPackage() - } - - return nil -} - -// LoadPackageDefinition loads and configures skeleton Zarf packages during package create. -func (*SkeletonCreator) LoadPackageDefinition(p *Packager) error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return err - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - return p.composeComponents() -} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 3590ecde9f..54c16abed6 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -88,17 +88,17 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - if err := p.creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { return err } - if err := p.creator.LoadPackageDefinition(p); err != nil { + if err := p.load(); err != nil { return err } - if err := p.creator.Assemble(p); err != nil { + if err := p.assembleSkeleton(); err != nil { return err } } else { - if err = p.source.LoadPackage(p.layout, false); err != nil { + if err := p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { diff --git a/src/test/e2e/08_create_differential_test.go b/src/test/e2e/08_create_differential_test.go index 1121f3514b..5d7db9fedc 100644 --- a/src/test/e2e/08_create_differential_test.go +++ b/src/test/e2e/08_create_differential_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "testing" - "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -17,15 +16,15 @@ import ( "github.com/stretchr/testify/require" ) -// TestCreateDifferential creates several differential packages and ensures the reference package images and repos are not included in the new package. +// TestCreateDifferential creates several differential packages and ensures the already built images and repos and not included in the new package func TestCreateDifferential(t *testing.T) { t.Log("E2E: Test Differential Package Behavior") tmpdir := t.TempDir() cachePath := filepath.Join(tmpdir, ".cache-location") packagePath := "src/test/packages/08-differential-package" - packageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0.tar.zst", e2e.Arch) - differentialPackageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0-differential-v0.26.0.tar.zst", e2e.Arch) + packageName := "zarf-package-differential-package-amd64-v0.25.0.tar.zst" + differentialPackageName := "zarf-package-differential-package-amd64-v0.25.0-differential-v0.26.0.tar.zst" differentialFlag := fmt.Sprintf("--differential=%s", packageName) // Build the package a first time @@ -36,7 +35,7 @@ func TestCreateDifferential(t *testing.T) { // Build the differential package without changing the version _, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.25.0", differentialFlag, "--confirm") require.Error(t, err, "zarf package create should have errored when a differential package was being created without updating the package version number") - require.Contains(t, e2e.StripMessageFormatting(stdErr), lang.PkgCreateErrDifferentialSameVersion) + require.Contains(t, stdErr, "unable to create a differential package with the same version") // Build the differential package stdOut, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.26.0", differentialFlag, "--confirm") From bf6d4b0ab08f4d5fa6d5d60540cc6fcb7106221f Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 16 Jan 2024 16:17:35 -0600 Subject: [PATCH 027/172] Get back to clean working slate --- src/pkg/packager/create_stages.go | 4 +++- src/test/e2e/08_create_differential_test.go | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 2ae7f06ad9..5c35c65e5a 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -46,9 +46,11 @@ func (p *Packager) cdToBaseDir(base string, cwd string) error { } func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + if p.isInitConfig() { p.cfg.Pkg.Metadata.Version = config.CLIVersion } diff --git a/src/test/e2e/08_create_differential_test.go b/src/test/e2e/08_create_differential_test.go index 5d7db9fedc..1121f3514b 100644 --- a/src/test/e2e/08_create_differential_test.go +++ b/src/test/e2e/08_create_differential_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -16,15 +17,15 @@ import ( "github.com/stretchr/testify/require" ) -// TestCreateDifferential creates several differential packages and ensures the already built images and repos and not included in the new package +// TestCreateDifferential creates several differential packages and ensures the reference package images and repos are not included in the new package. func TestCreateDifferential(t *testing.T) { t.Log("E2E: Test Differential Package Behavior") tmpdir := t.TempDir() cachePath := filepath.Join(tmpdir, ".cache-location") packagePath := "src/test/packages/08-differential-package" - packageName := "zarf-package-differential-package-amd64-v0.25.0.tar.zst" - differentialPackageName := "zarf-package-differential-package-amd64-v0.25.0-differential-v0.26.0.tar.zst" + packageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0.tar.zst", e2e.Arch) + differentialPackageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0-differential-v0.26.0.tar.zst", e2e.Arch) differentialFlag := fmt.Sprintf("--differential=%s", packageName) // Build the package a first time @@ -35,7 +36,7 @@ func TestCreateDifferential(t *testing.T) { // Build the differential package without changing the version _, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.25.0", differentialFlag, "--confirm") require.Error(t, err, "zarf package create should have errored when a differential package was being created without updating the package version number") - require.Contains(t, stdErr, "unable to create a differential package with the same version") + require.Contains(t, e2e.StripMessageFormatting(stdErr), lang.PkgCreateErrDifferentialSameVersion) // Build the differential package stdOut, stdErr, err = e2e.Zarf("package", "create", packagePath, "--zarf-cache", cachePath, "--set=PACKAGE_VERSION=v0.26.0", differentialFlag, "--confirm") From f1fa5c7a7dec186c9328263956bf7c132602f572 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 16 Jan 2024 16:47:12 -0600 Subject: [PATCH 028/172] Move p.cdToBaseDir to new creator package --- src/pkg/packager/create.go | 3 ++- src/pkg/packager/create_stages.go | 13 ------------- src/pkg/packager/creator/cd.go | 28 ++++++++++++++++++++++++++++ src/pkg/packager/dev.go | 3 ++- src/pkg/packager/publish.go | 3 ++- 5 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 src/pkg/packager/creator/cd.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 02f13b240b..f26e9900ad 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -20,7 +21,7 @@ func (p *Packager) Create() (err error) { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 5c35c65e5a..911f04a2ef 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -32,19 +32,6 @@ import ( "github.com/mholt/archiver/v3" ) -func (p *Packager) cdToBaseDir(base string, cwd string) error { - if err := os.Chdir(base); err != nil { - return fmt.Errorf("unable to access directory %q: %w", base, err) - } - message.Note(fmt.Sprintf("Using build directory %s", base)) - - // differentials are relative to the current working directory - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} - func (p *Packager) load() error { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) diff --git a/src/pkg/packager/creator/cd.go b/src/pkg/packager/creator/cd.go new file mode 100644 index 0000000000..a9d3c9c673 --- /dev/null +++ b/src/pkg/packager/creator/cd.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// CdToBaseDir changes into the specified base directory. +func CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { + if err := os.Chdir(createOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) + } + message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) + + // differentials are relative to the current working directory + if createOpts.DifferentialData.DifferentialPackagePath != "" { + createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b4e1744d46..d87a729d6c 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/types" ) @@ -24,7 +25,7 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 54c16abed6..39c0daf5f9 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -88,7 +89,7 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } if err := p.load(); err != nil { From 0a19f3acda821cd2535aae2010bf378e6a6603da Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 16 Jan 2024 21:37:26 -0600 Subject: [PATCH 029/172] Create new creator package and begin moving create logic to it --- go.mod | 2 + go.sum | 4 + src/pkg/packager/common.go | 7 +- src/pkg/packager/create.go | 7 +- src/pkg/packager/create_stages.go | 182 +------------ src/pkg/packager/{ => creator}/compose.go | 34 ++- src/pkg/packager/creator/{cd.go => io.go} | 2 +- src/pkg/packager/creator/new.go | 46 ++++ src/pkg/packager/creator/normal.go | 311 ++++++++++++++++++++++ src/pkg/packager/creator/skeleton.go | 167 ++++++++++++ src/pkg/packager/creator/template.go | 89 +++++++ src/pkg/packager/deploy.go | 6 +- src/pkg/packager/dev.go | 7 +- src/pkg/packager/extensions.go | 68 ----- src/pkg/packager/lint/lint.go | 5 +- src/pkg/packager/prepare.go | 14 +- src/pkg/packager/publish.go | 6 +- src/pkg/packager/variables.go | 84 ------ src/pkg/packager/yaml.go | 48 ---- src/pkg/utils/package.go | 12 + src/pkg/utils/yaml.go | 8 +- 21 files changed, 693 insertions(+), 416 deletions(-) rename src/pkg/packager/{ => creator}/compose.go (54%) rename src/pkg/packager/creator/{cd.go => io.go} (92%) create mode 100644 src/pkg/packager/creator/new.go create mode 100644 src/pkg/packager/creator/normal.go create mode 100644 src/pkg/packager/creator/skeleton.go create mode 100644 src/pkg/packager/creator/template.go delete mode 100644 src/pkg/packager/extensions.go create mode 100644 src/pkg/utils/package.go diff --git a/go.mod b/go.mod index 50fd25a90a..a42f414667 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/gofrs/flock v0.8.1 github.com/google/go-containerregistry v0.17.0 github.com/gosuri/uitable v0.0.4 + github.com/mholt/archiver v3.1.1+incompatible github.com/mholt/archiver/v3 v3.5.1 github.com/moby/moby v24.0.7+incompatible github.com/opencontainers/image-spec v1.1.0-rc5 @@ -378,6 +379,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect diff --git a/go.sum b/go.sum index 376522f5d2..b77ea206f7 100644 --- a/go.sum +++ b/go.sum @@ -1257,6 +1257,8 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQth github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= +github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 h1:tQRHcLQwnwrPq2j2Qra/NnyjyESBGwdeBeVdAE9kXYg= @@ -1410,6 +1412,8 @@ github.com/petergtz/pegomock v2.9.0+incompatible h1:BKfb5XfkJfehe5T+O1xD4Zm26Sb9 github.com/petergtz/pegomock v2.9.0+incompatible/go.mod h1:nuBLWZpVyv/fLo56qTwt/AUau7jgouO1h7bEvZCq82o= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index e1cbfb0a20..b6a9392e67 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -173,7 +173,7 @@ func GetInitPackageName(arch string) string { // GetPackageName returns the formatted name of the package. func (p *Packager) GetPackageName() string { - if p.isInitConfig() { + if utils.IsInitConfig(p.cfg.Pkg) { return GetInitPackageName(p.arch) } @@ -219,11 +219,6 @@ func (p *Packager) isConnectedToCluster() bool { return p.cluster != nil } -// isInitConfig returns whether the current packager instance is deploying an init config -func (p *Packager) isInitConfig() bool { - return p.cfg.Pkg.Kind == types.ZarfInitConfig -} - // hasImages returns whether the current package contains images func (p *Packager) hasImages() bool { for _, component := range p.cfg.Pkg.Components { diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index f26e9900ad..183f3742cd 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -25,7 +25,12 @@ func (p *Packager) Create() (err error) { return err } - if err := p.load(); err != nil { + c, err := creator.New(&p.cfg.CreateOpts) + if err != nil { + return err + } + + if err := c.LoadPackageDefinition(); err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 911f04a2ef..e76e4cb921 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -5,7 +5,6 @@ package packager import ( - "errors" "fmt" "os" "path/filepath" @@ -24,67 +23,14 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" "github.com/mholt/archiver/v3" ) -func (p *Packager) load() error { - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { - return err - } - - if p.cfg.CreateOpts.IsSkeleton { - return nil - } - - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { - return err - } - - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - - return nil -} - func (p *Packager) assemble() error { componentSBOMs := map[string]*layout.ComponentSBOM{} var imageList []transform.Image @@ -178,7 +124,12 @@ func (p *Packager) assemble() error { } func (p *Packager) assembleSkeleton() error { - if err := p.skeletonizeExtensions(); err != nil { + c, err := creator.New(&p.cfg.CreateOpts) + if err != nil { + return err + } + + if err := c.ProcessExtensions(); err != nil { return err } for _, warning := range p.warnings { @@ -630,122 +581,3 @@ func (p *Packager) generatePackageChecksums() (string, error) { // Calculate the checksum of the checksum file return utils.GetSHA256OfFile(checksumsFilePath) } - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil -} diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/creator/compose.go similarity index 54% rename from src/pkg/packager/compose.go rename to src/pkg/packager/creator/compose.go index 6bbbf22fff..b04be2117d 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package creator contains functions for creating Zarf packages. +package creator import ( "github.com/defenseunicorns/zarf/src/pkg/message" @@ -10,17 +10,16 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// composeComponents builds the composed components list for the current config. -func (p *Packager) composeComponents() error { +func composeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { components := []types.ZarfComponent{} - pkgVars := p.cfg.Pkg.Variables - pkgConsts := p.cfg.Pkg.Constants + pkgVars := pkg.Variables + pkgConsts := pkg.Constants - for i, component := range p.cfg.Pkg.Components { - arch := p.arch + for i, component := range pkg.Components { + arch := pkg.Build.Architecture // filter by architecture - if !composer.CompatibleComponent(component, arch, p.cfg.CreateOpts.Flavor) { + if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { continue } @@ -29,20 +28,19 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, i, p.cfg.Pkg.Metadata.Name, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, createOpts.Flavor) if err != nil { - return err + return nil, err } message.Debugf("%s", chain) // migrate any deprecated component configurations now - warnings := chain.Migrate(p.cfg.Pkg.Build) - p.warnings = append(p.warnings, warnings...) + warnings = chain.Migrate(pkg.Build) // get the composed component composed, err := chain.Compose() if err != nil { - return err + return nil, err } components = append(components, *composed) @@ -52,10 +50,10 @@ func (p *Packager) composeComponents() error { } // set the filtered + composed components - p.cfg.Pkg.Components = components + pkg.Components = components - p.cfg.Pkg.Variables = pkgVars - p.cfg.Pkg.Constants = pkgConsts + pkg.Variables = pkgVars + pkg.Constants = pkgConsts - return nil + return warnings, nil } diff --git a/src/pkg/packager/creator/cd.go b/src/pkg/packager/creator/io.go similarity index 92% rename from src/pkg/packager/creator/cd.go rename to src/pkg/packager/creator/io.go index a9d3c9c673..0ed6bebc0f 100644 --- a/src/pkg/packager/creator/cd.go +++ b/src/pkg/packager/creator/io.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for creating Zarf packages. +// Package creator contains functions for creating Zarf packages. package creator import ( diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go new file mode 100644 index 0000000000..6381b77099 --- /dev/null +++ b/src/pkg/packager/creator/new.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/types" +) + +// Creator is an interface for creating Zarf packages. +type Creator interface { + LoadPackageDefinition() error + ComposeComponents() (warnings []string, err error) + FillActiveTemplate() (warnings []string, err error) + ProcessExtensions() error + LoadDifferentialData() error + RemoveCopiesFromDifferentialPackage() error +} + +// New returns a new Creator based on the provided create options. +func New(createOpts *types.ZarfCreateOptions) (Creator, error) { + sc := &SkeletonCreator{} + pc := &PackageCreator{} + + if createOpts.IsSkeleton { + // If the temp directory is not set, set it to the default + if sc.layout == nil { + if err := sc.setTempDirectory(config.CommonOptions.TempDirectory); err != nil { + return nil, fmt.Errorf("unable to create package temp paths: %w", err) + } + } + return sc, nil + } + + // If the temp directory is not set, set it to the default + if pc.layout == nil { + if err := pc.setTempDirectory(config.CommonOptions.TempDirectory); err != nil { + return nil, fmt.Errorf("unable to create package temp paths: %w", err) + } + } + return pc, nil +} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go new file mode 100644 index 0000000000..24bff61301 --- /dev/null +++ b/src/pkg/packager/creator/normal.go @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mholt/archiver" +) + +var ( + // veryify that PackageCreator implements Creator + _ Creator = (*PackageCreator)(nil) +) + +// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. +type PackageCreator struct { + pkg types.ZarfPackage + createOpts types.ZarfCreateOptions + layout *layout.PackagePaths + arch string + warnings []string +} + +// LoadPackageDefinition loads and configure a zarf.yaml file during package create. +func (pc *PackageCreator) LoadPackageDefinition() (err error) { + if err := utils.ReadYaml(layout.ZarfYAML, pc.pkg); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + pc.arch = config.GetArch() + + if utils.IsInitConfig(pc.pkg) { + pc.pkg.Metadata.Version = config.CLIVersion + } + + if err := pc.setPackageBuildMetadata(); err != nil { + message.Warn(err.Error()) + } + + // Compose components into a single zarf.yaml file + if pc.warnings, err = pc.ComposeComponents(); err != nil { + return err + } + + // After components are composed, template the active package. + if pc.warnings, err = pc.FillActiveTemplate(); err != nil { + return fmt.Errorf("unable to fill values in template: %s", err.Error()) + } + + // After templates are filled process any create extensions + if err := pc.ProcessExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + if pc.createOpts.DifferentialData.DifferentialPackagePath != "" { + // Load the images and repos from the 'reference' package + if err := pc.LoadDifferentialData(); err != nil { + return err + } + // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building + // If the package versions are the same return an error + if pc.createOpts.DifferentialData.DifferentialPackageVersion == pc.pkg.Metadata.Version { + return errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pc.pkg.Metadata.Version == "" { + return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + } + + // Handle any potential differential images/repos before going forward + if err := pc.RemoveCopiesFromDifferentialPackage(); err != nil { + return err + } + } + + return nil +} + +// ComposeComponents builds the composed components list for the current config. +func (pc *PackageCreator) ComposeComponents() (warnings []string, err error) { + return composeComponents(&pc.pkg, pc.createOpts) +} + +// FillActiveTemplate handles setting the active variables and reloading the base template. +func (pc *PackageCreator) FillActiveTemplate() (warnings []string, err error) { + return fillActiveTemplate(&pc.pkg, pc.createOpts) +} + +func (pc *PackageCreator) ProcessExtensions() error { + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range pc.pkg.Components { + componentPaths, err := pc.layout.Components.Create(c) + if err != nil { + return err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Run(pc.pkg.Metadata.YOLO, componentPaths, c); err != nil { + return fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + pc.pkg.Components = components + + return nil +} + +// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, +// and creates a list of all images and repos that are in the reference package. +func (pc *PackageCreator) LoadDifferentialData() error { + // Save the fact that this is a differential build into the build data of the package + pc.pkg.Build.Differential = true + + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(pc.createOpts.DifferentialData.DifferentialPackagePath) { + remote, err := oci.NewOrasRemote(pc.createOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + pkg, err := remote.FetchZarfYAML() + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return err + } + } else { + if err := archiver.Extract(pc.createOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + pc.createOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + pc.createOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + pc.createOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return nil +} + +// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. +func (pc *PackageCreator) RemoveCopiesFromDifferentialPackage() error { + // If a differential build was not requested, continue on as normal + if pc.createOpts.DifferentialData.DifferentialPackagePath == "" { + return nil + } + + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range pc.pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !pc.createOpts.DifferentialData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !pc.createOpts.DifferentialData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + pc.pkg.Components[idx] = component + } + + return nil +} + +// setTempDirectory sets the temp directory for the PackageCreator. +func (pc *PackageCreator) setTempDirectory(path string) error { + dir, err := utils.MakeTempDir(path) + if err != nil { + return fmt.Errorf("unable to create package temp paths: %w", err) + } + + pc.layout = layout.New(dir) + return nil +} + +// setPackageBuildMetadata sets various package build metadata. +func (pc *PackageCreator) setPackageBuildMetadata() error { + now := time.Now() + // Just use $USER env variable to avoid CGO issue. + // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. + // Record the name of the user creating the package. + if runtime.GOOS == "windows" { + pc.pkg.Build.User = os.Getenv("USERNAME") + } else { + pc.pkg.Build.User = os.Getenv("USER") + } + + hostname, err := os.Hostname() + if err != nil { + return err + } + + pc.pkg.Metadata.Architecture = pc.arch + pc.pkg.Build.Architecture = pc.arch + + // Record the time of package creation. + pc.pkg.Build.Timestamp = now.Format(time.RFC1123Z) + + // Record the Zarf Version the CLI was built with. + pc.pkg.Build.Version = config.CLIVersion + + // Record the hostname of the package creation terminal. + pc.pkg.Build.Terminal = hostname + + // Record the migrations that will be run on the package. + pc.pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Record the flavor of Zarf used to build this package (if any). + pc.pkg.Build.Flavor = pc.createOpts.Flavor + + pc.pkg.Build.RegistryOverrides = pc.createOpts.RegistryOverrides + + // Record the latest version of Zarf without breaking changes to the package structure. + pc.pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + + return nil +} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go new file mode 100644 index 0000000000..d5e3d3a7fd --- /dev/null +++ b/src/pkg/packager/creator/skeleton.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "os" + "runtime" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" +) + +var ( + // veryify that SkeletonCreator implements Creator + _ Creator = (*SkeletonCreator)(nil) +) + +// SkeletonCreator provides methods for creating skeleton Zarf packages. +type SkeletonCreator struct { + pkg *types.ZarfPackage + createOpts types.ZarfCreateOptions + layout *layout.PackagePaths + arch string + warnings []string +} + +// LoadPackageDefinition loads and configure a zarf.yaml file during package create. +func (sc *SkeletonCreator) LoadPackageDefinition() (err error) { + if err := utils.ReadYaml(layout.ZarfYAML, &sc.pkg); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + sc.arch = config.GetArch(sc.pkg.Metadata.Architecture, sc.pkg.Build.Architecture) + + if utils.IsInitConfig(*sc.pkg) { + sc.pkg.Metadata.Version = config.CLIVersion + } + + if err := sc.setPackageBuildMetadata(); err != nil { + message.Warn(err.Error()) + } + + // Compose components into a single zarf.yaml file + sc.warnings, err = sc.ComposeComponents() + if err != nil { + return err + } + + return nil +} + +// ComposeComponents builds the composed components list for the current config. +func (sc *SkeletonCreator) ComposeComponents() (warnings []string, err error) { + return composeComponents(sc.pkg, sc.createOpts) +} + +// FillActiveTemplate handles setting the active variables and reloading the base template. +func (sc *SkeletonCreator) FillActiveTemplate() (warnings []string, err error) { + return nil, fmt.Errorf("not implemented") +} + +func (sc *SkeletonCreator) ProcessExtensions() error { + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range sc.pkg.Components { + componentPaths, err := sc.layout.Components.Create(c) + if err != nil { + return err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { + return fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + sc.pkg.Components = components + + return nil +} + +// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, +// and creates a list of all images and repos that are in the reference package. +// +// This is not implemented. +func (sc *SkeletonCreator) LoadDifferentialData() error { + return fmt.Errorf("not implemented") +} + +// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. +// +// This is not implemented. +func (sc *SkeletonCreator) RemoveCopiesFromDifferentialPackage() error { + return fmt.Errorf("not implemented") +} + +// setTempDirectory sets the temp directory for the SkeletonCreator. +func (sc *SkeletonCreator) setTempDirectory(path string) error { + dir, err := utils.MakeTempDir(path) + if err != nil { + return fmt.Errorf("unable to create package temp paths: %w", err) + } + + sc.layout = layout.New(dir) + return nil +} + +// setPackageBuildMetadata sets various package build metadata. +func (sc *SkeletonCreator) setPackageBuildMetadata() error { + now := time.Now() + // Just use $USER env variable to avoid CGO issue. + // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. + // Record the name of the user creating the package. + if runtime.GOOS == "windows" { + sc.pkg.Build.User = os.Getenv("USERNAME") + } else { + sc.pkg.Build.User = os.Getenv("USER") + } + + hostname, err := os.Hostname() + if err != nil { + return err + } + + sc.pkg.Metadata.Architecture = "skeleton" + sc.pkg.Build.Architecture = "skeleton" + + // Record the time of package creation. + sc.pkg.Build.Timestamp = now.Format(time.RFC1123Z) + + // Record the Zarf Version the CLI was built with. + sc.pkg.Build.Version = config.CLIVersion + + // Record the hostname of the package creation terminal. + sc.pkg.Build.Terminal = hostname + + // Record the migrations that will be run on the package. + sc.pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Record the flavor of Zarf used to build this package (if any). + sc.pkg.Build.Flavor = sc.createOpts.Flavor + + sc.pkg.Build.RegistryOverrides = sc.createOpts.RegistryOverrides + + // Record the latest version of Zarf without breaking changes to the package structure. + sc.pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + + return nil +} diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go new file mode 100644 index 0000000000..6cae02af85 --- /dev/null +++ b/src/pkg/packager/creator/template.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" +) + +func fillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { + templateMap := map[string]string{} + + promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { + yamlTemplates, err := utils.FindYamlTemplates(pkg, templatePrefix, "###") + if err != nil { + return err + } + + for key := range yamlTemplates { + if deprecated { + warnings = append(warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + } + + _, present := createOpts.SetVariables[key] + if !present && !config.CommonOptions.Confirm { + setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ + Name: key, + }) + + if err == nil { + createOpts.SetVariables[key] = setVal + } else { + return err + } + } else if !present { + return fmt.Errorf("template '%s' must be '--set' when using the '--confirm' flag", key) + } + } + + for key, value := range createOpts.SetVariables { + templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value + } + + return nil + } + + // update the component templates on the package + if err := reloadComponentTemplatesInPackage(pkg); err != nil { + return nil, err + } + + if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { + return nil, err + } + // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility + if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { + return nil, err + } + + // Add special variable for the current package architecture + templateMap[types.ZarfPackageArch] = pkg.Build.Architecture + + if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { + return nil, err + } + + return warnings, nil +} + +// reloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func reloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for _, component := range zarfPackage.Components { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + + if err := utils.ReloadYamlTemplate(&component, mappings); err != nil { + return err + } + } + return nil +} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 4158159a28..2a0fc43958 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -120,7 +120,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // If this component requires a cluster, connect to one if requiresCluster(component) { timeout := cluster.DefaultTimeout - if p.isInitConfig() { + if utils.IsInitConfig(p.cfg.Pkg) { timeout = 5 * time.Minute } @@ -150,7 +150,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // Deploy the component var charts []types.InstalledChart var deployErr error - if p.isInitConfig() { + if utils.IsInitConfig(p.cfg.Pkg) { charts, deployErr = p.deployInitComponent(component) } else { charts, deployErr = p.deployComponent(component, false /* keep img checksum */, false /* always push images */) @@ -620,7 +620,7 @@ func (p *Packager) installChartAndManifests(componentPaths *layout.ComponentPath func (p *Packager) printTablesForDeployment(componentsToDeploy []types.DeployedComponent) { // If not init config, print the application connection table - if !p.isInitConfig() { + if !utils.IsInitConfig(p.cfg.Pkg) { message.PrintConnectStringTable(p.connectStrings) } else { if p.cluster != nil { diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index d87a729d6c..c6cefdd6f1 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -29,7 +29,12 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.load(); err != nil { + c, err := creator.New(&p.cfg.CreateOpts) + if err != nil { + return err + } + + if err := c.LoadPackageDefinition(); err != nil { return err } diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go deleted file mode 100644 index b5cc0ab17d..0000000000 --- a/src/pkg/packager/extensions.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/types" -) - -// Check for any extensions in use and runs the appropriate functions. -func (p *Packager) processExtensions() (err error) { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range p.cfg.Pkg.Components { - componentPaths, err := p.layout.Components.Create(c) - if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Run(p.cfg.Pkg.Metadata.YOLO, componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - p.cfg.Pkg.Components = components - - return nil -} - -// Check for any extensions in use and skeletonize their local files. -func (p *Packager) skeletonizeExtensions() (err error) { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range p.cfg.Pkg.Components { - componentPaths, err := p.layout.Components.Create(c) - if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - p.cfg.Pkg.Components = components - - return nil -} diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index bc8b5df336..08c4964cd4 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -104,8 +103,10 @@ func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { } func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { + componentMap := map[string]string{} + componentMap[types.ZarfComponentName] = node.ZarfComponent.Name - err := packager.ReloadComponentTemplate(&node.ZarfComponent) + err := utils.ReloadYamlTemplate(&node.ZarfComponent, componentMap) if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index c3fb5e7543..17837a4390 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -52,19 +53,24 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if err := p.composeComponents(); err != nil { + c, err := creator.New(&p.cfg.CreateOpts) + if err != nil { return nil, err } - for _, warning := range p.warnings { - message.Warn(warning) + if p.warnings, err = c.ComposeComponents(); err != nil { + return nil, err } // After components are composed, template the active package - if err := p.fillActiveTemplate(); err != nil { + if p.warnings, err = c.FillActiveTemplate(); err != nil { return nil, fmt.Errorf("unable to fill values in template: %s", err.Error()) } + for _, warning := range p.warnings { + message.Warn(warning) + } + for _, component := range p.cfg.Pkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 39c0daf5f9..9465b564b2 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -92,7 +92,11 @@ func (p *Packager) Publish() (err error) { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := p.load(); err != nil { + c, err := creator.New(&p.cfg.CreateOpts) + if err != nil { + return err + } + if err := c.LoadPackageDefinition(); err != nil { return err } if err := p.assembleSkeleton(); err != nil { diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index fb3e20f6d3..80f1b4c3f1 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -9,94 +9,10 @@ import ( "regexp" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) -// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads -// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name -func ReloadComponentTemplate(component *types.ZarfComponent) error { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(component, mappings) - if err != nil { - return err - } - return nil -} - -// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i := range zarfPackage.Components { - if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { - return err - } - } - - return nil -} - -// fillActiveTemplate handles setting the active variables and reloading the base template. -func (p *Packager) fillActiveTemplate() error { - templateMap := map[string]string{} - - promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(&p.cfg.Pkg, templatePrefix, "###") - if err != nil { - return err - } - - for key := range yamlTemplates { - if deprecated { - p.warnings = append(p.warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) - } - - _, present := p.cfg.CreateOpts.SetVariables[key] - if !present && !config.CommonOptions.Confirm { - setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ - Name: key, - }) - - if err == nil { - p.cfg.CreateOpts.SetVariables[key] = setVal - } else { - return err - } - } else if !present { - return fmt.Errorf("template '%s' must be '--set' when using the '--confirm' flag", key) - } - } - - for key, value := range p.cfg.CreateOpts.SetVariables { - templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value - } - - return nil - } - - // update the component templates on the package - err := ReloadComponentTemplatesInPackage(&p.cfg.Pkg) - if err != nil { - return err - } - - if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return err - } - // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return err - } - - // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = p.arch - - return utils.ReloadYamlTemplate(&p.cfg.Pkg, templateMap) -} - // setVariableMapInConfig handles setting the active variables used to template component files. func (p *Packager) setVariableMapInConfig() error { for name, value := range p.cfg.PkgOpts.SetVariables { diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 5a1b4fd462..656253813e 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -5,13 +5,9 @@ package packager import ( - "os" "runtime" - "time" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -49,49 +45,5 @@ func (p *Packager) filterComponents() { // writeYaml adds build information and writes the config to the temp directory. func (p *Packager) writeYaml() error { - now := time.Now() - // Just use $USER env variable to avoid CGO issue. - // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. - // Record the name of the user creating the package. - if runtime.GOOS == "windows" { - p.cfg.Pkg.Build.User = os.Getenv("USERNAME") - } else { - p.cfg.Pkg.Build.User = os.Getenv("USER") - } - hostname, hostErr := os.Hostname() - - // Normalize these for the package confirmation. - p.cfg.Pkg.Metadata.Architecture = p.arch - p.cfg.Pkg.Build.Architecture = p.arch - - if p.cfg.CreateOpts.IsSkeleton { - p.cfg.Pkg.Build.Architecture = "skeleton" - } - - // Record the time of package creation. - p.cfg.Pkg.Build.Timestamp = now.Format(time.RFC1123Z) - - // Record the Zarf Version the CLI was built with. - p.cfg.Pkg.Build.Version = config.CLIVersion - - if hostErr == nil { - // Record the hostname of the package creation terminal. - p.cfg.Pkg.Build.Terminal = hostname - } - - // Record the migrations that will be run on the package. - p.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Record the flavor of Zarf used to build this package (if any). - p.cfg.Pkg.Build.Flavor = p.cfg.CreateOpts.Flavor - - p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides - - // Record the latest version of Zarf without breaking changes to the package structure. - p.cfg.Pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400) } diff --git a/src/pkg/utils/package.go b/src/pkg/utils/package.go new file mode 100644 index 0000000000..b6e3583985 --- /dev/null +++ b/src/pkg/utils/package.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic utility functions. +package utils + +import "github.com/defenseunicorns/zarf/src/types" + +// IsInitConfig returns whether the provided Zarf package is an init config. +func IsInitConfig(pkg types.ZarfPackage) bool { + return pkg.Kind == types.ZarfInitConfig +} diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index fcf81c497c..9b95c43969 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -123,7 +123,7 @@ func AddRootHint(hints map[string]string, rootKey string, hintText string) map[s } // ReadYaml reads a yaml file and unmarshals it into a given config. -func ReadYaml(path string, destConfig any) error { +func ReadYaml(path string, destConfig interface{}) error { message.Debugf("Reading YAML at %s", path) file, err := os.ReadFile(path) @@ -131,7 +131,7 @@ func ReadYaml(path string, destConfig any) error { return err } - return goyaml.Unmarshal(file, destConfig) + return goyaml.Unmarshal(file, &destConfig) } // WriteYaml writes a given config to a yaml file on disk. @@ -146,7 +146,7 @@ func WriteYaml(path string, srcConfig any, perm fs.FileMode) error { } // ReloadYamlTemplate marshals a given config, replaces strings and unmarshals it back. -func ReloadYamlTemplate(config any, mappings map[string]string) error { +func ReloadYamlTemplate(config interface{}, mappings map[string]string) error { text, err := goyaml.Marshal(config) if err != nil { @@ -164,7 +164,7 @@ func ReloadYamlTemplate(config any, mappings map[string]string) error { text = []byte(strings.ReplaceAll(string(text), template, value)) } - return goyaml.Unmarshal(text, config) + return goyaml.Unmarshal(text, &config) } // FindYamlTemplates finds strings with a given prefix in a config. From dbe2cd449dabee6c9e5cfba701b7a91230caeaac Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 17 Jan 2024 11:34:46 -0600 Subject: [PATCH 030/172] Fix create --- src/pkg/packager/create.go | 9 ++++++--- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/creator/new.go | 4 ++-- src/pkg/packager/creator/normal.go | 22 +++++++++++----------- src/pkg/packager/creator/skeleton.go | 14 +++++++------- src/pkg/packager/creator/template.go | 2 +- src/pkg/packager/dev.go | 7 +++++-- src/pkg/packager/prepare.go | 2 +- src/pkg/packager/publish.go | 6 ++++-- src/pkg/utils/yaml.go | 4 ++-- 10 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 183f3742cd..2591c2d416 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -25,17 +25,20 @@ func (p *Packager) Create() (err error) { return err } - c, err := creator.New(&p.cfg.CreateOpts) + c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - if err := c.LoadPackageDefinition(); err != nil { + pkg, err := c.LoadPackageDefinition() + if err != nil { return err } + p.cfg.Pkg = *pkg + // Perform early package validation. - if err := validate.Run(p.cfg.Pkg); err != nil { + if err := validate.Run(*pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index e76e4cb921..cdfd5c4583 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -124,7 +124,7 @@ func (p *Packager) assemble() error { } func (p *Packager) assembleSkeleton() error { - c, err := creator.New(&p.cfg.CreateOpts) + c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 6381b77099..35645926c0 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -13,7 +13,7 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition() error + LoadPackageDefinition() (pkg *types.ZarfPackage, err error) ComposeComponents() (warnings []string, err error) FillActiveTemplate() (warnings []string, err error) ProcessExtensions() error @@ -22,7 +22,7 @@ type Creator interface { } // New returns a new Creator based on the provided create options. -func New(createOpts *types.ZarfCreateOptions) (Creator, error) { +func New(createOpts types.ZarfCreateOptions) (Creator, error) { sc := &SkeletonCreator{} pc := &PackageCreator{} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 24bff61301..cc0bc468dd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -43,9 +43,9 @@ type PackageCreator struct { } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition() (err error) { - if err := utils.ReadYaml(layout.ZarfYAML, pc.pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) +func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, err error) { + if err := utils.ReadYaml(layout.ZarfYAML, &pc.pkg); err != nil { + return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } pc.arch = config.GetArch() @@ -59,41 +59,41 @@ func (pc *PackageCreator) LoadPackageDefinition() (err error) { // Compose components into a single zarf.yaml file if pc.warnings, err = pc.ComposeComponents(); err != nil { - return err + return nil, err } // After components are composed, template the active package. if pc.warnings, err = pc.FillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) + return nil, fmt.Errorf("unable to fill values in template: %s", err.Error()) } // After templates are filled process any create extensions if err := pc.ProcessExtensions(); err != nil { - return err + return nil, err } // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package if pc.createOpts.DifferentialData.DifferentialPackagePath != "" { // Load the images and repos from the 'reference' package if err := pc.LoadDifferentialData(); err != nil { - return err + return nil, err } // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building // If the package versions are the same return an error if pc.createOpts.DifferentialData.DifferentialPackageVersion == pc.pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) + return nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pc.pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + return nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } // Handle any potential differential images/repos before going forward if err := pc.RemoveCopiesFromDifferentialPackage(); err != nil { - return err + return nil, err } } - return nil + return &pc.pkg, nil } // ComposeComponents builds the composed components list for the current config. diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index d5e3d3a7fd..e7ca8d280e 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -26,7 +26,7 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - pkg *types.ZarfPackage + pkg types.ZarfPackage createOpts types.ZarfCreateOptions layout *layout.PackagePaths arch string @@ -34,13 +34,13 @@ type SkeletonCreator struct { } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition() (err error) { +func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, err error) { if err := utils.ReadYaml(layout.ZarfYAML, &sc.pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } sc.arch = config.GetArch(sc.pkg.Metadata.Architecture, sc.pkg.Build.Architecture) - if utils.IsInitConfig(*sc.pkg) { + if utils.IsInitConfig(sc.pkg) { sc.pkg.Metadata.Version = config.CLIVersion } @@ -51,15 +51,15 @@ func (sc *SkeletonCreator) LoadPackageDefinition() (err error) { // Compose components into a single zarf.yaml file sc.warnings, err = sc.ComposeComponents() if err != nil { - return err + return nil, err } - return nil + return &sc.pkg, nil } // ComposeComponents builds the composed components list for the current config. func (sc *SkeletonCreator) ComposeComponents() (warnings []string, err error) { - return composeComponents(sc.pkg, sc.createOpts) + return composeComponents(&sc.pkg, sc.createOpts) } // FillActiveTemplate handles setting the active variables and reloading the base template. diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 6cae02af85..f4e347addb 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -67,7 +67,7 @@ func fillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptio // Add special variable for the current package architecture templateMap[types.ZarfPackageArch] = pkg.Build.Architecture - if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { + if err := utils.ReloadYamlTemplate(pkg, templateMap); err != nil { return nil, err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index c6cefdd6f1..479f1cb4e4 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -29,15 +29,18 @@ func (p *Packager) DevDeploy() error { return err } - c, err := creator.New(&p.cfg.CreateOpts) + c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - if err := c.LoadPackageDefinition(); err != nil { + pkg, err := c.LoadPackageDefinition() + if err != nil { return err } + p.cfg.Pkg = *pkg + // Filter out components that are not compatible with this system p.filterComponents() diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 17837a4390..2870d45912 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -53,7 +53,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - c, err := creator.New(&p.cfg.CreateOpts) + c, err := creator.New(p.cfg.CreateOpts) if err != nil { return nil, err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 9465b564b2..bbb1986a7e 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -92,13 +92,15 @@ func (p *Packager) Publish() (err error) { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - c, err := creator.New(&p.cfg.CreateOpts) + c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - if err := c.LoadPackageDefinition(); err != nil { + pkg, err := c.LoadPackageDefinition() + if err != nil { return err } + p.cfg.Pkg = *pkg if err := p.assembleSkeleton(); err != nil { return err } diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index 9b95c43969..52faaef21f 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -123,7 +123,7 @@ func AddRootHint(hints map[string]string, rootKey string, hintText string) map[s } // ReadYaml reads a yaml file and unmarshals it into a given config. -func ReadYaml(path string, destConfig interface{}) error { +func ReadYaml(path string, destConfig any) error { message.Debugf("Reading YAML at %s", path) file, err := os.ReadFile(path) @@ -131,7 +131,7 @@ func ReadYaml(path string, destConfig interface{}) error { return err } - return goyaml.Unmarshal(file, &destConfig) + return goyaml.Unmarshal(file, destConfig) } // WriteYaml writes a given config to a yaml file on disk. From 1a6503d023ffa28e267e058e2f6944372ef3e027 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 19 Jan 2024 00:40:52 -0600 Subject: [PATCH 031/172] Cleanup Creator interface and function signatures --- src/pkg/packager/create.go | 12 +- src/pkg/packager/create_stages.go | 10 +- src/pkg/packager/creator/compose.go | 10 +- src/pkg/packager/creator/differential.go | 135 +++++++++++ src/pkg/packager/creator/extensions.go | 45 ++++ src/pkg/packager/creator/new.go | 30 +-- src/pkg/packager/creator/normal.go | 295 +++-------------------- src/pkg/packager/creator/skeleton.go | 146 +---------- src/pkg/packager/creator/template.go | 16 +- src/pkg/packager/creator/utils.go | 77 ++++++ src/pkg/packager/dev.go | 8 +- src/pkg/packager/prepare.go | 13 +- src/pkg/packager/publish.go | 7 +- src/pkg/utils/yaml.go | 6 +- 14 files changed, 348 insertions(+), 462 deletions(-) create mode 100644 src/pkg/packager/creator/differential.go create mode 100644 src/pkg/packager/creator/extensions.go create mode 100644 src/pkg/packager/creator/utils.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 2591c2d416..10b8982488 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,7 +10,9 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -24,18 +26,20 @@ func (p *Packager) Create() (err error) { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %w", err) + } c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - pkg, err := c.LoadPackageDefinition() + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) if err != nil { return err } - - p.cfg.Pkg = *pkg + p.warnings = append(p.warnings, warnings...) // Perform early package validation. if err := validate.Run(*pkg); err != nil { @@ -55,5 +59,7 @@ func (p *Packager) Create() (err error) { return err } + p.cfg.Pkg = *pkg + return p.output() } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index cdfd5c4583..fc826d9e4f 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -124,18 +124,14 @@ func (p *Packager) assemble() error { } func (p *Packager) assembleSkeleton() error { - c, err := creator.New(p.cfg.CreateOpts) + pkg, err := creator.ProcessExtensions(&p.cfg.Pkg, p.cfg.CreateOpts, p.layout) if err != nil { - return err - } - - if err := c.ProcessExtensions(); err != nil { - return err + return nil } for _, warning := range p.warnings { message.Warn(warning) } - for idx, component := range p.cfg.Pkg.Components { + for idx, component := range pkg.Components { if err := p.addComponent(idx, component); err != nil { return err } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index b04be2117d..746c049d5b 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,7 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func composeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { +func ComposeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { components := []types.ZarfComponent{} pkgVars := pkg.Variables @@ -30,7 +30,7 @@ func composeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOption // build the import chain chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, createOpts.Flavor) if err != nil { - return nil, err + return nil, nil, err } message.Debugf("%s", chain) @@ -40,7 +40,7 @@ func composeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOption // get the composed component composed, err := chain.Compose() if err != nil { - return nil, err + return nil, nil, err } components = append(components, *composed) @@ -55,5 +55,7 @@ func composeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOption pkg.Variables = pkgVars pkg.Constants = pkgConsts - return warnings, nil + composedPkg = pkg + + return composedPkg, warnings, nil } diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go new file mode 100644 index 0000000000..13f11689b2 --- /dev/null +++ b/src/pkg/packager/creator/differential.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mholt/archiver" +) + +// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, +// and creates a list of all images and repos that are in the reference package. +func LoadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(diffData.DifferentialPackagePath) { + remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath) + if err != nil { + return nil, err + } + pkg, err := remote.FetchZarfYAML() + if err != nil { + return nil, err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return nil, err + } + } else { + if err := archiver.Extract(diffData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return nil, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return nil, fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + diffData.DifferentialImages = allIncludedImagesMap + diffData.DifferentialRepos = allIncludedReposMap + diffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return diffData, nil +} + +// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. +func RemoveCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return nil, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !diffData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return nil, err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !diffData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + pkg.Components[idx] = component + } + + return pkg, nil +} diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go new file mode 100644 index 0000000000..a314d372ec --- /dev/null +++ b/src/pkg/packager/creator/extensions.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/types" +) + +func ProcessExtensions(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range pkg.Components { + componentPaths, err := layout.Components.Create(c) + if err != nil { + return nil, err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if createOpts.IsSkeleton { + if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + pkg.Components = components + + return pkg, nil +} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 35645926c0..d71bd571f4 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -5,42 +5,18 @@ package creator import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/types" ) // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition() (pkg *types.ZarfPackage, err error) - ComposeComponents() (warnings []string, err error) - FillActiveTemplate() (warnings []string, err error) - ProcessExtensions() error - LoadDifferentialData() error - RemoveCopiesFromDifferentialPackage() error + LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) } // New returns a new Creator based on the provided create options. func New(createOpts types.ZarfCreateOptions) (Creator, error) { - sc := &SkeletonCreator{} - pc := &PackageCreator{} - if createOpts.IsSkeleton { - // If the temp directory is not set, set it to the default - if sc.layout == nil { - if err := sc.setTempDirectory(config.CommonOptions.TempDirectory); err != nil { - return nil, fmt.Errorf("unable to create package temp paths: %w", err) - } - } - return sc, nil - } - - // If the temp directory is not set, set it to the default - if pc.layout == nil { - if err := pc.setTempDirectory(config.CommonOptions.TempDirectory); err != nil { - return nil, fmt.Errorf("unable to create package temp paths: %w", err) - } + return &SkeletonCreator{}, nil } - return pc, nil + return &PackageCreator{}, nil } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index cc0bc468dd..df9f15bcdd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -7,25 +7,11 @@ package creator import ( "errors" "fmt" - "os" - "path/filepath" - "runtime" - "time" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver" ) var ( @@ -35,277 +21,60 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { - pkg types.ZarfPackage - createOpts types.ZarfCreateOptions - layout *layout.PackagePaths - arch string - warnings []string + cfg types.PackagerConfig + layout *layout.PackagePaths } -// LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, err error) { - if err := utils.ReadYaml(layout.ZarfYAML, &pc.pkg); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - pc.arch = config.GetArch() - - if utils.IsInitConfig(pc.pkg) { - pc.pkg.Metadata.Version = config.CLIVersion - } - - if err := pc.setPackageBuildMetadata(); err != nil { +// LoadPackageDefinition loads and configures a zarf.yaml file during package create. +func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + pkg, err = setPackageMetadata(pkg, pc.cfg.CreateOpts) + if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - if pc.warnings, err = pc.ComposeComponents(); err != nil { - return nil, err + pkg, composeWarnings, err := ComposeComponents(pkg, pc.cfg.CreateOpts) + if err != nil { + return nil, nil, err } - // After components are composed, template the active package. - if pc.warnings, err = pc.FillActiveTemplate(); err != nil { - return nil, fmt.Errorf("unable to fill values in template: %s", err.Error()) - } + warnings = append(warnings, composeWarnings...) - // After templates are filled process any create extensions - if err := pc.ProcessExtensions(); err != nil { - return nil, err + // After components are composed, template the active package. + templateWarnings, err := FillActiveTemplate(pkg, pc.cfg.CreateOpts) + if err != nil { + return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if pc.createOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package - if err := pc.LoadDifferentialData(); err != nil { - return nil, err - } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if pc.createOpts.DifferentialData.DifferentialPackageVersion == pc.pkg.Metadata.Version { - return nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pc.pkg.Metadata.Version == "" { - return nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } + warnings = append(warnings, templateWarnings...) - // Handle any potential differential images/repos before going forward - if err := pc.RemoveCopiesFromDifferentialPackage(); err != nil { - return nil, err - } + // After templates are filled process any create extensions + pkg, err = ProcessExtensions(pkg, pc.cfg.CreateOpts, pc.layout) + if err != nil { + return nil, nil, err } - return &pc.pkg, nil -} - -// ComposeComponents builds the composed components list for the current config. -func (pc *PackageCreator) ComposeComponents() (warnings []string, err error) { - return composeComponents(&pc.pkg, pc.createOpts) -} - -// FillActiveTemplate handles setting the active variables and reloading the base template. -func (pc *PackageCreator) FillActiveTemplate() (warnings []string, err error) { - return fillActiveTemplate(&pc.pkg, pc.createOpts) -} - -func (pc *PackageCreator) ProcessExtensions() error { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range pc.pkg.Components { - componentPaths, err := pc.layout.Components.Create(c) + // If we are creating a differential package, remove duplicate images and repos. + if pkg.Build.Differential { + diffData, err := LoadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Run(pc.pkg.Metadata.YOLO, componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } + return nil, nil, err } - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - pc.pkg.Components = components - - return nil -} - -// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, -// and creates a list of all images and repos that are in the reference package. -func (pc *PackageCreator) LoadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - pc.pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(pc.createOpts.DifferentialData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(pc.createOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err + if pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pc.cfg.Pkg.Metadata.Version { + return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - } else { - if err := archiver.Extract(pc.createOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + if pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pc.cfg.Pkg.Metadata.Version == "" { + return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } - } - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - pc.createOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - pc.createOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - pc.createOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func (pc *PackageCreator) RemoveCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if pc.createOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range pc.pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !pc.createOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !pc.createOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } + pkg, err = RemoveCopiesFromDifferentialPackage(pkg, diffData) + if err != nil { + return nil, nil, err } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component } - // Update the package with the new component list - for idx, component := range componentMap { - pc.pkg.Components[idx] = component - } - - return nil -} - -// setTempDirectory sets the temp directory for the PackageCreator. -func (pc *PackageCreator) setTempDirectory(path string) error { - dir, err := utils.MakeTempDir(path) - if err != nil { - return fmt.Errorf("unable to create package temp paths: %w", err) - } - - pc.layout = layout.New(dir) - return nil -} - -// setPackageBuildMetadata sets various package build metadata. -func (pc *PackageCreator) setPackageBuildMetadata() error { - now := time.Now() - // Just use $USER env variable to avoid CGO issue. - // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. - // Record the name of the user creating the package. - if runtime.GOOS == "windows" { - pc.pkg.Build.User = os.Getenv("USERNAME") - } else { - pc.pkg.Build.User = os.Getenv("USER") - } - - hostname, err := os.Hostname() - if err != nil { - return err - } - - pc.pkg.Metadata.Architecture = pc.arch - pc.pkg.Build.Architecture = pc.arch - - // Record the time of package creation. - pc.pkg.Build.Timestamp = now.Format(time.RFC1123Z) - - // Record the Zarf Version the CLI was built with. - pc.pkg.Build.Version = config.CLIVersion - - // Record the hostname of the package creation terminal. - pc.pkg.Build.Terminal = hostname - - // Record the migrations that will be run on the package. - pc.pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Record the flavor of Zarf used to build this package (if any). - pc.pkg.Build.Flavor = pc.createOpts.Flavor - - pc.pkg.Build.RegistryOverrides = pc.createOpts.RegistryOverrides - - // Record the latest version of Zarf without breaking changes to the package structure. - pc.pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + pc.cfg.Pkg = *pkg - return nil + return &pc.cfg.Pkg, warnings, nil } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index e7ca8d280e..4e1dc4b622 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -5,17 +5,7 @@ package creator import ( - "fmt" - "os" - "runtime" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -26,142 +16,24 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - pkg types.ZarfPackage - createOpts types.ZarfCreateOptions - layout *layout.PackagePaths - arch string - warnings []string + cfg types.PackagerConfig } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, err error) { - if err := utils.ReadYaml(layout.ZarfYAML, &sc.pkg); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - sc.arch = config.GetArch(sc.pkg.Metadata.Architecture, sc.pkg.Build.Architecture) - - if utils.IsInitConfig(sc.pkg) { - sc.pkg.Metadata.Version = config.CLIVersion - } - - if err := sc.setPackageBuildMetadata(); err != nil { +func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + pkg, err = setPackageMetadata(pkg, sc.cfg.CreateOpts) + if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - sc.warnings, err = sc.ComposeComponents() - if err != nil { - return nil, err - } - - return &sc.pkg, nil -} - -// ComposeComponents builds the composed components list for the current config. -func (sc *SkeletonCreator) ComposeComponents() (warnings []string, err error) { - return composeComponents(&sc.pkg, sc.createOpts) -} - -// FillActiveTemplate handles setting the active variables and reloading the base template. -func (sc *SkeletonCreator) FillActiveTemplate() (warnings []string, err error) { - return nil, fmt.Errorf("not implemented") -} - -func (sc *SkeletonCreator) ProcessExtensions() error { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range sc.pkg.Components { - componentPaths, err := sc.layout.Components.Create(c) - if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - sc.pkg.Components = components - - return nil -} - -// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, -// and creates a list of all images and repos that are in the reference package. -// -// This is not implemented. -func (sc *SkeletonCreator) LoadDifferentialData() error { - return fmt.Errorf("not implemented") -} - -// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -// -// This is not implemented. -func (sc *SkeletonCreator) RemoveCopiesFromDifferentialPackage() error { - return fmt.Errorf("not implemented") -} - -// setTempDirectory sets the temp directory for the SkeletonCreator. -func (sc *SkeletonCreator) setTempDirectory(path string) error { - dir, err := utils.MakeTempDir(path) + pkg, composeWarnings, err := ComposeComponents(pkg, sc.cfg.CreateOpts) if err != nil { - return fmt.Errorf("unable to create package temp paths: %w", err) + return nil, nil, err } + warnings = append(warnings, composeWarnings...) - sc.layout = layout.New(dir) - return nil -} - -// setPackageBuildMetadata sets various package build metadata. -func (sc *SkeletonCreator) setPackageBuildMetadata() error { - now := time.Now() - // Just use $USER env variable to avoid CGO issue. - // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. - // Record the name of the user creating the package. - if runtime.GOOS == "windows" { - sc.pkg.Build.User = os.Getenv("USERNAME") - } else { - sc.pkg.Build.User = os.Getenv("USER") - } - - hostname, err := os.Hostname() - if err != nil { - return err - } - - sc.pkg.Metadata.Architecture = "skeleton" - sc.pkg.Build.Architecture = "skeleton" - - // Record the time of package creation. - sc.pkg.Build.Timestamp = now.Format(time.RFC1123Z) - - // Record the Zarf Version the CLI was built with. - sc.pkg.Build.Version = config.CLIVersion - - // Record the hostname of the package creation terminal. - sc.pkg.Build.Terminal = hostname - - // Record the migrations that will be run on the package. - sc.pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Record the flavor of Zarf used to build this package (if any). - sc.pkg.Build.Flavor = sc.createOpts.Flavor - - sc.pkg.Build.RegistryOverrides = sc.createOpts.RegistryOverrides - - // Record the latest version of Zarf without breaking changes to the package structure. - sc.pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + sc.cfg.Pkg = *pkg - return nil + return &sc.cfg.Pkg, warnings, nil } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index f4e347addb..066fda4bf1 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func fillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { +func FillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { @@ -33,14 +33,13 @@ func fillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptio setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ Name: key, }) - - if err == nil { - createOpts.SetVariables[key] = setVal - } else { + if err != nil { return err } + createOpts.SetVariables[key] = setVal } else if !present { - return fmt.Errorf("template '%s' must be '--set' when using the '--confirm' flag", key) + // erroring out here + return fmt.Errorf("template %q must be '--set' when using the '--confirm' flag", key) } } @@ -75,9 +74,8 @@ func fillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptio } // reloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func reloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for _, component := range zarfPackage.Components { +func reloadComponentTemplatesInPackage(pkg *types.ZarfPackage) error { + for _, component := range pkg.Components { mappings := map[string]string{} mappings[types.ZarfComponentName] = component.Name diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go new file mode 100644 index 0000000000..0a80dc693d --- /dev/null +++ b/src/pkg/packager/creator/utils.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "os" + "runtime" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" +) + +// setPackageMetadata sets various package metadata. +func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (*types.ZarfPackage, error) { + now := time.Now() + // Just use $USER env variable to avoid CGO issue. + // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. + // Record the name of the user creating the package. + if runtime.GOOS == "windows" { + pkg.Build.User = os.Getenv("USERNAME") + } else { + pkg.Build.User = os.Getenv("USER") + } + + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + if utils.IsInitConfig(*pkg) { + pkg.Metadata.Version = config.CLIVersion + } + + // Set package architecture + if createOpts.IsSkeleton { + pkg.Metadata.Architecture = "skeleton" + } + if pkg.Metadata.Architecture == "" { + pkg.Metadata.Architecture = config.GetArch() + } + pkg.Build.Architecture = pkg.Metadata.Architecture + + // Record the time of package creation. + pkg.Build.Timestamp = now.Format(time.RFC1123Z) + + // Record the Zarf Version the CLI was built with. + pkg.Build.Version = config.CLIVersion + + // Record the hostname of the package creation terminal. + pkg.Build.Terminal = hostname + + // If the --differential flag was used, record that this is a differential package. + if createOpts.DifferentialData.DifferentialPackagePath != "" { + pkg.Build.Differential = true + } + + // Record the migrations that will be run on the package. + pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Record the flavor of Zarf used to build this package (if any). + pkg.Build.Flavor = createOpts.Flavor + + pkg.Build.RegistryOverrides = createOpts.RegistryOverrides + + // Record the latest version of Zarf without breaking changes to the package structure. + pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + + return pkg, nil +} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 479f1cb4e4..06d2d917f1 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -10,8 +10,10 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -28,16 +30,20 @@ func (p *Packager) DevDeploy() error { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %w", err) + } c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - pkg, err := c.LoadPackageDefinition() + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) p.cfg.Pkg = *pkg diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 2870d45912..e9539c8c47 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -53,20 +53,21 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - c, err := creator.New(p.cfg.CreateOpts) + pkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, p.cfg.CreateOpts) if err != nil { return nil, err } - if p.warnings, err = c.ComposeComponents(); err != nil { - return nil, err - } + p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - if p.warnings, err = c.FillActiveTemplate(); err != nil { - return nil, fmt.Errorf("unable to fill values in template: %s", err.Error()) + templateWarnings, err := creator.FillActiveTemplate(pkg, p.cfg.CreateOpts) + if err != nil { + return nil, fmt.Errorf("unable to fill values in template: %w", err) } + p.warnings = append(p.warnings, templateWarnings...) + for _, warning := range p.warnings { message.Warn(warning) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index bbb1986a7e..ff4ba6bd65 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -92,14 +93,18 @@ func (p *Packager) Publish() (err error) { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } + if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %w", err) + } c, err := creator.New(p.cfg.CreateOpts) if err != nil { return err } - pkg, err := c.LoadPackageDefinition() + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) p.cfg.Pkg = *pkg if err := p.assembleSkeleton(); err != nil { return err diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index 52faaef21f..595551e0b1 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -125,8 +125,8 @@ func AddRootHint(hints map[string]string, rootKey string, hintText string) map[s // ReadYaml reads a yaml file and unmarshals it into a given config. func ReadYaml(path string, destConfig any) error { message.Debugf("Reading YAML at %s", path) - file, err := os.ReadFile(path) + file, err := os.ReadFile(path) if err != nil { return err } @@ -148,7 +148,6 @@ func WriteYaml(path string, srcConfig any, perm fs.FileMode) error { // ReloadYamlTemplate marshals a given config, replaces strings and unmarshals it back. func ReloadYamlTemplate(config interface{}, mappings map[string]string) error { text, err := goyaml.Marshal(config) - if err != nil { return err } @@ -172,9 +171,8 @@ func FindYamlTemplates(config any, prefix string, suffix string) (map[string]str mappings := map[string]string{} text, err := goyaml.Marshal(config) - if err != nil { - return mappings, err + return nil, err } // Find all strings that are between the given prefix and suffix From 5ee5c89015b44fa14d6505e6808f1bc238fc4c97 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 19 Jan 2024 10:49:46 -0600 Subject: [PATCH 032/172] Fix package templating --- src/pkg/packager/create.go | 2 +- src/pkg/packager/create_stages.go | 2 +- src/pkg/packager/creator/compose.go | 2 +- src/pkg/packager/creator/extensions.go | 2 +- src/pkg/packager/creator/new.go | 3 ++- src/pkg/packager/creator/normal.go | 25 ++++++++++--------------- src/pkg/packager/creator/skeleton.go | 15 ++++++--------- src/pkg/packager/creator/template.go | 2 +- src/pkg/packager/creator/utils.go | 2 +- src/pkg/packager/dev.go | 2 +- src/pkg/packager/prepare.go | 4 ++-- src/pkg/packager/publish.go | 2 +- 12 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 10b8982488..11ea7c289c 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -35,7 +35,7 @@ func (p *Packager) Create() (err error) { return err } - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { return err } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index fc826d9e4f..b5245b8c8c 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -124,7 +124,7 @@ func (p *Packager) assemble() error { } func (p *Packager) assembleSkeleton() error { - pkg, err := creator.ProcessExtensions(&p.cfg.Pkg, p.cfg.CreateOpts, p.layout) + pkg, err := creator.ProcessExtensions(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { return nil } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index 746c049d5b..fdff75a6f6 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,7 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func ComposeComponents(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { +func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { components := []types.ZarfComponent{} pkgVars := pkg.Variables diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index a314d372ec..317d45bd20 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -12,7 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func ProcessExtensions(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { +func ProcessExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index d71bd571f4..933e6e9b5c 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -5,12 +5,13 @@ package creator import ( + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ) // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) + LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index df9f15bcdd..96ad618a1f 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -20,20 +20,17 @@ var ( ) // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. -type PackageCreator struct { - cfg types.PackagerConfig - layout *layout.PackagePaths -} +type PackageCreator struct{} // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - pkg, err = setPackageMetadata(pkg, pc.cfg.CreateOpts) +func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + pkg, err = setPackageMetadata(pkg, createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(pkg, pc.cfg.CreateOpts) + pkg, composeWarnings, err := ComposeComponents(pkg, createOpts) if err != nil { return nil, nil, err } @@ -41,7 +38,7 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedP warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templateWarnings, err := FillActiveTemplate(pkg, pc.cfg.CreateOpts) + templateWarnings, err := FillActiveTemplate(pkg, createOpts) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -49,22 +46,22 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedP warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - pkg, err = ProcessExtensions(pkg, pc.cfg.CreateOpts, pc.layout) + pkg, err = ProcessExtensions(pkg, createOpts, layout) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. if pkg.Build.Differential { - diffData, err := LoadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) + diffData, err := LoadDifferentialData(&createOpts.DifferentialData) if err != nil { return nil, nil, err } - if pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pc.cfg.Pkg.Metadata.Version { + if createOpts.DifferentialData.DifferentialPackageVersion == pkg.Metadata.Version { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - if pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pc.cfg.Pkg.Metadata.Version == "" { + if createOpts.DifferentialData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" { return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } @@ -74,7 +71,5 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedP } } - pc.cfg.Pkg = *pkg - - return &pc.cfg.Pkg, warnings, nil + return pkg, warnings, nil } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 4e1dc4b622..647d756b55 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -5,6 +5,7 @@ package creator import ( + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -15,25 +16,21 @@ var ( ) // SkeletonCreator provides methods for creating skeleton Zarf packages. -type SkeletonCreator struct { - cfg types.PackagerConfig -} +type SkeletonCreator struct{} // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - pkg, err = setPackageMetadata(pkg, sc.cfg.CreateOpts) +func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, _ *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + pkg, err = setPackageMetadata(pkg, createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(pkg, sc.cfg.CreateOpts) + pkg, composeWarnings, err := ComposeComponents(pkg, createOpts) if err != nil { return nil, nil, err } warnings = append(warnings, composeWarnings...) - sc.cfg.Pkg = *pkg - - return &sc.cfg.Pkg, warnings, nil + return pkg, warnings, nil } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 066fda4bf1..32dd62f62d 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func FillActiveTemplate(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (warnings []string, err error) { +func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (warnings []string, err error) { templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 0a80dc693d..36a1fcf6af 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -16,7 +16,7 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) (*types.ZarfPackage, error) { +func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (*types.ZarfPackage, error) { now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 06d2d917f1..5f0c60de0a 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -39,7 +39,7 @@ func (p *Packager) DevDeploy() error { return err } - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { return err } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index e9539c8c47..5107e2c84b 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -53,7 +53,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - pkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, p.cfg.CreateOpts) + pkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templateWarnings, err := creator.FillActiveTemplate(pkg, p.cfg.CreateOpts) + templateWarnings, err := creator.FillActiveTemplate(pkg, &p.cfg.CreateOpts) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index ff4ba6bd65..510993e4c9 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -100,7 +100,7 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg) + pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { return err } From 9b108e930fdedfa5b7ce9425b6c5210ea8a3588a Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 19 Jan 2024 13:28:46 -0600 Subject: [PATCH 033/172] Fix ReloadYamlTemplate --- src/pkg/utils/yaml.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index 595551e0b1..f2c73d5f77 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -146,7 +146,7 @@ func WriteYaml(path string, srcConfig any, perm fs.FileMode) error { } // ReloadYamlTemplate marshals a given config, replaces strings and unmarshals it back. -func ReloadYamlTemplate(config interface{}, mappings map[string]string) error { +func ReloadYamlTemplate(config any, mappings map[string]string) error { text, err := goyaml.Marshal(config) if err != nil { return err @@ -163,7 +163,7 @@ func ReloadYamlTemplate(config interface{}, mappings map[string]string) error { text = []byte(strings.ReplaceAll(string(text), template, value)) } - return goyaml.Unmarshal(text, &config) + return goyaml.Unmarshal(text, config) } // FindYamlTemplates finds strings with a given prefix in a config. From 41cb8a850d9aac3c9ad955c82daf29dfe745f07e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Sun, 21 Jan 2024 21:06:03 -0600 Subject: [PATCH 034/172] Ensure package arch is set --- src/pkg/packager/common.go | 2 +- src/pkg/packager/create.go | 4 ---- src/pkg/packager/create_stages.go | 4 ++-- src/pkg/packager/creator/compose.go | 2 +- src/pkg/packager/prepare.go | 3 ++- src/pkg/packager/yaml.go | 6 ------ 6 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index b6a9392e67..a2e83a1cb0 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -183,7 +183,7 @@ func (p *Packager) GetPackageName() string { suffix = "tar" } - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.cfg.Pkg.Metadata.Architecture) if p.cfg.Pkg.Build.Differential { packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) } else if p.cfg.Pkg.Metadata.Version != "" { diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 11ea7c289c..85ecd6b56e 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -17,12 +17,10 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { - cwd, err := os.Getwd() if err != nil { return err } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } @@ -59,7 +57,5 @@ func (p *Packager) Create() (err error) { return err } - p.cfg.Pkg = *pkg - return p.output() } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index b5245b8c8c..bbc5521419 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -146,7 +146,7 @@ func (p *Packager) assembleSkeleton() error { } p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - return p.writeYaml() + return utils.WriteYaml(p.layout.ZarfYAML, pkg, 0400) } // output assumes it is running from cwd, not the build directory @@ -168,7 +168,7 @@ func (p *Packager) output() error { p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum // Save the transformed config. - if err := p.writeYaml(); err != nil { + if err := utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index fdff75a6f6..31c49aa4a6 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -17,7 +17,7 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio pkgConsts := pkg.Constants for i, component := range pkg.Components { - arch := pkg.Build.Architecture + arch := pkg.Metadata.Architecture // filter by architecture if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { continue diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 5107e2c84b..5134c22d1d 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -51,7 +51,8 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return nil, err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) + + p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) pkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) if err != nil { diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 656253813e..1d8573ebed 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -8,7 +8,6 @@ import ( "runtime" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -42,8 +41,3 @@ func (p *Packager) filterComponents() { // Update the active package with the filtered components. p.cfg.Pkg.Components = filteredComponents } - -// writeYaml adds build information and writes the config to the temp directory. -func (p *Packager) writeYaml() error { - return utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400) -} From 15b283b4b63275b2fdf2cbd489541f22987eb41c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Sun, 21 Jan 2024 21:15:16 -0600 Subject: [PATCH 035/172] Use archiver/v3 --- go.mod | 2 -- go.sum | 4 ---- src/pkg/packager/creator/differential.go | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/go.mod b/go.mod index a42f414667..50fd25a90a 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/gofrs/flock v0.8.1 github.com/google/go-containerregistry v0.17.0 github.com/gosuri/uitable v0.0.4 - github.com/mholt/archiver v3.1.1+incompatible github.com/mholt/archiver/v3 v3.5.1 github.com/moby/moby v24.0.7+incompatible github.com/opencontainers/image-spec v1.1.0-rc5 @@ -379,7 +378,6 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect diff --git a/go.sum b/go.sum index b77ea206f7..376522f5d2 100644 --- a/go.sum +++ b/go.sum @@ -1257,8 +1257,6 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQth github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= -github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 h1:tQRHcLQwnwrPq2j2Qra/NnyjyESBGwdeBeVdAE9kXYg= @@ -1412,8 +1410,6 @@ github.com/petergtz/pegomock v2.9.0+incompatible h1:BKfb5XfkJfehe5T+O1xD4Zm26Sb9 github.com/petergtz/pegomock v2.9.0+incompatible/go.mod h1:nuBLWZpVyv/fLo56qTwt/AUau7jgouO1h7bEvZCq82o= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 13f11689b2..b03867958b 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -19,7 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver" + "github.com/mholt/archiver/v3" ) // LoadDifferentialData extracts the Zarf config of a designated 'reference' package, From 5ccfdeeaf7416c86b67585bae86e0b3e9ddb7f9f Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Sun, 21 Jan 2024 22:31:11 -0600 Subject: [PATCH 036/172] Fix component migrations on create --- src/pkg/packager/create.go | 1 + src/pkg/packager/creator/compose.go | 3 ++- src/pkg/packager/creator/utils.go | 6 ------ src/pkg/packager/deprecated/common.go | 2 ++ src/test/e2e/03_deprecations_test.go | 1 - 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 85ecd6b56e..de5abd045f 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -37,6 +37,7 @@ func (p *Packager) Create() (err error) { if err != nil { return err } + p.warnings = append(p.warnings, warnings...) // Perform early package validation. diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index 31c49aa4a6..717bba5117 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -35,7 +35,8 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio message.Debugf("%s", chain) // migrate any deprecated component configurations now - warnings = chain.Migrate(pkg.Build) + warning := chain.Migrate(pkg.Build) + warnings = append(warnings, warning...) // get the composed component composed, err := chain.Compose() diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 36a1fcf6af..cb4dfdd5f7 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -59,12 +59,6 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti pkg.Build.Differential = true } - // Record the migrations that will be run on the package. - pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - // Record the flavor of Zarf used to build this package (if any). pkg.Build.Flavor = createOpts.Flavor diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 2fdbfdcaf3..8afd90d245 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -54,6 +54,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) var warning string if migratedComponent, warning = migrateScriptsToActions(migratedComponent); warning != "" { warnings = append(warnings, warning) + build.Migrations = append(build.Migrations, ScriptsToActionsMigrated) } } @@ -65,6 +66,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) var warning string if migratedComponent, warning = migrateSetVariableToSetVariables(migratedComponent); warning != "" { warnings = append(warnings, warning) + build.Migrations = append(build.Migrations, PluralizeSetVariable) } } diff --git a/src/test/e2e/03_deprecations_test.go b/src/test/e2e/03_deprecations_test.go index 808a5b25af..1ecde29d18 100644 --- a/src/test/e2e/03_deprecations_test.go +++ b/src/test/e2e/03_deprecations_test.go @@ -24,7 +24,6 @@ func TestDeprecatedComponentScripts(t *testing.T) { "test-deprecated-deploy-after-hook.txt", } allArtifacts := append(deployArtifacts, prepareArtifact) - e2e.CleanFiles(allArtifacts...) defer e2e.CleanFiles(allArtifacts...) // 1. Try creating the package to test the create scripts From 90c34cd74ee39274581b4eeb3708e1336103bf8b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Sun, 21 Jan 2024 22:36:43 -0600 Subject: [PATCH 037/172] Fix ProcessExtensions bug --- src/pkg/packager/creator/extensions.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index 317d45bd20..f185bba8db 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -28,9 +28,10 @@ func ProcessExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { return nil, fmt.Errorf("unable to process bigbang extension: %w", err) } - } - if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { - return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } else { + if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } } } From 7a08474691056860153f0ad55d746c83a47ee093 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Sun, 21 Jan 2024 23:06:26 -0600 Subject: [PATCH 038/172] Fix component migratioin warnings --- src/pkg/packager/creator/compose.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index 717bba5117..f1214e15fd 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -35,8 +35,8 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio message.Debugf("%s", chain) // migrate any deprecated component configurations now - warning := chain.Migrate(pkg.Build) - warnings = append(warnings, warning...) + migrationWarnings := chain.Migrate(pkg.Build) + warnings = append(warnings, migrationWarnings...) // get the composed component composed, err := chain.Compose() From 8df5a39d62c26214753a372c451b02065a9e81bd Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 00:13:09 -0600 Subject: [PATCH 039/172] Set pkg build migrations in p.output on create --- src/pkg/packager/create_stages.go | 7 +++++++ src/pkg/packager/deprecated/common.go | 2 -- src/test/e2e/03_deprecations_test.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index bbc5521419..4096bec3b5 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -24,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -167,6 +168,12 @@ func (p *Packager) output() error { } p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + // Record the migrations that will be ran on the package. + p.cfg.Pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + // Save the transformed config. if err := utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index 8afd90d245..2fdbfdcaf3 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -54,7 +54,6 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) var warning string if migratedComponent, warning = migrateScriptsToActions(migratedComponent); warning != "" { warnings = append(warnings, warning) - build.Migrations = append(build.Migrations, ScriptsToActionsMigrated) } } @@ -66,7 +65,6 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) var warning string if migratedComponent, warning = migrateSetVariableToSetVariables(migratedComponent); warning != "" { warnings = append(warnings, warning) - build.Migrations = append(build.Migrations, PluralizeSetVariable) } } diff --git a/src/test/e2e/03_deprecations_test.go b/src/test/e2e/03_deprecations_test.go index 1ecde29d18..f8db32e6a2 100644 --- a/src/test/e2e/03_deprecations_test.go +++ b/src/test/e2e/03_deprecations_test.go @@ -80,7 +80,7 @@ func TestDeprecatedSetAndPackageVariables(t *testing.T) { // Check that the command still errors out stdOut, stdErr, err := e2e.Zarf("package", "create", testPackageDirPath, outputFlag, "--confirm") require.Error(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "template 'ECHO' must be '--set'") + require.Contains(t, stdErr, "template \"ECHO\" must be '--set'") // Check that the command displays a warning on create stdOut, stdErr, err = e2e.Zarf("package", "create", testPackageDirPath, outputFlag, "--confirm", "--set", "ECHO=Zarf-The-Axolotl") From 8dddf3bd2f69b34a3ca94f6b2146dbf5298fe8ff Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 01:49:23 -0600 Subject: [PATCH 040/172] Go back to original ReloadComponentTemplatesInPackage --- src/pkg/packager/creator/template.go | 25 ++++++++++++++++++------- src/pkg/packager/lint/lint.go | 3 ++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 32dd62f62d..716001c833 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -51,7 +51,7 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti } // update the component templates on the package - if err := reloadComponentTemplatesInPackage(pkg); err != nil { + if err := ReloadComponentTemplatesInPackage(pkg); err != nil { return nil, err } @@ -73,15 +73,26 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti return warnings, nil } -// reloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func reloadComponentTemplatesInPackage(pkg *types.ZarfPackage) error { - for _, component := range pkg.Components { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name +// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads +// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name +func ReloadComponentTemplate(component *types.ZarfComponent) error { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(component, mappings) + if err != nil { + return err + } + return nil +} - if err := utils.ReloadYamlTemplate(&component, mappings); err != nil { +// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for i := range zarfPackage.Components { + if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { return err } } + return nil } diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 08c4964cd4..24fe47c0a3 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -106,7 +107,7 @@ func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts componentMap := map[string]string{} componentMap[types.ZarfComponentName] = node.ZarfComponent.Name - err := utils.ReloadYamlTemplate(&node.ZarfComponent, componentMap) + err := creator.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), From d9188f4daac9614cacea04d6be3b4b15ea5f7984 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 11:01:03 -0600 Subject: [PATCH 041/172] Remove named return parameters from LoadPackageDef --- src/pkg/packager/creator/new.go | 2 +- src/pkg/packager/creator/normal.go | 6 ++++-- src/pkg/packager/creator/skeleton.go | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 933e6e9b5c..c826b3976b 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -11,7 +11,7 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) + LoadPackageDefinition(*types.ZarfPackage, *types.ZarfCreateOptions, *layout.PackagePaths) (*types.ZarfPackage, []string, error) } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 96ad618a1f..8a87f147ef 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -23,8 +23,10 @@ var ( type PackageCreator struct{} // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - pkg, err = setPackageMetadata(pkg, createOpts) +func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, []string, error) { + var warnings []string + + pkg, err := setPackageMetadata(pkg, createOpts) if err != nil { message.Warn(err.Error()) } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 647d756b55..53d2e27fd9 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -19,8 +19,10 @@ var ( type SkeletonCreator struct{} // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, _ *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - pkg, err = setPackageMetadata(pkg, createOpts) +func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, []string, error) { + var warnings []string + + pkg, err := setPackageMetadata(pkg, createOpts) if err != nil { message.Warn(err.Error()) } From 72e218196f91399cb739c0f6ff13d8758863d1f2 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 11:12:59 -0600 Subject: [PATCH 042/172] creator.New() does not need to return an error --- src/pkg/packager/create.go | 5 +---- src/pkg/packager/creator/new.go | 6 +++--- src/pkg/packager/dev.go | 5 +---- src/pkg/packager/publish.go | 5 +---- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index de5abd045f..5c609f4fc4 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -28,10 +28,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c, err := creator.New(p.cfg.CreateOpts) - if err != nil { - return err - } + c := creator.New(p.cfg.CreateOpts) pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index c826b3976b..2c3825800d 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -15,9 +15,9 @@ type Creator interface { } // New returns a new Creator based on the provided create options. -func New(createOpts types.ZarfCreateOptions) (Creator, error) { +func New(createOpts types.ZarfCreateOptions) Creator { if createOpts.IsSkeleton { - return &SkeletonCreator{}, nil + return &SkeletonCreator{} } - return &PackageCreator{}, nil + return &PackageCreator{} } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 5f0c60de0a..705356f9f5 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -34,10 +34,7 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c, err := creator.New(p.cfg.CreateOpts) - if err != nil { - return err - } + c := creator.New(p.cfg.CreateOpts) pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 510993e4c9..5d6a67d598 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -96,10 +96,7 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c, err := creator.New(p.cfg.CreateOpts) - if err != nil { - return err - } + c := creator.New(p.cfg.CreateOpts) pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) if err != nil { return err From f33b7000ce07b900a733cdfc7eb8defab2e7f94f Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 12:53:48 -0600 Subject: [PATCH 043/172] Embed dependencies in Creator implementations instead of passed as parameters --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/new.go | 3 +-- src/pkg/packager/creator/normal.go | 26 ++++++++++++++------------ src/pkg/packager/creator/skeleton.go | 15 ++++++++------- src/pkg/packager/dev.go | 8 +++----- src/pkg/packager/publish.go | 3 +-- 6 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 5c609f4fc4..3b0d2a75c9 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -30,7 +30,7 @@ func (p *Packager) Create() (err error) { c := creator.New(p.cfg.CreateOpts) - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) + pkg, warnings, err := c.LoadPackageDefinition() if err != nil { return err } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 2c3825800d..3c476ffdc6 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -5,13 +5,12 @@ package creator import ( - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ) // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(*types.ZarfPackage, *types.ZarfCreateOptions, *layout.PackagePaths) (*types.ZarfPackage, []string, error) + LoadPackageDefinition() (*types.ZarfPackage, []string, error) } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 8a87f147ef..c0fb47360a 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -20,19 +20,21 @@ var ( ) // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. -type PackageCreator struct{} +type PackageCreator struct { + pkg *types.ZarfPackage + createOpts *types.ZarfCreateOptions + layout *layout.PackagePaths +} // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, []string, error) { - var warnings []string - - pkg, err := setPackageMetadata(pkg, createOpts) +func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(pc.pkg, pc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(pkg, createOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.createOpts) if err != nil { return nil, nil, err } @@ -40,7 +42,7 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOp warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templateWarnings, err := FillActiveTemplate(pkg, createOpts) + templateWarnings, err := FillActiveTemplate(composedPkg, pc.createOpts) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -48,26 +50,26 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOp warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - pkg, err = ProcessExtensions(pkg, createOpts, layout) + extendedPkg, err := ProcessExtensions(composedPkg, pc.createOpts, pc.layout) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. if pkg.Build.Differential { - diffData, err := LoadDifferentialData(&createOpts.DifferentialData) + diffData, err := LoadDifferentialData(&pc.createOpts.DifferentialData) if err != nil { return nil, nil, err } - if createOpts.DifferentialData.DifferentialPackageVersion == pkg.Metadata.Version { + if pc.createOpts.DifferentialData.DifferentialPackageVersion == pkg.Metadata.Version { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - if createOpts.DifferentialData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" { + if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" { return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } - pkg, err = RemoveCopiesFromDifferentialPackage(pkg, diffData) + pkg, err = RemoveCopiesFromDifferentialPackage(extendedPkg, diffData) if err != nil { return nil, nil, err } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 53d2e27fd9..6aeaf9fce2 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -5,7 +5,6 @@ package creator import ( - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -16,19 +15,21 @@ var ( ) // SkeletonCreator provides methods for creating skeleton Zarf packages. -type SkeletonCreator struct{} +type SkeletonCreator struct { + pkg *types.ZarfPackage + createOpts *types.ZarfCreateOptions + // layout *layout.PackagePaths +} // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, []string, error) { - var warnings []string - - pkg, err := setPackageMetadata(pkg, createOpts) +func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(sc.pkg, sc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(pkg, createOpts) + pkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.createOpts) if err != nil { return nil, nil, err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 705356f9f5..b1027d1f97 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -36,14 +36,12 @@ func (p *Packager) DevDeploy() error { c := creator.New(p.cfg.CreateOpts) - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) + pkg, warnings, err := c.LoadPackageDefinition() if err != nil { return err } p.warnings = append(p.warnings, warnings...) - p.cfg.Pkg = *pkg - // Filter out components that are not compatible with this system p.filterComponents() @@ -52,9 +50,9 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.cfg.Pkg.Components = p.getSelectedComponents() + pkg.Components = p.getSelectedComponents() - if err := validate.Run(p.cfg.Pkg); err != nil { + if err := validate.Run(*pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 5d6a67d598..f4282d7db7 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -97,12 +97,11 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } c := creator.New(p.cfg.CreateOpts) - pkg, warnings, err := c.LoadPackageDefinition(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) + _, warnings, err := c.LoadPackageDefinition() if err != nil { return err } p.warnings = append(p.warnings, warnings...) - p.cfg.Pkg = *pkg if err := p.assembleSkeleton(); err != nil { return err } From 53107fa908043b406420f39d441b8f0e4eb966ed Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 16:52:16 -0600 Subject: [PATCH 044/172] Fix nil pointer when instantiating creator --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/differential.go | 8 ++++---- src/pkg/packager/creator/extensions.go | 1 + src/pkg/packager/creator/new.go | 17 +++++++++++++---- src/pkg/packager/creator/normal.go | 9 +++++---- src/pkg/packager/dev.go | 2 +- src/pkg/packager/publish.go | 2 +- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 3b0d2a75c9..5a1b2fdc5c 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -28,7 +28,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg) pkg, warnings, err := c.LoadPackageDefinition() if err != nil { diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index b03867958b..160135efc8 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -22,9 +22,9 @@ import ( "github.com/mholt/archiver/v3" ) -// LoadDifferentialData extracts the Zarf config of a designated 'reference' package, +// loadDifferentialData extracts the Zarf config of a designated 'reference' package, // and creates a list of all images and repos that are in the reference package. -func LoadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { +func loadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) defer os.RemoveAll(tmpDir) @@ -72,8 +72,8 @@ func LoadDifferentialData(diffData *types.DifferentialData) (*types.Differential return diffData, nil } -// RemoveCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func RemoveCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { +// removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package. +func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { // Loop through all of the components to determine if any of them are using already included images or repos componentMap := make(map[int]types.ZarfComponent) for idx, component := range pkg.Components { diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index f185bba8db..1b86f9d7db 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// TODO: @lucasrod16: make this function private once assemble() logic is moved to the creator package. func ProcessExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { components := []types.ZarfComponent{} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 3c476ffdc6..939bf56e6a 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -5,6 +5,7 @@ package creator import ( + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ) @@ -14,9 +15,17 @@ type Creator interface { } // New returns a new Creator based on the provided create options. -func New(createOpts types.ZarfCreateOptions) Creator { - if createOpts.IsSkeleton { - return &SkeletonCreator{} +func New(pkgCfg *types.PackagerConfig) Creator { + if pkgCfg.CreateOpts.IsSkeleton { + return &SkeletonCreator{ + &pkgCfg.Pkg, + &pkgCfg.CreateOpts, + } + } + + return &PackageCreator{ + &pkgCfg.Pkg, + &pkgCfg.CreateOpts, + layout.New(pkgCfg.CreateOpts.BaseDir), } - return &PackageCreator{} } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index c0fb47360a..87b65ed806 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -56,8 +56,8 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni } // If we are creating a differential package, remove duplicate images and repos. - if pkg.Build.Differential { - diffData, err := LoadDifferentialData(&pc.createOpts.DifferentialData) + if pc.pkg.Build.Differential { + diffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) if err != nil { return nil, nil, err } @@ -69,11 +69,12 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } - pkg, err = RemoveCopiesFromDifferentialPackage(extendedPkg, diffData) + diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, diffData) if err != nil { return nil, nil, err } + return diffPkg, nil, nil } - return pkg, warnings, nil + return extendedPkg, warnings, nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b1027d1f97..f407cb3ff7 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -34,7 +34,7 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg) pkg, warnings, err := c.LoadPackageDefinition() if err != nil { diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index f4282d7db7..f085410729 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -96,7 +96,7 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg) _, warnings, err := c.LoadPackageDefinition() if err != nil { return err From 08ca541ac87ccece9bad1fb80eacb3b65d9d22db Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 17:05:46 -0600 Subject: [PATCH 045/172] Fix nil pointer when creating differential packages --- src/pkg/packager/creator/normal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 87b65ed806..9f41ba929a 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -62,10 +62,10 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni return nil, nil, err } - if pc.createOpts.DifferentialData.DifferentialPackageVersion == pkg.Metadata.Version { + if pc.createOpts.DifferentialData.DifferentialPackageVersion == pc.pkg.Metadata.Version { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" { + if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pc.pkg.Metadata.Version == "" { return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") } From 1bfb46702354cc38e52bb9306826f024366c4dab Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 22 Jan 2024 17:33:26 -0600 Subject: [PATCH 046/172] Remove unused componentMap from fillComponentTemplate --- src/pkg/packager/lint/lint.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index 24fe47c0a3..8ac60076d1 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -104,9 +104,6 @@ func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { } func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { - componentMap := map[string]string{} - componentMap[types.ZarfComponentName] = node.ZarfComponent.Name - err := creator.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ From f6091a76e57d622d1a262a9a1cfb452c4ca48780 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 23 Jan 2024 15:08:08 -0600 Subject: [PATCH 047/172] Move actions and variables logic to separate packages --- src/pkg/packager/{ => actions}/actions.go | 33 +++++----- src/pkg/packager/create_stages.go | 9 +-- src/pkg/packager/deploy.go | 12 ++-- src/pkg/packager/dev.go | 3 +- src/pkg/packager/remove.go | 9 +-- src/pkg/packager/variables.go | 76 ----------------------- src/pkg/packager/variables/variables.go | 76 +++++++++++++++++++++++ 7 files changed, 112 insertions(+), 106 deletions(-) rename src/pkg/packager/{ => actions}/actions.go (86%) delete mode 100644 src/pkg/packager/variables.go create mode 100644 src/pkg/packager/variables/variables.go diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions/actions.go similarity index 86% rename from src/pkg/packager/actions.go rename to src/pkg/packager/actions/actions.go index 78340d515f..329db5cf6d 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions/actions.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying zarf packages. -package packager +// Package actions contains functions for running component actions within Zarf packages. +package actions import ( "context" @@ -14,15 +14,16 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) -func (p *Packager) runActions(defaultCfg types.ZarfComponentActionDefaults, actions []types.ZarfComponentAction, valueTemplate *template.Values) error { +func Run(cfg *types.PackagerConfig, defaultCfg types.ZarfComponentActionDefaults, actions []types.ZarfComponentAction, valueTemplate *template.Values) error { for _, a := range actions { - if err := p.runAction(defaultCfg, a, valueTemplate); err != nil { + if err := runAction(cfg, defaultCfg, a, valueTemplate); err != nil { return err } } @@ -30,7 +31,7 @@ func (p *Packager) runActions(defaultCfg types.ZarfComponentActionDefaults, acti } // Run commands that a component has provided. -func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, action types.ZarfComponentAction, valueTemplate *template.Values) error { +func runAction(cfg *types.PackagerConfig, defaultCfg types.ZarfComponentActionDefaults, action types.ZarfComponentAction, valueTemplate *template.Values) error { var ( ctx context.Context cancel context.CancelFunc @@ -87,23 +88,23 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio vars, _ = valueTemplate.GetVariables(types.ZarfComponent{}) } - cfg := actionGetCfg(defaultCfg, action, vars) + actionDefaults := actionGetCfg(defaultCfg, action, vars) - if cmd, err = actionCmdMutation(cmd, cfg.Shell); err != nil { + if cmd, err = actionCmdMutation(cmd, actionDefaults.Shell); err != nil { spinner.Errorf(err, "Error mutating command: %s", cmdEscaped) } - duration := time.Duration(cfg.MaxTotalSeconds) * time.Second + duration := time.Duration(actionDefaults.MaxTotalSeconds) * time.Second timeout := time.After(duration) // Keep trying until the max retries is reached. retryCmd: - for remaining := cfg.MaxRetries + 1; remaining > 0; remaining-- { + for remaining := actionDefaults.MaxRetries + 1; remaining > 0; remaining-- { // Perform the action run. tryCmd := func(ctx context.Context) error { // Try running the command and continue the retry loop if it fails. - if out, err = actionRun(ctx, cfg, cmd, cfg.Shell, spinner); err != nil { + if out, err = actionRun(ctx, actionDefaults, cmd, actionDefaults.Shell, spinner); err != nil { return err } @@ -111,8 +112,8 @@ retryCmd: // If an output variable is defined, set it. for _, v := range action.SetVariables { - p.setVariableInConfig(v.Name, out, v.Sensitive, v.AutoIndent, v.Type) - if err := p.checkVariablePattern(v.Name, v.Pattern); err != nil { + variables.SetVariableInConfig(cfg, v.Name, out, v.Sensitive, v.AutoIndent, v.Type) + if err := variables.CheckVariablePattern(cfg, v.Name, v.Pattern); err != nil { message.WarnErr(err, err.Error()) return err } @@ -130,7 +131,7 @@ retryCmd: } // If no timeout is set, run the command and return or continue retrying. - if cfg.MaxTotalSeconds < 1 { + if actionDefaults.MaxTotalSeconds < 1 { spinner.Updatef("Waiting for \"%s\" (no timeout)", cmdEscaped) if err := tryCmd(context.TODO()); err != nil { continue retryCmd @@ -140,7 +141,7 @@ retryCmd: } // Run the command on repeat until success or timeout. - spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, cfg.MaxTotalSeconds) + spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, actionDefaults.MaxTotalSeconds) select { // On timeout break the loop to abort. case <-timeout: @@ -161,11 +162,11 @@ retryCmd: select { case <-timeout: // If we reached this point, the timeout was reached. - return fmt.Errorf("command \"%s\" timed out after %d seconds", cmdEscaped, cfg.MaxTotalSeconds) + return fmt.Errorf("command \"%s\" timed out after %d seconds", cmdEscaped, actionDefaults.MaxTotalSeconds) default: // If we reached this point, the retry limit was reached. - return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, cfg.MaxRetries) + return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, actionDefaults.MaxRetries) } } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go index 4096bec3b5..97e8ec206b 100644 --- a/src/pkg/packager/create_stages.go +++ b/src/pkg/packager/create_stages.go @@ -23,6 +23,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -38,7 +39,7 @@ func (p *Packager) assemble() error { for idx, component := range p.cfg.Pkg.Components { onCreate := component.Actions.OnCreate onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.OnFailure, nil); err != nil { message.Debugf("unable to run component failure action: %s", err.Error()) } } @@ -47,7 +48,7 @@ func (p *Packager) assemble() error { return fmt.Errorf("unable to add component %q: %w", component.Name, err) } - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { onFailure() return fmt.Errorf("unable to run component success action: %w", err) } @@ -318,7 +319,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent) error onCreate := component.Actions.OnCreate if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { + if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.Before, nil); err != nil { return fmt.Errorf("unable to run component before action: %w", err) } } @@ -548,7 +549,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent) error } if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { + if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.After, nil); err != nil { return fmt.Errorf("unable to run component after action: %w", err) } } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 2a0fc43958..120124b760 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -23,6 +23,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -65,7 +67,7 @@ func (p *Packager) Deploy() (err error) { } // Set variables and prompt if --confirm is not set - if err := p.setVariableMapInConfig(); err != nil { + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { return fmt.Errorf("unable to set the active variables: %w", err) } @@ -159,7 +161,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon onDeploy := component.Actions.OnDeploy onFailure := func() { - if err := p.runActions(onDeploy.Defaults, onDeploy.OnFailure, p.valueTemplate); err != nil { + if err := actions.Run(p.cfg, onDeploy.Defaults, onDeploy.OnFailure, p.valueTemplate); err != nil { message.Debugf("unable to run component failure action: %s", err.Error()) } } @@ -187,7 +189,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon } } - if err := p.runActions(onDeploy.Defaults, onDeploy.OnSuccess, p.valueTemplate); err != nil { + if err := actions.Run(p.cfg, onDeploy.Defaults, onDeploy.OnSuccess, p.valueTemplate); err != nil { onFailure() return deployedComponents, fmt.Errorf("unable to run component success action: %w", err) } @@ -275,7 +277,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if err = p.runActions(onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil { + if err = actions.Run(p.cfg, onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil { return charts, fmt.Errorf("unable to run component before action: %w", err) } @@ -313,7 +315,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if err = p.runActions(onDeploy.Defaults, onDeploy.After, p.valueTemplate); err != nil { + if err = actions.Run(p.cfg, onDeploy.Defaults, onDeploy.After, p.valueTemplate); err != nil { return charts, fmt.Errorf("unable to run component after action: %w", err) } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index f407cb3ff7..d378caa0e2 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -71,7 +72,7 @@ func (p *Packager) DevDeploy() error { message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) // Set variables and prompt if --confirm is not set - if err := p.setVariableMapInConfig(); err != nil { + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { return fmt.Errorf("unable to set the active variables: %w", err) } diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index d552943252..da2d2dcaab 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -127,12 +128,12 @@ func (p *Packager) removeComponent(deployedPackage *types.DeployedPackage, deplo onRemove := c.Actions.OnRemove onFailure := func() { - if err := p.runActions(onRemove.Defaults, onRemove.OnFailure, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.OnFailure, nil); err != nil { message.Debugf("Unable to run the failure action: %s", err) } } - if err := p.runActions(onRemove.Defaults, onRemove.Before, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.Before, nil); err != nil { onFailure() return nil, fmt.Errorf("unable to run the before action for component (%s): %w", c.Name, err) } @@ -161,12 +162,12 @@ func (p *Packager) removeComponent(deployedPackage *types.DeployedPackage, deplo p.updatePackageSecret(*deployedPackage) } - if err := p.runActions(onRemove.Defaults, onRemove.After, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.After, nil); err != nil { onFailure() return deployedPackage, fmt.Errorf("unable to run the after action: %w", err) } - if err := p.runActions(onRemove.Defaults, onRemove.OnSuccess, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.OnSuccess, nil); err != nil { onFailure() return deployedPackage, fmt.Errorf("unable to run the success action: %w", err) } diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go deleted file mode 100644 index 80f1b4c3f1..0000000000 --- a/src/pkg/packager/variables.go +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "fmt" - "regexp" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/types" -) - -// setVariableMapInConfig handles setting the active variables used to template component files. -func (p *Packager) setVariableMapInConfig() error { - for name, value := range p.cfg.PkgOpts.SetVariables { - p.setVariableInConfig(name, value, false, false, "") - } - - for _, variable := range p.cfg.Pkg.Variables { - _, present := p.cfg.SetVariableMap[variable.Name] - - // Variable is present, no need to continue checking - if present { - p.cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive - p.cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent - p.cfg.SetVariableMap[variable.Name].Type = variable.Type - if err := p.checkVariablePattern(variable.Name, variable.Pattern); err != nil { - return err - } - continue - } - - // First set default (may be overridden by prompt) - p.setVariableInConfig(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) - - // Variable is set to prompt the user - if variable.Prompt && !config.CommonOptions.Confirm { - // Prompt the user for the variable - val, err := interactive.PromptVariable(variable) - - if err != nil { - return err - } - - p.setVariableInConfig(variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) - } - - if err := p.checkVariablePattern(variable.Name, variable.Pattern); err != nil { - return err - } - } - - return nil -} - -func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoIndent bool, varType types.VariableType) { - p.cfg.SetVariableMap[name] = &types.ZarfSetVariable{ - Name: name, - Value: value, - Sensitive: sensitive, - AutoIndent: autoIndent, - Type: varType, - } -} - -// checkVariablePattern checks to see if a current variable is set to a value that matches its pattern -func (p *Packager) checkVariablePattern(name, pattern string) error { - if regexp.MustCompile(pattern).MatchString(p.cfg.SetVariableMap[name].Value) { - return nil - } - - return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) -} diff --git a/src/pkg/packager/variables/variables.go b/src/pkg/packager/variables/variables.go new file mode 100644 index 0000000000..3acc33d59f --- /dev/null +++ b/src/pkg/packager/variables/variables.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package variables contains functions for working with variables within Zarf packages. +package variables + +import ( + "fmt" + "regexp" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/types" +) + +// SetVariableMapInConfig handles setting the active variables used to template component files. +func SetVariableMapInConfig(cfg *types.PackagerConfig) error { + for name, value := range cfg.PkgOpts.SetVariables { + SetVariableInConfig(cfg, name, value, false, false, "") + } + + for _, variable := range cfg.Pkg.Variables { + _, present := cfg.SetVariableMap[variable.Name] + + // Variable is present, no need to continue checking + if present { + cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive + cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent + cfg.SetVariableMap[variable.Name].Type = variable.Type + if err := CheckVariablePattern(cfg, variable.Name, variable.Pattern); err != nil { + return err + } + continue + } + + // First set default (may be overridden by prompt) + SetVariableInConfig(cfg, variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) + + // Variable is set to prompt the user + if variable.Prompt && !config.CommonOptions.Confirm { + // Prompt the user for the variable + val, err := interactive.PromptVariable(variable) + + if err != nil { + return err + } + + SetVariableInConfig(cfg, variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) + } + + if err := CheckVariablePattern(cfg, variable.Name, variable.Pattern); err != nil { + return err + } + } + + return nil +} + +func SetVariableInConfig(cfg *types.PackagerConfig, name, value string, sensitive bool, autoIndent bool, varType types.VariableType) { + cfg.SetVariableMap[name] = &types.ZarfSetVariable{ + Name: name, + Value: value, + Sensitive: sensitive, + AutoIndent: autoIndent, + Type: varType, + } +} + +// CheckVariablePattern checks to see if a current variable is set to a value that matches its pattern +func CheckVariablePattern(cfg *types.PackagerConfig, name, pattern string) error { + if regexp.MustCompile(pattern).MatchString(cfg.SetVariableMap[name].Value) { + return nil + } + + return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) +} From 797c6da905aa4262209e77e685d10e2ca95ca8c4 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 23 Jan 2024 18:46:54 -0600 Subject: [PATCH 048/172] Progress --- checksums.txt | 2 + src/cmd/initialize.go | 2 +- src/config/lang/english.go | 2 +- src/pkg/layout/package.go | 83 ++++ src/pkg/packager/common.go | 113 +---- src/pkg/packager/create.go | 6 +- src/pkg/packager/create_stages.go | 587 ----------------------- src/pkg/packager/creator/differential.go | 15 +- src/pkg/packager/creator/new.go | 18 +- src/pkg/packager/creator/normal.go | 457 +++++++++++++++++- src/pkg/packager/creator/skeleton.go | 234 ++++++++- src/pkg/packager/creator/utils.go | 33 ++ src/pkg/packager/dev.go | 4 +- src/pkg/packager/publish.go | 9 +- src/pkg/utils/io.go | 8 +- src/pkg/utils/package.go | 38 +- 16 files changed, 865 insertions(+), 746 deletions(-) create mode 100644 checksums.txt delete mode 100644 src/pkg/packager/create_stages.go diff --git a/checksums.txt b/checksums.txt new file mode 100644 index 0000000000..1a821db48b --- /dev/null +++ b/checksums.txt @@ -0,0 +1,2 @@ +1415cff3b115b190eb400050d5b120f0b4d028291867d2e6f123997d1b729892 sboms.tar +8857db36e454a99d2e97d4135693b5d927e97d1076db124411122eff6e637bd1 components/load-eksctl.tar diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 799896b066..19ac939d13 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -42,7 +42,7 @@ var initCmd = &cobra.Command{ } // Continue running package deploy for all components like any other package - initPackageName := packager.GetInitPackageName("") + initPackageName := utils.GetInitPackageName("") pkgConfig.PkgOpts.PackageSource = initPackageName // Try to use an init-package in the executable directory if none exist in current working directory diff --git a/src/config/lang/english.go b/src/config/lang/english.go index b256d47c75..3de396b707 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -621,7 +621,7 @@ const ( // Package create const ( PkgCreateErrDifferentialSameVersion = "unable to create differential package. Please ensure the differential package version and reference package version are not the same. The package version must be incremented." - PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure the package version is set." + PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure both package versions are set." ) // Package deploy diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index a9dfb36b97..2074e124c4 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -5,15 +5,19 @@ package layout import ( + "encoding/json" + "fmt" "os" "path/filepath" "strings" "github.com/Masterminds/semver/v3" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" + "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -147,6 +151,85 @@ func (pp *PackagePaths) AddSignature(keyPath string) *PackagePaths { return pp } +func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { + pp.AddSignature(signingKeyPath) + + passwordFunc := func(_ bool) ([]byte, error) { + if signingKeyPassword != "" { + return []byte(signingKeyPassword), nil + } + return interactive.PromptSigPassword() + } + _, err := utils.CosignSignBlob(ZarfYAML, Signature, signingKeyPath, passwordFunc) + if err != nil { + return fmt.Errorf("unable to sign the package: %w", err) + } + + return nil +} + +func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSizeMB int) error { + spinner := message.NewProgressSpinner("Writing %s to %s", pp.Base, destinationTarball) + defer spinner.Stop() + + // Make the archive + archiveSrc := []string{pp.Base + string(os.PathSeparator)} + if err := archiver.Archive(archiveSrc, destinationTarball); err != nil { + return fmt.Errorf("unable to create package: %w", err) + } + spinner.Updatef("Wrote %s to %s", pp.Base, destinationTarball) + + fi, err := os.Stat(destinationTarball) + if err != nil { + return fmt.Errorf("unable to read the package archive: %w", err) + } + + // Convert Megabytes to bytes. + chunkSize := maxPackageSizeMB * 1000 * 1000 + + // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. + if maxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { + spinner.Updatef("Package is larger than %dMB, splitting into multiple files", maxPackageSizeMB) + chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) + if err != nil { + return fmt.Errorf("unable to split the package archive into multiple files: %w", err) + } + if len(chunks) > 999 { + return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") + } + + status := fmt.Sprintf("Package split into %d files, original sha256sum is %s", len(chunks)+1, sha256sum) + spinner.Updatef(status) + message.Debug(status) + _ = os.RemoveAll(destinationTarball) + + // Marshal the data into a json file. + jsonData, err := json.Marshal(types.ZarfSplitPackageData{ + Count: len(chunks), + Bytes: fi.Size(), + Sha256Sum: sha256sum, + }) + if err != nil { + return fmt.Errorf("unable to marshal the split package data: %w", err) + } + + // Prepend the json data to the first chunk. + chunks = append([][]byte{jsonData}, chunks...) + + for idx, chunk := range chunks { + path := fmt.Sprintf("%s.part%03d", destinationTarball, idx) + status := fmt.Sprintf("Writing %s", path) + spinner.Updatef(status) + message.Debug(status) + if err := os.WriteFile(path, chunk, 0644); err != nil { + return fmt.Errorf("unable to write the file %s: %w", path, err) + } + } + } + spinner.Successf("Package saved to %q", destinationTarball) + return nil +} + // AddImages sets the default image paths. func (pp *PackagePaths) AddImages() *PackagePaths { pp.Images.Base = filepath.Join(pp.Base, ImagesDir) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index a2e83a1cb0..bf162f7a1d 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -5,7 +5,6 @@ package packager import ( - "encoding/json" "errors" "fmt" "os" @@ -22,10 +21,8 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" @@ -54,7 +51,7 @@ var ( ZarfPackagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) // Find zarf-init packages on the local system - ZarfInitPattern = regexp.MustCompile(GetInitPackageName("") + "$") + ZarfInitPattern = regexp.MustCompile(utils.GetInitPackageName("") + "$") ) // Modifier is a function that modifies the packager. @@ -162,37 +159,6 @@ func (p *Packager) setTempDirectory(path string) error { return nil } -// GetInitPackageName returns the formatted name of the init package. -func GetInitPackageName(arch string) string { - if arch == "" { - // No package has been loaded yet so lookup GetArch() with no package info - arch = config.GetArch() - } - return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) -} - -// GetPackageName returns the formatted name of the package. -func (p *Packager) GetPackageName() string { - if utils.IsInitConfig(p.cfg.Pkg) { - return GetInitPackageName(p.arch) - } - - packageName := p.cfg.Pkg.Metadata.Name - suffix := "tar.zst" - if p.cfg.Pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.cfg.Pkg.Metadata.Architecture) - if p.cfg.Pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) - } else if p.cfg.Pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) - } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) -} - // ClearTempPaths removes the temp directory and any files within it. func (p *Packager) ClearTempPaths() { // Remove the temp directory, but don't throw an error if it fails @@ -316,83 +282,6 @@ func (p *Packager) validateLastNonBreakingVersion() (err error) { return nil } -func (p *Packager) archivePackage(destinationTarball string) error { - spinner := message.NewProgressSpinner("Writing %s to %s", p.layout.Base, destinationTarball) - defer spinner.Stop() - - // Make the archive - archiveSrc := []string{p.layout.Base + string(os.PathSeparator)} - if err := archiver.Archive(archiveSrc, destinationTarball); err != nil { - return fmt.Errorf("unable to create package: %w", err) - } - spinner.Updatef("Wrote %s to %s", p.layout.Base, destinationTarball) - - fi, err := os.Stat(destinationTarball) - if err != nil { - return fmt.Errorf("unable to read the package archive: %w", err) - } - - // Convert Megabytes to bytes. - chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 - - // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { - spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) - chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) - if err != nil { - return fmt.Errorf("unable to split the package archive into multiple files: %w", err) - } - if len(chunks) > 999 { - return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") - } - - status := fmt.Sprintf("Package split into %d files, original sha256sum is %s", len(chunks)+1, sha256sum) - spinner.Updatef(status) - message.Debug(status) - _ = os.RemoveAll(destinationTarball) - - // Marshal the data into a json file. - jsonData, err := json.Marshal(types.ZarfSplitPackageData{ - Count: len(chunks), - Bytes: fi.Size(), - Sha256Sum: sha256sum, - }) - if err != nil { - return fmt.Errorf("unable to marshal the split package data: %w", err) - } - - // Prepend the json data to the first chunk. - chunks = append([][]byte{jsonData}, chunks...) - - for idx, chunk := range chunks { - path := fmt.Sprintf("%s.part%03d", destinationTarball, idx) - status := fmt.Sprintf("Writing %s", path) - spinner.Updatef(status) - message.Debug(status) - if err := os.WriteFile(path, chunk, 0644); err != nil { - return fmt.Errorf("unable to write the file %s: %w", path, err) - } - } - } - spinner.Successf("Package saved to %q", destinationTarball) - return nil -} - -func (p *Packager) signPackage(signingKeyPath, signingKeyPassword string) error { - p.layout = p.layout.AddSignature(signingKeyPath) - passwordFunc := func(_ bool) ([]byte, error) { - if signingKeyPassword != "" { - return []byte(signingKeyPassword), nil - } - return interactive.PromptSigPassword() - } - _, err := utils.CosignSignBlob(p.layout.ZarfYAML, p.layout.Signature, signingKeyPath, passwordFunc) - if err != nil { - return fmt.Errorf("unable to sign the package: %w", err) - } - return nil -} - func (p *Packager) stageSBOMViewFiles() error { if p.layout.SBOMs.IsTarball() { return fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", p.layout.SBOMs.Path) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 5a1b2fdc5c..8f6f241fa1 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -28,7 +28,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg) + c := creator.New(p.cfg, p.layout) pkg, warnings, err := c.LoadPackageDefinition() if err != nil { @@ -46,7 +46,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := p.assemble(); err != nil { + if err := c.Assemble(); err != nil { return err } @@ -55,5 +55,5 @@ func (p *Packager) Create() (err error) { return err } - return p.output() + return c.Output() } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go deleted file mode 100644 index 97e8ec206b..0000000000 --- a/src/pkg/packager/create_stages.go +++ /dev/null @@ -1,587 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "fmt" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/packager/actions" - "github.com/defenseunicorns/zarf/src/pkg/packager/creator" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" -) - -func (p *Packager) assemble() error { - componentSBOMs := map[string]*layout.ComponentSBOM{} - var imageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - if err := p.addComponent(idx, component); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - - if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if !p.cfg.CreateOpts.SkipSBOM { - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - return fmt.Errorf("unable to create component SBOM: %w", err) - } - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - imageList = append(imageList, refInfo) - } - } - - imageList = helpers.Unique(imageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - var err error - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - return nil -} - -func (p *Packager) assembleSkeleton() error { - pkg, err := creator.ProcessExtensions(&p.cfg.Pkg, &p.cfg.CreateOpts, p.layout) - if err != nil { - return nil - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return utils.WriteYaml(p.layout.ZarfYAML, pkg, 0400) -} - -// output assumes it is running from cwd, not the build directory -func (p *Packager) output() error { - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range p.cfg.Pkg.Components { - // Make the component a tar archive - if err := p.layout.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Record the migrations that will be ran on the package. - p.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Save the transformed config. - if err := utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // Sign the config file if a key was provided - if p.cfg.CreateOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - // Create a remote ref + client for the package (if output is OCI) - // then publish the package to the remote. - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) - if err != nil { - return err - } - remote, err := oci.NewOrasRemote(ref) - if err != nil { - return err - } - - err = remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := p.archivePackage(packageName); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := p.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := p.layout.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = p.layout.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if p.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - return nil -} - -func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return nil, err - } - // Create an struct to hold the SBOM information for this component. - componentSBOM := &layout.ComponentSBOM{ - Files: []string{}, - Component: componentPaths, - } - - appendSBOMFiles := func(path string) { - if utils.IsDir(path) { - files, _ := utils.RecursiveFileList(path, nil, false) - componentSBOM.Files = append(componentSBOM.Files, files...) - } else { - componentSBOM.Files = append(componentSBOM.Files, path) - } - } - - for filesIdx, file := range component.Files { - path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - appendSBOMFiles(path) - } - - for dataIdx, data := range component.DataInjections { - path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - - appendSBOMFiles(path) - } - - return componentSBOM, nil -} - -func (p *Packager) addComponent(index int, component types.ZarfComponent) error { - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - - isSkeleton := p.cfg.CreateOpts.IsSkeleton - - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return err - } - - if isSkeleton && component.DeprecatedCosignKeyPath != "" { - dst := filepath.Join(componentPaths.Base, "cosign.pub") - err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) - if err != nil { - return err - } - p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" - } - - // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - if isSkeleton { - component.Actions.OnCreate.Defaults.Dir = "" - resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for idx := range actions { - actions[idx].Dir = nil - } - return actions - } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) - } - - onCreate := component.Actions.OnCreate - if !isSkeleton { - if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.Before, nil); err != nil { - return fmt.Errorf("unable to run component before action: %w", err) - } - } - - // If any helm charts are defined, process them. - for chartIdx, chart := range component.Charts { - - helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) - - if isSkeleton { - if chart.LocalPath != "" { - rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - err := utils.CreatePathAndCopy(chart.LocalPath, dst) - if err != nil { - return err - } - - p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel - } - - for valuesIdx, path := range chart.ValuesFiles { - if helpers.IsURL(path) { - continue - } - - rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) - p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel - - if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) - } - } - } else { - err := helmCfg.PackageChart(componentPaths.Charts) - if err != nil { - return err - } - } - } - - for filesIdx, file := range component.Files { - message.Debugf("Loading %#v", file) - - rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - dst := filepath.Join(componentPaths.Base, rel) - destinationDir := filepath.Dir(dst) - - if helpers.IsURL(file.Source) { - if isSkeleton { - continue - } - - if file.ExtractPath != "" { - - // get the compressedFileName from the source - compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) - if err != nil { - return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) - } - - compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) - - // If the file is an archive, download it to the componentPath.Temp - if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - - err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) - } - - } else { - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - } - - } else { - if file.ExtractPath != "" { - if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) - } - } - - } - - if file.ExtractPath != "" { - // Make sure dst reflects the actual file or directory. - updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) - if updatedExtractedFileOrDir != dst { - if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) - } - } - } - - if isSkeleton { - // Change the source to the new relative source directory (any remote files will have been skipped above) - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel - // Remove the extractPath from a skeleton since it will already extract it - p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" - } - - // Abort packaging on invalid shasum (if one is specified). - if file.Shasum != "" { - if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err - } - } - - if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) - } else { - _ = os.Chmod(dst, 0600) - } - } - - if len(component.DataInjections) > 0 { - spinner := message.NewProgressSpinner("Loading data injections") - defer spinner.Stop() - - for dataIdx, data := range component.DataInjections { - spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) - - rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - dst := filepath.Join(componentPaths.Base, rel) - - if helpers.IsURL(data.Source) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) - } - if isSkeleton { - p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel - } - } - } - spinner.Success() - } - - if len(component.Manifests) > 0 { - // Get the proper count of total manifests to add. - manifestCount := 0 - - for _, manifest := range component.Manifests { - manifestCount += len(manifest.Files) - manifestCount += len(manifest.Kustomizations) - } - - spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) - defer spinner.Stop() - - // Iterate over all manifests. - for manifestIdx, manifest := range component.Manifests { - for fileIdx, path := range manifest.Files { - rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - // Copy manifests without any processing. - spinner.Updatef("Copying manifest %s", path) - if helpers.IsURL(path) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, path, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel - } - } - } - - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) - } - } - if isSkeleton { - // remove kustomizations - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil - } - } - spinner.Success() - } - - // Load all specified git repos. - if len(component.Repos) > 0 && !isSkeleton { - spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) - defer spinner.Stop() - - for _, url := range component.Repos { - // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { - return fmt.Errorf("unable to pull git repo %s: %w", url, err) - } - } - spinner.Success() - } - - if !isSkeleton { - if err := actions.Run(p.cfg, onCreate.Defaults, onCreate.After, nil); err != nil { - return fmt.Errorf("unable to run component after action: %w", err) - } - } - - return nil -} - -// generateChecksum walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksum returns a SHA256 checksum of the checksums.txt file. -func (p *Packager) generatePackageChecksums() (string, error) { - // Loop over the "loaded" files - var checksumsData = []string{} - for rel, abs := range p.layout.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) - } - slices.Sort(checksumsData) - - // Create the checksums file - checksumsFilePath := p.layout.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 160135efc8..999241e364 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -22,13 +22,16 @@ import ( "github.com/mholt/archiver/v3" ) -// loadDifferentialData extracts the Zarf config of a designated 'reference' package, -// and creates a list of all images and repos that are in the reference package. +// loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. func loadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) - // Load the package spec of the package we're using as a 'reference' for the differential build + // Load the reference package. if helpers.IsOCIURL(diffData.DifferentialPackagePath) { remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath) if err != nil { @@ -50,12 +53,12 @@ func loadDifferentialData(diffData *types.DifferentialData) (*types.Differential var differentialZarfConfig types.ZarfPackage if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return nil, fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + return nil, fmt.Errorf("unable to load the differential zarf package spec: %w", err) } - // Generate a map of all the images and repos that are included in the provided package allIncludedImagesMap := map[string]bool{} allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { for _, image := range component.Images { allIncludedImagesMap[image] = true diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 939bf56e6a..e262b41398 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -12,20 +12,14 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { LoadPackageDefinition() (*types.ZarfPackage, []string, error) + Assemble() error + Output() error } // New returns a new Creator based on the provided create options. -func New(pkgCfg *types.PackagerConfig) Creator { - if pkgCfg.CreateOpts.IsSkeleton { - return &SkeletonCreator{ - &pkgCfg.Pkg, - &pkgCfg.CreateOpts, - } - } - - return &PackageCreator{ - &pkgCfg.Pkg, - &pkgCfg.CreateOpts, - layout.New(pkgCfg.CreateOpts.BaseDir), +func New(cfg *types.PackagerConfig, layout *layout.PackagePaths) Creator { + if cfg.CreateOpts.IsSkeleton { + return &SkeletonCreator{cfg, layout} } + return &PackageCreator{cfg, layout} } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 9f41ba929a..bfa7f045f7 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -7,11 +7,29 @@ package creator import ( "errors" "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/images" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" + "github.com/mholt/archiver/v3" ) var ( @@ -21,20 +39,19 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { - pkg *types.ZarfPackage - createOpts *types.ZarfCreateOptions - layout *layout.PackagePaths + cfg *types.PackagerConfig + layout *layout.PackagePaths } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(pc.pkg, pc.createOpts) + configuredPkg, err := setPackageMetadata(&pc.cfg.Pkg, &pc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.createOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &pc.cfg.CreateOpts) if err != nil { return nil, nil, err } @@ -42,7 +59,7 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templateWarnings, err := FillActiveTemplate(composedPkg, pc.createOpts) + templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -50,23 +67,26 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := ProcessExtensions(composedPkg, pc.createOpts, pc.layout) + extendedPkg, err := ProcessExtensions(composedPkg, &pc.cfg.CreateOpts, pc.layout) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. - if pc.pkg.Build.Differential { - diffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) + if pc.cfg.Pkg.Build.Differential { + diffData, err := loadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) if err != nil { return nil, nil, err } - if pc.createOpts.DifferentialData.DifferentialPackageVersion == pc.pkg.Metadata.Version { + versionsMatch := pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pc.cfg.Pkg.Metadata.Version + if versionsMatch { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - if pc.createOpts.DifferentialData.DifferentialPackageVersion == "" || pc.pkg.Metadata.Version == "" { - return nil, nil, fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + + noVersionSet := pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pc.cfg.Pkg.Metadata.Version == "" + if noVersionSet { + return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, diffData) @@ -78,3 +98,416 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni return extendedPkg, warnings, nil } + +func (pc *PackageCreator) Assemble() error { + var imageList []transform.Image + + skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM + componentSBOMs := map[string]*layout.ComponentSBOM{} + + for idx, component := range pc.cfg.Pkg.Components { + onCreate := component.Actions.OnCreate + + onFailure := func() { + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + + if err := pc.addComponent(idx, component); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !skipSBOMFlagUsed { + componentSBOM, err := pc.getFilesToSBOM(component) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + pc.layout = pc.layout.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: pc.layout.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{pc.cfg.Pkg.Metadata.Architecture, pc.cfg.Pkg.Build.Architecture}, + RegistryOverrides: pc.cfg.CreateOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := pc.layout.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if skipSBOMFlagUsed { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + pc.layout = pc.layout.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, pc.layout); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} + +func (pc *PackageCreator) addComponent(index int, component types.ZarfComponent) error { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + componentPaths, err := pc.layout.Components.Create(component) + if err != nil { + return err + } + + onCreate := component.Actions.OnCreate + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.Before, nil); err != nil { + return fmt.Errorf("unable to run component before action: %w", err) + } + + // If any helm charts are defined, process them. + for _, chart := range component.Charts { + helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) + if err := helmCfg.PackageChart(componentPaths.Charts); err != nil { + return err + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if helpers.IsURL(file.Source) { + if file.ExtractPath != "" { + // get the compressedFileName from the source + compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) + if err != nil { + return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) + } + + compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) + + // If the file is an archive, download it to the componentPath.Temp + if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + + err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) + } + } else { + if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + } + } else { + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + } + + if file.ExtractPath != "" { + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, 0700) + } else { + _ = os.Chmod(dst, 0600) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if helpers.IsURL(data.Source) { + if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + } + } + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for _, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + if helpers.IsURL(path) { + if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, path, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + } + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + } + } + spinner.Success() + } + + // Load all specified git repos. + if len(component.Repos) > 0 { + spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) + defer spinner.Stop() + + for _, url := range component.Repos { + // Pull all the references if there is no `@` in the string. + gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) + if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { + return fmt.Errorf("unable to pull git repo %s: %w", url, err) + } + } + spinner.Success() + } + + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.After, nil); err != nil { + return fmt.Errorf("unable to run component after action: %w", err) + } + + return nil +} + +// Output assumes it is running from cwd, not the build directory +func (pc *PackageCreator) Output() error { + // Process the component directories into compressed tarballs + // NOTE: This is purposefully being done after the SBOM cataloging + for _, component := range pc.cfg.Pkg.Components { + // Make the component a tar archive + if err := pc.layout.Components.Archive(component, true); err != nil { + return fmt.Errorf("unable to archive component: %s", err.Error()) + } + } + + // Calculate all the checksums + checksumChecksum, err := pc.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + // Record the migrations that will be ran on the package. + pc.cfg.Pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Save the transformed config. + if err := utils.WriteYaml(pc.layout.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the config file if a key was provided + if pc.cfg.CreateOpts.SigningKeyPath != "" { + if err := pc.layout.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { + return err + } + } + + // Create a remote ref + client for the package (if output is OCI) + // then publish the package to the remote. + if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { + ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) + if err != nil { + return err + } + remote, err := oci.NewOrasRemote(ref) + if err != nil { + return err + } + + err = remote.PublishPackage(&pc.cfg.Pkg, pc.layout, config.CommonOptions.OCIConcurrency) + if err != nil { + return fmt.Errorf("unable to publish package: %w", err) + } + message.HorizontalRule() + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + } else { + // Use the output path if the user specified it. + packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) + + // Try to remove the package if it already exists. + _ = os.Remove(packageName) + + // Create the package tarball. + if err := pc.layout.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { + return fmt.Errorf("unable to archive package: %w", err) + } + } + + // Output the SBOM files into a directory if specified. + if pc.cfg.CreateOpts.ViewSBOM || pc.cfg.CreateOpts.SBOMOutputDir != "" { + outputSBOM := pc.cfg.CreateOpts.SBOMOutputDir + var sbomDir string + if err := pc.layout.SBOMs.Unarchive(); err != nil { + return fmt.Errorf("unable to unarchive SBOMs: %w", err) + } + sbomDir = pc.layout.SBOMs.Path + + if outputSBOM != "" { + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) + if err != nil { + return err + } + sbomDir = out + } + + if pc.cfg.CreateOpts.ViewSBOM { + sbom.ViewSBOMFiles(sbomDir) + } + } + return nil +} + +func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { + componentPaths, err := pc.layout.Components.Create(component) + if err != nil { + return nil, err + } + // Create an struct to hold the SBOM information for this component. + componentSBOM := &layout.ComponentSBOM{ + Files: []string{}, + Component: componentPaths, + } + + appendSBOMFiles := func(path string) { + if utils.IsDir(path) { + files, _ := utils.RecursiveFileList(path, nil, false) + componentSBOM.Files = append(componentSBOM.Files, files...) + } else { + componentSBOM.Files = append(componentSBOM.Files, path) + } + } + + for filesIdx, file := range component.Files { + path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + appendSBOMFiles(path) + } + + for dataIdx, data := range component.DataInjections { + path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + + appendSBOMFiles(path) + } + + return componentSBOM, nil +} + +func (pc *PackageCreator) generatePackageChecksums() (string, error) { + return generateChecksums(pc.layout) +} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 6aeaf9fce2..5ae2cbe299 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -5,8 +5,21 @@ package creator import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" + "github.com/mholt/archiver/v3" ) var ( @@ -16,20 +29,19 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - pkg *types.ZarfPackage - createOpts *types.ZarfCreateOptions - // layout *layout.PackagePaths + cfg *types.PackagerConfig + layout *layout.PackagePaths } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(sc.pkg, sc.createOpts) + configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.createOpts) + pkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) if err != nil { return nil, nil, err } @@ -37,3 +49,215 @@ func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warn return pkg, warnings, nil } + +// TODO: print warnings somewhere else in the skeleton create flow. +func (sc *SkeletonCreator) Assemble() error { + pkg, err := ProcessExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, sc.layout) + if err != nil { + return nil + } + + for idx, component := range pkg.Components { + if err := sc.addComponent(idx, component); err != nil { + return err + } + + if err := sc.layout.Components.Archive(component, false); err != nil { + return err + } + } + + checksumChecksum, err := sc.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + sc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return nil +} + +func (sc *SkeletonCreator) Output() error { + return utils.WriteYaml(sc.layout.ZarfYAML, sc.cfg.Pkg, 0400) +} + +func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent) error { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + componentPaths, err := sc.layout.Components.Create(component) + if err != nil { + return err + } + + if component.DeprecatedCosignKeyPath != "" { + dst := filepath.Join(componentPaths.Base, "cosign.pub") + err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) + if err != nil { + return err + } + sc.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" + } + + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + component.Actions.OnCreate.Defaults.Dir = "" + + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + + component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + + // If any helm charts are defined, process them. + for chartIdx, chart := range component.Charts { + + if chart.LocalPath != "" { + rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + err := utils.CreatePathAndCopy(chart.LocalPath, dst) + if err != nil { + return err + } + + sc.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel + } + + for valuesIdx, path := range chart.ValuesFiles { + if helpers.IsURL(path) { + continue + } + + rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) + sc.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel + + if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { + return fmt.Errorf("unable to copy chart values file %s: %w", path, err) + } + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + + // Change the source to the new relative source directory (any remote files will have been skipped above) + sc.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + + // Remove the extractPath from a skeleton since it will already extract it + sc.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, 0700) + } else { + _ = os.Chmod(dst, 0600) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + + sc.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel + } + + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for manifestIdx, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + + sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + + sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) + } + // remove kustomizations + sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil + } + + spinner.Success() + } + + return nil +} + +func (sc *SkeletonCreator) generatePackageChecksums() (string, error) { + return generateChecksums(sc.layout) +} diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index cb4dfdd5f7..bc11fb8e43 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -5,11 +5,15 @@ package creator import ( + "fmt" "os" "runtime" + "slices" + "strings" "time" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -69,3 +73,32 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti return pkg, nil } + +// generateChecksums walks through all of the files starting at the base path and generates a checksum file. +// Each file within the basePath represents a layer within the Zarf package. +// generateChecksums returns a SHA256 checksum of the checksums.txt file. +func generateChecksums(layout *layout.PackagePaths) (string, error) { + // Loop over the "loaded" files + var checksumsData = []string{} + for rel, abs := range layout.Files() { + if rel == layout.ZarfYAML || rel == layout.Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) + } + slices.Sort(checksumsData) + + // Create the checksums file + checksumsFilePath := layout.Checksums + if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(checksumsFilePath) +} diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index d378caa0e2..b650b70350 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -35,7 +35,7 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg) + c := creator.New(p.cfg, p.layout) pkg, warnings, err := c.LoadPackageDefinition() if err != nil { @@ -65,7 +65,7 @@ func (p *Packager) DevDeploy() error { } } - if err := p.assemble(); err != nil { + if err := c.Assemble(); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index f085410729..c167cb003a 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -96,13 +96,16 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg) + c := creator.New(p.cfg, p.layout) _, warnings, err := c.LoadPackageDefinition() if err != nil { return err } p.warnings = append(p.warnings, warnings...) - if err := p.assembleSkeleton(); err != nil { + if err := c.Assemble(); err != nil { + return err + } + if err := c.Output(); err != nil { return err } } else { @@ -127,7 +130,7 @@ func (p *Packager) Publish() (err error) { // Sign the package if a key has been provided if p.cfg.PublishOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { + if err := p.layout.SignPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { return err } } diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index c02ce6b55e..a2bab43048 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -46,9 +46,15 @@ func MakeTempDir(basePath string) (string, error) { return "", err } } + tmp, err := os.MkdirTemp(basePath, tmpPathPrefix) + if err != nil { + return "", err + } + message.Debug("Using temporary directory:", tmp) - return tmp, err + + return tmp, nil } // VerifyBinary returns true if binary is available. diff --git a/src/pkg/utils/package.go b/src/pkg/utils/package.go index b6e3583985..982c4e5793 100644 --- a/src/pkg/utils/package.go +++ b/src/pkg/utils/package.go @@ -4,9 +4,45 @@ // Package utils provides generic utility functions. package utils -import "github.com/defenseunicorns/zarf/src/types" +import ( + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/types" +) // IsInitConfig returns whether the provided Zarf package is an init config. func IsInitConfig(pkg types.ZarfPackage) bool { return pkg.Kind == types.ZarfInitConfig } + +// GetInitPackageName returns the formatted name of the init package. +func GetInitPackageName(arch string) string { + if arch == "" { + // No package has been loaded yet so lookup GetArch() with no package info + arch = config.GetArch() + } + return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) +} + +// GetPackageName returns the formatted name of the package. +func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { + if IsInitConfig(pkg) { + return GetInitPackageName(pkg.Metadata.Architecture) + } + + packageName := pkg.Metadata.Name + suffix := "tar.zst" + if pkg.Metadata.Uncompressed { + suffix = "tar" + } + + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) + if pkg.Build.Differential { + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) + } else if pkg.Metadata.Version != "" { + packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) + } + + return fmt.Sprintf("%s.%s", packageFileName, suffix) +} From bc9860ff3b87b32f072dc088ba00ed15ccdeb315 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 23 Jan 2024 18:52:44 -0600 Subject: [PATCH 049/172] Merge main --- src/cmd/initialize.go | 2 +- src/cmd/tools/zarf.go | 2 +- src/internal/packager/images/pull.go | 6 ----- src/pkg/oci/common.go | 26 +++++++++------------- src/pkg/packager/composer/oci.go | 2 +- src/pkg/packager/creator/differential.go | 2 +- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/publish.go | 4 ++-- src/pkg/packager/sources/new.go | 2 +- src/pkg/utils/concurrency.go | 14 ------------ src/test/e2e/50_oci_publish_deploy_test.go | 4 ++-- 11 files changed, 20 insertions(+), 46 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 19ac939d13..c5210fc512 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -138,7 +138,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) + remote, err := oci.NewOrasRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index d835674ed6..27cc36da6a 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -183,7 +183,7 @@ var downloadInitCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { url := oci.GetInitPackageURL(config.CLIVersion) - remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) + remote, err := oci.NewOrasRemote(url, oci.PlatformForArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index ed115a1d65..160c0d506a 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -78,8 +78,6 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { // Create a closure so that we can pass the src into the goroutine refInfo := refInfo go func() { - // Make sure to call Done() on the WaitGroup when the goroutine finishes - defer metadataImageConcurrency.WaitGroupDone() if metadataImageConcurrency.IsDone() { return @@ -202,7 +200,6 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { // and https://github.com/google/go-containerregistry/blob/v0.15.2/pkg/v1/layout/write.go#L198-L262 // with modifications. This allows us to dedupe layers for all images and write them concurrently. go func() { - defer layerWritingConcurrency.WaitGroupDone() digest, err := layer.Digest() if errors.Is(err, stream.ErrNotComputed) { // Allow digest errors, since streams may not have calculated the hash @@ -347,9 +344,6 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { // Create a closure so that we can pass the refInfo and img into the goroutine refInfo, img := refInfo, img go func() { - // Make sure to call Done() on the WaitGroup when the goroutine finishes - defer imageSavingConcurrency.WaitGroupDone() - // Save the image via crane err := cranePath.WriteImage(img) diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index c228ec0675..873d5e2846 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -75,38 +75,32 @@ func WithInsecureSkipVerify(insecure bool) Modifier { } } -// WithTargetPlatform sets the target platform for the remote -func WithTargetPlatform(platform *ocispec.Platform) Modifier { - return func(o *OrasRemote) { - o.targetPlatform = platform - } -} - -// WithSkeletonArch sets the target architecture for the remote to skeleton -func WithSkeletonArch() Modifier { - return WithTargetPlatform(&ocispec.Platform{ +// PlatformForSkeleton sets the target architecture for the remote to skeleton +func PlatformForSkeleton() ocispec.Platform { + return ocispec.Platform{ OS: MultiOS, Architecture: SkeletonArch, - }) + } } -// WithArch sets the target architecture for the remote -func WithArch(arch string) Modifier { - return WithTargetPlatform(&ocispec.Platform{ +// PlatformForArch sets the target architecture for the remote +func PlatformForArch(arch string) ocispec.Platform { + return ocispec.Platform{ OS: MultiOS, Architecture: arch, - }) + } } // NewOrasRemote returns an oras remote repository client and context for the given url. // // Registry auth is handled by the Docker CLI's credential store and checked before returning the client -func NewOrasRemote(url string, mods ...Modifier) (*OrasRemote, error) { +func NewOrasRemote(url string, platform ocispec.Platform, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) } o := &OrasRemote{} + o.targetPlatform = &platform if err := o.setRepository(ref); err != nil { return nil, err diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 0477d04b97..56418eadfe 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -27,7 +27,7 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { return ic.remote, nil } var err error - ic.remote, err = oci.NewOrasRemote(url, oci.WithSkeletonArch()) + ic.remote, err = oci.NewOrasRemote(url, oci.PlatformForSkeleton()) if err != nil { return nil, err } diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 999241e364..3fa65451e2 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -33,7 +33,7 @@ func loadDifferentialData(diffData *types.DifferentialData) (*types.Differential // Load the reference package. if helpers.IsOCIURL(diffData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath) + remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) if err != nil { return nil, err } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index bfa7f045f7..6bfc8cc0ae 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -419,7 +419,7 @@ func (pc *PackageCreator) Output() error { if err != nil { return err } - remote, err := oci.NewOrasRemote(ref) + remote, err := oci.NewOrasRemote(ref, oci.PlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index c167cb003a..4a6c444792 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -39,7 +39,7 @@ func (p *Packager) Publish() (err error) { p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName arch := config.GetArch() - dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.WithArch(arch)) + dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.PlatformForArch(arch)) if err != nil { return err } @@ -123,7 +123,7 @@ func (p *Packager) Publish() (err error) { return err } - remote, err := oci.NewOrasRemote(ref) + remote, err := oci.NewOrasRemote(ref, oci.PlatformForArch(config.GetArch())) if err != nil { return err } diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index 9cd8c07c1a..2ac083f129 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -67,7 +67,7 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } arch := config.GetArch() - remote, err := oci.NewOrasRemote(pkgSrc, oci.WithArch(arch)) + remote, err := oci.NewOrasRemote(pkgSrc, oci.PlatformForArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/utils/concurrency.go b/src/pkg/utils/concurrency.go index 66fba76ff0..135d5caacc 100644 --- a/src/pkg/utils/concurrency.go +++ b/src/pkg/utils/concurrency.go @@ -7,7 +7,6 @@ package utils import ( "context" - "sync" ) // ConcurrencyTools is a struct that contains channels and a context for use in concurrent routines @@ -16,7 +15,6 @@ type ConcurrencyTools[P any, E any] struct { ErrorChan chan E context context.Context Cancel context.CancelFunc - waitGroup *sync.WaitGroup routineCount int } @@ -30,16 +28,11 @@ func NewConcurrencyTools[P any, E any](length int) *ConcurrencyTools[P, E] { errorChan := make(chan E, length) - waitGroup := sync.WaitGroup{} - - waitGroup.Add(length) - concurrencyTools := ConcurrencyTools[P, E]{ ProgressChan: progressChan, ErrorChan: errorChan, context: ctx, Cancel: cancel, - waitGroup: &waitGroup, routineCount: length, } @@ -57,11 +50,6 @@ func (ct *ConcurrencyTools[P, E]) IsDone() bool { } } -// WaitGroupDone decrements the internal WaitGroup counter by one. -func (ct *ConcurrencyTools[P, E]) WaitGroupDone() { - ct.waitGroup.Done() -} - // WaitWithProgress waits for all routines to finish // // onProgress is a callback function that is called when a routine sends a progress update @@ -78,7 +66,6 @@ func (ct *ConcurrencyTools[P, E]) WaitWithProgress(onProgress func(P, int), onEr onProgress(progress, i) } } - ct.waitGroup.Wait() return nil } @@ -95,6 +82,5 @@ func (ct *ConcurrencyTools[P, E]) WaitWithoutProgress(onError func(E) error) err case <-ct.ProgressChan: } } - ct.waitGroup.Wait() return nil } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 55a038715b..d4011ee951 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -128,10 +128,10 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - src, err := oci.NewOrasRemote(ref, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) + src, err := oci.NewOrasRemote(ref, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) - dst, err := oci.NewOrasRemote(dstRef, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) + dst, err := oci.NewOrasRemote(dstRef, oci.PlatformForArch(e2e.Arch), oci.WithPlainHTTP(true)) suite.NoError(err) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) From 3381c3d9c1fe5175634f38f613929785f6de49e6 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 10:43:05 -0600 Subject: [PATCH 050/172] Fix generateChecksums --- checksums.txt | 2 -- src/pkg/packager/create.go | 8 ++--- src/pkg/packager/creator/new.go | 12 +++---- src/pkg/packager/creator/normal.go | 53 +++++++++++++--------------- src/pkg/packager/creator/skeleton.go | 27 ++++++-------- src/pkg/packager/creator/utils.go | 6 ++-- src/pkg/packager/dev.go | 6 ++-- src/pkg/packager/publish.go | 8 ++--- src/test/e2e/00_use_cli_test.go | 2 +- 9 files changed, 56 insertions(+), 68 deletions(-) delete mode 100644 checksums.txt diff --git a/checksums.txt b/checksums.txt deleted file mode 100644 index 1a821db48b..0000000000 --- a/checksums.txt +++ /dev/null @@ -1,2 +0,0 @@ -1415cff3b115b190eb400050d5b120f0b4d028291867d2e6f123997d1b729892 sboms.tar -8857db36e454a99d2e97d4135693b5d927e97d1076db124411122eff6e637bd1 components/load-eksctl.tar diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 8f6f241fa1..0c45ee4f93 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -28,9 +28,9 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg, p.layout) + c := creator.New(p.cfg) - pkg, warnings, err := c.LoadPackageDefinition() + pkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -46,7 +46,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } @@ -55,5 +55,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output() + return c.Output(p.layout) } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index e262b41398..ad0e3936a8 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -11,15 +11,15 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition() (*types.ZarfPackage, []string, error) - Assemble() error - Output() error + LoadPackageDefinition(dst *layout.PackagePaths) (*types.ZarfPackage, []string, error) + Assemble(dst *layout.PackagePaths) error + Output(dst *layout.PackagePaths) error } // New returns a new Creator based on the provided create options. -func New(cfg *types.PackagerConfig, layout *layout.PackagePaths) Creator { +func New(cfg *types.PackagerConfig) Creator { if cfg.CreateOpts.IsSkeleton { - return &SkeletonCreator{cfg, layout} + return &SkeletonCreator{cfg} } - return &PackageCreator{cfg, layout} + return &PackageCreator{cfg} } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 6bfc8cc0ae..64e2947430 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -39,12 +39,11 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { - cfg *types.PackagerConfig - layout *layout.PackagePaths + cfg *types.PackagerConfig } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { configuredPkg, err := setPackageMetadata(&pc.cfg.Pkg, &pc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) @@ -67,7 +66,7 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := ProcessExtensions(composedPkg, &pc.cfg.CreateOpts, pc.layout) + extendedPkg, err := ProcessExtensions(composedPkg, &pc.cfg.CreateOpts, dst) if err != nil { return nil, nil, err } @@ -99,7 +98,7 @@ func (pc *PackageCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warni return extendedPkg, warnings, nil } -func (pc *PackageCreator) Assemble() error { +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { var imageList []transform.Image skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM @@ -114,7 +113,7 @@ func (pc *PackageCreator) Assemble() error { } } - if err := pc.addComponent(idx, component); err != nil { + if err := pc.addComponent(idx, component, dst); err != nil { onFailure() return fmt.Errorf("unable to add component %q: %w", component.Name, err) } @@ -125,7 +124,7 @@ func (pc *PackageCreator) Assemble() error { } if !skipSBOMFlagUsed { - componentSBOM, err := pc.getFilesToSBOM(component) + componentSBOM, err := pc.getFilesToSBOM(component, dst) if err != nil { return fmt.Errorf("unable to create component SBOM: %w", err) } @@ -151,14 +150,14 @@ func (pc *PackageCreator) Assemble() error { if len(imageList) > 0 { message.HeaderInfof("📦 PACKAGE IMAGES") - pc.layout = pc.layout.AddImages() + dst.AddImages() var pulled []images.ImgInfo var err error doPull := func() error { imgConfig := images.ImageConfig{ - ImagesPath: pc.layout.Images.Base, + ImagesPath: dst.Images.Base, ImageList: imageList, Insecure: config.CommonOptions.Insecure, Architectures: []string{pc.cfg.Pkg.Metadata.Architecture, pc.cfg.Pkg.Build.Architecture}, @@ -174,7 +173,7 @@ func (pc *PackageCreator) Assemble() error { } for _, imgInfo := range pulled { - if err := pc.layout.Images.AddV1Image(imgInfo.Img); err != nil { + if err := dst.Images.AddV1Image(imgInfo.Img); err != nil { return err } if imgInfo.HasImageLayers { @@ -187,8 +186,8 @@ func (pc *PackageCreator) Assemble() error { if skipSBOMFlagUsed { message.Debug("Skipping image SBOM processing per --skip-sbom flag") } else { - pc.layout = pc.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, pc.layout); err != nil { + dst.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, dst); err != nil { return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) } } @@ -196,10 +195,10 @@ func (pc *PackageCreator) Assemble() error { return nil } -func (pc *PackageCreator) addComponent(index int, component types.ZarfComponent) error { +func (pc *PackageCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - componentPaths, err := pc.layout.Components.Create(component) + componentPaths, err := dst.Components.Create(component) if err != nil { return err } @@ -377,18 +376,18 @@ func (pc *PackageCreator) addComponent(index int, component types.ZarfComponent) } // Output assumes it is running from cwd, not the build directory -func (pc *PackageCreator) Output() error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range pc.cfg.Pkg.Components { // Make the component a tar archive - if err := pc.layout.Components.Archive(component, true); err != nil { + if err := dst.Components.Archive(component, true); err != nil { return fmt.Errorf("unable to archive component: %s", err.Error()) } } // Calculate all the checksums - checksumChecksum, err := pc.generatePackageChecksums() + checksumChecksum, err := generateChecksums(dst) if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } @@ -401,13 +400,13 @@ func (pc *PackageCreator) Output() error { } // Save the transformed config. - if err := utils.WriteYaml(pc.layout.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } // Sign the config file if a key was provided if pc.cfg.CreateOpts.SigningKeyPath != "" { - if err := pc.layout.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { + if err := dst.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { return err } } @@ -424,7 +423,7 @@ func (pc *PackageCreator) Output() error { return err } - err = remote.PublishPackage(&pc.cfg.Pkg, pc.layout, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -445,7 +444,7 @@ func (pc *PackageCreator) Output() error { _ = os.Remove(packageName) // Create the package tarball. - if err := pc.layout.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { + if err := dst.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { return fmt.Errorf("unable to archive package: %w", err) } } @@ -454,10 +453,10 @@ func (pc *PackageCreator) Output() error { if pc.cfg.CreateOpts.ViewSBOM || pc.cfg.CreateOpts.SBOMOutputDir != "" { outputSBOM := pc.cfg.CreateOpts.SBOMOutputDir var sbomDir string - if err := pc.layout.SBOMs.Unarchive(); err != nil { + if err := dst.SBOMs.Unarchive(); err != nil { return fmt.Errorf("unable to unarchive SBOMs: %w", err) } - sbomDir = pc.layout.SBOMs.Path + sbomDir = dst.SBOMs.Path if outputSBOM != "" { out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) @@ -474,8 +473,8 @@ func (pc *PackageCreator) Output() error { return nil } -func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := pc.layout.Components.Create(component) +func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { + componentPaths, err := dst.Components.Create(component) if err != nil { return nil, err } @@ -507,7 +506,3 @@ func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent) (*layout return componentSBOM, nil } - -func (pc *PackageCreator) generatePackageChecksums() (string, error) { - return generateChecksums(pc.layout) -} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 5ae2cbe299..86226186c3 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -29,12 +29,11 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - cfg *types.PackagerConfig - layout *layout.PackagePaths + cfg *types.PackagerConfig } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warnings []string, err error) { +func (sc *SkeletonCreator) LoadPackageDefinition(_ *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) @@ -51,23 +50,23 @@ func (sc *SkeletonCreator) LoadPackageDefinition() (pkg *types.ZarfPackage, warn } // TODO: print warnings somewhere else in the skeleton create flow. -func (sc *SkeletonCreator) Assemble() error { - pkg, err := ProcessExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, sc.layout) +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { + pkg, err := ProcessExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, dst) if err != nil { return nil } for idx, component := range pkg.Components { - if err := sc.addComponent(idx, component); err != nil { + if err := sc.addComponent(idx, component, dst); err != nil { return err } - if err := sc.layout.Components.Archive(component, false); err != nil { + if err := dst.Components.Archive(component, false); err != nil { return err } } - checksumChecksum, err := sc.generatePackageChecksums() + checksumChecksum, err := generateChecksums(dst) if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } @@ -76,14 +75,14 @@ func (sc *SkeletonCreator) Assemble() error { return nil } -func (sc *SkeletonCreator) Output() error { - return utils.WriteYaml(sc.layout.ZarfYAML, sc.cfg.Pkg, 0400) +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { + return utils.WriteYaml(dst.ZarfYAML, sc.cfg.Pkg, 0400) } -func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent) error { +func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - componentPaths, err := sc.layout.Components.Create(component) + componentPaths, err := dst.Components.Create(component) if err != nil { return err } @@ -257,7 +256,3 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent return nil } - -func (sc *SkeletonCreator) generatePackageChecksums() (string, error) { - return generateChecksums(sc.layout) -} diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index bc11fb8e43..b7f5a2d99e 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -77,10 +77,10 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti // generateChecksums walks through all of the files starting at the base path and generates a checksum file. // Each file within the basePath represents a layer within the Zarf package. // generateChecksums returns a SHA256 checksum of the checksums.txt file. -func generateChecksums(layout *layout.PackagePaths) (string, error) { +func generateChecksums(dst *layout.PackagePaths) (string, error) { // Loop over the "loaded" files var checksumsData = []string{} - for rel, abs := range layout.Files() { + for rel, abs := range dst.Files() { if rel == layout.ZarfYAML || rel == layout.Checksums { continue } @@ -94,7 +94,7 @@ func generateChecksums(layout *layout.PackagePaths) (string, error) { slices.Sort(checksumsData) // Create the checksums file - checksumsFilePath := layout.Checksums + checksumsFilePath := dst.Checksums if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { return "", err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b650b70350..b63cfbef16 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -35,9 +35,9 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg, p.layout) + c := creator.New(p.cfg) - pkg, warnings, err := c.LoadPackageDefinition() + pkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -65,7 +65,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 4a6c444792..bc8241218d 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -96,16 +96,16 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg, p.layout) - _, warnings, err := c.LoadPackageDefinition() + c := creator.New(p.cfg) + _, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } p.warnings = append(p.warnings, warnings...) - if err := c.Assemble(); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } - if err := c.Output(); err != nil { + if err := c.Output(p.layout); err != nil { return err } } else { diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index 2eaf6a6ed8..9b1fec2b98 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -163,7 +163,7 @@ func TestUseCLI(t *testing.T) { require.FileExists(t, "binaries/eksctl_Darwin_arm64") require.FileExists(t, "binaries/eksctl_Linux_x86_64") - e2e.CleanFiles("binaries/eksctl_Darwin_x86_64", "binaries/eksctl_Darwin_arm64", "binaries/eksctl_Linux_x86_64", path) + e2e.CleanFiles("binaries/eksctl_Darwin_x86_64", "binaries/eksctl_Darwin_arm64", "binaries/eksctl_Linux_x86_64", "eks.yaml", path) }) t.Run("zarf package create with tmpdir and cache", func(t *testing.T) { From 9118917fc94c7fbd12e49d6e2d4b14e66ec269db Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 11:27:09 -0600 Subject: [PATCH 051/172] Move generateChecksums to its own file --- src/pkg/packager/creator/checksums.go | 43 +++++++++++++++++++++++++++ src/pkg/packager/creator/utils.go | 33 -------------------- 2 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 src/pkg/packager/creator/checksums.go diff --git a/src/pkg/packager/creator/checksums.go b/src/pkg/packager/creator/checksums.go new file mode 100644 index 0000000000..ddfde66486 --- /dev/null +++ b/src/pkg/packager/creator/checksums.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "slices" + "strings" + + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/utils" +) + +// generateChecksums walks through all of the files starting at the base path and generates a checksum file. +// Each file within the basePath represents a layer within the Zarf package. +// generateChecksums returns a SHA256 checksum of the checksums.txt file. +func generateChecksums(dst *layout.PackagePaths) (string, error) { + // Loop over the "loaded" files + var checksumsData = []string{} + for rel, abs := range dst.Files() { + if rel == layout.ZarfYAML || rel == layout.Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) + } + slices.Sort(checksumsData) + + // Create the checksums file + checksumsFilePath := dst.Checksums + if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(checksumsFilePath) +} diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index b7f5a2d99e..cb4dfdd5f7 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -5,15 +5,11 @@ package creator import ( - "fmt" "os" "runtime" - "slices" - "strings" "time" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -73,32 +69,3 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti return pkg, nil } - -// generateChecksums walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksums returns a SHA256 checksum of the checksums.txt file. -func generateChecksums(dst *layout.PackagePaths) (string, error) { - // Loop over the "loaded" files - var checksumsData = []string{} - for rel, abs := range dst.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) - } - slices.Sort(checksumsData) - - // Create the checksums file - checksumsFilePath := dst.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} From a9a6846888a9a82c25cc975706ea7cc0567c79fd Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 12:01:28 -0600 Subject: [PATCH 052/172] Fix package signing --- src/pkg/layout/package.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 2074e124c4..875453dada 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -160,7 +160,7 @@ func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) e } return interactive.PromptSigPassword() } - _, err := utils.CosignSignBlob(ZarfYAML, Signature, signingKeyPath, passwordFunc) + _, err := utils.CosignSignBlob(pp.ZarfYAML, pp.Signature, signingKeyPath, passwordFunc) if err != nil { return fmt.Errorf("unable to sign the package: %w", err) } From 54425bbf12ec131ce89b84ed48334a968dd78d4c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 12:05:32 -0600 Subject: [PATCH 053/172] Remove unused index parameter from pc.addComponent --- src/pkg/packager/creator/normal.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 64e2947430..517caa6339 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -104,7 +104,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM componentSBOMs := map[string]*layout.ComponentSBOM{} - for idx, component := range pc.cfg.Pkg.Components { + for _, component := range pc.cfg.Pkg.Components { onCreate := component.Actions.OnCreate onFailure := func() { @@ -113,7 +113,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { } } - if err := pc.addComponent(idx, component, dst); err != nil { + if err := pc.addComponent(component, dst); err != nil { onFailure() return fmt.Errorf("unable to add component %q: %w", component.Name, err) } @@ -195,7 +195,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { return nil } -func (pc *PackageCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { +func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) componentPaths, err := dst.Components.Create(component) From ab5abc5e79d60dd48de00c0515c10b198754d907 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 12:08:25 -0600 Subject: [PATCH 054/172] Make processExtensions private --- src/pkg/packager/creator/extensions.go | 3 +-- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/creator/skeleton.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index 1b86f9d7db..536f143e67 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -12,8 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// TODO: @lucasrod16: make this function private once assemble() logic is moved to the creator package. -func ProcessExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { +func processExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 517caa6339..f7bdfaa551 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -66,7 +66,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := ProcessExtensions(composedPkg, &pc.cfg.CreateOpts, dst) + extendedPkg, err := processExtensions(composedPkg, &pc.cfg.CreateOpts, dst) if err != nil { return nil, nil, err } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 86226186c3..0754b02e75 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -51,7 +51,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(_ *layout.PackagePaths) (pkg *t // TODO: print warnings somewhere else in the skeleton create flow. func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { - pkg, err := ProcessExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, dst) + pkg, err := processExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, dst) if err != nil { return nil } From 6ba4916bf7c524010d86e32ff961be56e7197b1b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 24 Jan 2024 16:25:16 -0600 Subject: [PATCH 055/172] Do not process files from URL source in skeleton packages --- src/pkg/packager/creator/skeleton.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 0754b02e75..8ed682a000 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -143,6 +143,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent for filesIdx, file := range component.Files { message.Debugf("Loading %#v", file) + if helpers.IsURL(file.Source) { + continue + } + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) dst := filepath.Join(componentPaths.Base, rel) destinationDir := filepath.Dir(dst) From 9fb95af7bb55aa19c742fe4eeec1e4dee0c72be3 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 12:14:04 -0600 Subject: [PATCH 056/172] Make addSignature() a private method --- src/pkg/layout/package.go | 18 +++++++++--------- src/pkg/layout/package_test.go | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 875453dada..8e2dab301b 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -143,16 +143,8 @@ func (pp *PackagePaths) IsLegacyLayout() bool { return pp.isLegacyLayout } -// AddSignature sets the signature path if the keyPath is not empty. -func (pp *PackagePaths) AddSignature(keyPath string) *PackagePaths { - if keyPath != "" { - pp.Signature = filepath.Join(pp.Base, Signature) - } - return pp -} - func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { - pp.AddSignature(signingKeyPath) + pp.addSignature(signingKeyPath) passwordFunc := func(_ bool) ([]byte, error) { if signingKeyPassword != "" { @@ -168,6 +160,14 @@ func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) e return nil } +// addSignature sets the signature path if the keyPath is not empty. +func (pp *PackagePaths) addSignature(keyPath string) *PackagePaths { + if keyPath != "" { + pp.Signature = filepath.Join(pp.Base, Signature) + } + return pp +} + func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSizeMB int) error { spinner := message.NewProgressSpinner("Writing %s to %s", pp.Base, destinationTarball) defer spinner.Stop() diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index 55c3aeb695..c3e543642b 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -36,14 +36,14 @@ func TestPackageFiles(t *testing.T) { require.Equal(t, expected, files) - pp = pp.AddSignature("") + pp = pp.addSignature("") files = pp.Files() - // AddSignature will only add the signature if it is not empty + // addSignature will only add the signature if it is not empty require.Equal(t, expected, files) - pp = pp.AddSignature("key.priv") + pp = pp.addSignature("key.priv") files = pp.Files() From 6fb3881b249a3c16e1961c52101fb728dbc483cf Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 12:48:37 -0600 Subject: [PATCH 057/172] Move processExtensions() from Assemble() to LoadPackageDefinition() for skeleton --- src/pkg/packager/creator/normal.go | 10 +++++----- src/pkg/packager/creator/skeleton.go | 16 ++++++++-------- src/pkg/packager/creator/template.go | 12 ++++++------ src/pkg/packager/prepare.go | 8 ++++---- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index f7bdfaa551..4a591258cd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -58,7 +58,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) + templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -66,24 +66,24 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := processExtensions(composedPkg, &pc.cfg.CreateOpts, dst) + extendedPkg, err := processExtensions(templatedPkg, &pc.cfg.CreateOpts, dst) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. - if pc.cfg.Pkg.Build.Differential { + if extendedPkg.Build.Differential { diffData, err := loadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) if err != nil { return nil, nil, err } - versionsMatch := pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == pc.cfg.Pkg.Metadata.Version + versionsMatch := diffData.DifferentialPackageVersion == extendedPkg.Metadata.Version if versionsMatch { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := pc.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || pc.cfg.Pkg.Metadata.Version == "" + noVersionSet := diffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" if noVersionSet { return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 8ed682a000..fb54a62d87 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -33,30 +33,30 @@ type SkeletonCreator struct { } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(_ *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) if err != nil { return nil, nil, err } warnings = append(warnings, composeWarnings...) + pkg, err = processExtensions(composedPkg, &sc.cfg.CreateOpts, dst) + if err != nil { + return nil, nil, err + } + return pkg, warnings, nil } // TODO: print warnings somewhere else in the skeleton create flow. func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { - pkg, err := processExtensions(&sc.cfg.Pkg, &sc.cfg.CreateOpts, dst) - if err != nil { - return nil - } - - for idx, component := range pkg.Components { + for idx, component := range sc.cfg.Pkg.Components { if err := sc.addComponent(idx, component, dst); err != nil { return err } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 716001c833..f757d097d9 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (warnings []string, err error) { +func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (templatedPkg *types.ZarfPackage, warnings []string, err error) { templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { @@ -52,25 +52,25 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti // update the component templates on the package if err := ReloadComponentTemplatesInPackage(pkg); err != nil { - return nil, err + return nil, nil, err } if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return nil, err + return nil, nil, err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return nil, err + return nil, nil, err } // Add special variable for the current package architecture templateMap[types.ZarfPackageArch] = pkg.Build.Architecture if err := utils.ReloadYamlTemplate(pkg, templateMap); err != nil { - return nil, err + return nil, nil, err } - return warnings, nil + return templatedPkg, warnings, nil } // ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 5134c22d1d..f2d908fd1d 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -54,7 +54,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - pkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) + composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templateWarnings, err := creator.FillActiveTemplate(pkg, &p.cfg.CreateOpts) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, &p.cfg.CreateOpts) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -73,7 +73,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { message.Warn(warning) } - for _, component := range p.cfg.Pkg.Components { + for _, component := range templatedPkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + "if any repos contain helm charts you want to template and " + @@ -84,7 +84,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { componentDefinition := "\ncomponents:\n" - for _, component := range p.cfg.Pkg.Components { + for _, component := range templatedPkg.Components { if len(component.Charts)+len(component.Manifests)+len(component.Repos) < 1 { // Skip if it doesn't have what we need From e0f6830e3a5290636687a088d599f9a6b51eeef3 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 12:56:27 -0600 Subject: [PATCH 058/172] Ensure FillActiveTemplate() does not return a nil pointer --- src/pkg/packager/creator/template.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index f757d097d9..3b55230e77 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -70,6 +70,8 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti return nil, nil, err } + templatedPkg = pkg + return templatedPkg, warnings, nil } From c6fd2a153d381d4c865b93b5e2a0b352c6884aa2 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 14:55:48 -0600 Subject: [PATCH 059/172] Move component archiving and generateChecksums logic to Ouput() for skeleton --- src/pkg/packager/creator/normal.go | 196 +++++++++++++-------------- src/pkg/packager/creator/skeleton.go | 9 +- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 4a591258cd..e4af16f1fd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -195,6 +195,104 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { return nil } +// Output assumes it is running from cwd, not the build directory +func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { + // Process the component directories into compressed tarballs + // NOTE: This is purposefully being done after the SBOM cataloging + for _, component := range pc.cfg.Pkg.Components { + // Make the component a tar archive + if err := dst.Components.Archive(component, true); err != nil { + return fmt.Errorf("unable to archive component: %s", err.Error()) + } + } + + // Calculate all the checksums + checksumChecksum, err := generateChecksums(dst) + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + // Record the migrations that will be ran on the package. + pc.cfg.Pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Save the transformed config. + if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the config file if a key was provided + if pc.cfg.CreateOpts.SigningKeyPath != "" { + if err := dst.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { + return err + } + } + + // Create a remote ref + client for the package (if output is OCI) + // then publish the package to the remote. + if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { + ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) + if err != nil { + return err + } + remote, err := oci.NewOrasRemote(ref, oci.PlatformForArch(config.GetArch())) + if err != nil { + return err + } + + err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) + if err != nil { + return fmt.Errorf("unable to publish package: %w", err) + } + message.HorizontalRule() + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + } else { + // Use the output path if the user specified it. + packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) + + // Try to remove the package if it already exists. + _ = os.Remove(packageName) + + // Create the package tarball. + if err := dst.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { + return fmt.Errorf("unable to archive package: %w", err) + } + } + + // Output the SBOM files into a directory if specified. + if pc.cfg.CreateOpts.ViewSBOM || pc.cfg.CreateOpts.SBOMOutputDir != "" { + outputSBOM := pc.cfg.CreateOpts.SBOMOutputDir + var sbomDir string + if err := dst.SBOMs.Unarchive(); err != nil { + return fmt.Errorf("unable to unarchive SBOMs: %w", err) + } + sbomDir = dst.SBOMs.Path + + if outputSBOM != "" { + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) + if err != nil { + return err + } + sbomDir = out + } + + if pc.cfg.CreateOpts.ViewSBOM { + sbom.ViewSBOMFiles(sbomDir) + } + } + return nil +} + func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) @@ -375,104 +473,6 @@ func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layou return nil } -// Output assumes it is running from cwd, not the build directory -func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range pc.cfg.Pkg.Components { - // Make the component a tar archive - if err := dst.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := generateChecksums(dst) - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Record the migrations that will be ran on the package. - pc.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Save the transformed config. - if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // Sign the config file if a key was provided - if pc.cfg.CreateOpts.SigningKeyPath != "" { - if err := dst.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - // Create a remote ref + client for the package (if output is OCI) - // then publish the package to the remote. - if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) - if err != nil { - return err - } - remote, err := oci.NewOrasRemote(ref, oci.PlatformForArch(config.GetArch())) - if err != nil { - return err - } - - err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := dst.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if pc.cfg.CreateOpts.ViewSBOM || pc.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := pc.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := dst.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = dst.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if pc.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - return nil -} - func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { componentPaths, err := dst.Components.Create(component) if err != nil { diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index fb54a62d87..a42a7e3d70 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -60,7 +60,12 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { if err := sc.addComponent(idx, component, dst); err != nil { return err } + } + return nil +} +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { + for _, component := range sc.cfg.Pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } @@ -72,10 +77,6 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { } sc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - return nil -} - -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { return utils.WriteYaml(dst.ZarfYAML, sc.cfg.Pkg, 0400) } From 05010a400252a024cf0c9e16fb053198b30a6667 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 21:29:09 -0600 Subject: [PATCH 060/172] Use ZarfPackage struct instead of pointers in creator functions Pass ZarfPackage in as argument to Assemble and Output --- src/pkg/packager/create.go | 9 +++-- src/pkg/packager/creator/compose.go | 21 +++++----- src/pkg/packager/creator/differential.go | 38 ++++++++++-------- src/pkg/packager/creator/extensions.go | 15 +++---- src/pkg/packager/creator/new.go | 6 +-- src/pkg/packager/creator/normal.go | 50 ++++++++++++------------ src/pkg/packager/creator/skeleton.go | 28 +++++++------ src/pkg/packager/creator/template.go | 21 +++++----- src/pkg/packager/creator/utils.go | 38 +++++++++--------- src/pkg/packager/dev.go | 19 ++++----- src/pkg/packager/prepare.go | 4 +- src/pkg/packager/publish.go | 12 ++++-- 12 files changed, 138 insertions(+), 123 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 0c45ee4f93..9e0453e27d 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -21,6 +21,7 @@ func (p *Packager) Create() (err error) { if err != nil { return err } + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } @@ -30,7 +31,7 @@ func (p *Packager) Create() (err error) { c := creator.New(p.cfg) - pkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) if err != nil { return err } @@ -38,7 +39,7 @@ func (p *Packager) Create() (err error) { p.warnings = append(p.warnings, warnings...) // Perform early package validation. - if err := validate.Run(*pkg); err != nil { + if err := validate.Run(loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } @@ -46,7 +47,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(p.layout); err != nil { + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } @@ -55,5 +56,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output(p.layout) + return c.Output(loadedPkg, p.layout) } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index f1214e15fd..aa977fcf08 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,11 +10,12 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { - components := []types.ZarfComponent{} +func ComposeComponents(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (composedPkg types.ZarfPackage, warnings []string, err error) { + composedPkg = pkg - pkgVars := pkg.Variables - pkgConsts := pkg.Constants + components := []types.ZarfComponent{} + pkgVars := []types.ZarfPackageVariable{} + pkgConsts := []types.ZarfPackageConstant{} for i, component := range pkg.Components { arch := pkg.Metadata.Architecture @@ -30,7 +31,7 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio // build the import chain chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, createOpts.Flavor) if err != nil { - return nil, nil, err + return pkg, nil, err } message.Debugf("%s", chain) @@ -41,7 +42,7 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio // get the composed component composed, err := chain.Compose() if err != nil { - return nil, nil, err + return pkg, nil, err } components = append(components, *composed) @@ -51,12 +52,10 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio } // set the filtered + composed components - pkg.Components = components + composedPkg.Components = components - pkg.Variables = pkgVars - pkg.Constants = pkgConsts - - composedPkg = pkg + composedPkg.Variables = pkgVars + composedPkg.Constants = pkgConsts return composedPkg, warnings, nil } diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 3fa65451e2..8e82e62cb1 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -23,10 +23,12 @@ import ( ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func loadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { +func loadDifferentialData(diffData types.DifferentialData) (loadedDiffData types.DifferentialData, err error) { + loadedDiffData = diffData + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - return nil, err + return diffData, err } defer os.RemoveAll(tmpDir) @@ -35,25 +37,25 @@ func loadDifferentialData(diffData *types.DifferentialData) (*types.Differential if helpers.IsOCIURL(diffData.DifferentialPackagePath) { remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) if err != nil { - return nil, err + return diffData, err } pkg, err := remote.FetchZarfYAML() if err != nil { - return nil, err + return diffData, err } err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) if err != nil { - return nil, err + return diffData, err } } else { if err := archiver.Extract(diffData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return nil, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + return diffData, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) } } var differentialZarfConfig types.ZarfPackage if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return nil, fmt.Errorf("unable to load the differential zarf package spec: %w", err) + return diffData, fmt.Errorf("unable to load the differential zarf package spec: %w", err) } allIncludedImagesMap := map[string]bool{} @@ -68,18 +70,20 @@ func loadDifferentialData(diffData *types.DifferentialData) (*types.Differential } } - diffData.DifferentialImages = allIncludedImagesMap - diffData.DifferentialRepos = allIncludedReposMap - diffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + loadedDiffData.DifferentialImages = allIncludedImagesMap + loadedDiffData.DifferentialRepos = allIncludedReposMap + loadedDiffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - return diffData, nil + return loadedDiffData, nil } // removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { +func removeCopiesFromDifferentialPackage(pkg types.ZarfPackage, diffData types.DifferentialData) (diffPkg types.ZarfPackage, err error) { + diffPkg = pkg + // Loop through all of the components to determine if any of them are using already included images or repos componentMap := make(map[int]types.ZarfComponent) - for idx, component := range pkg.Components { + for idx, component := range diffPkg.Components { newImageList := []string{} newRepoList := []string{} // Generate a list of all unique images for this component @@ -87,7 +91,7 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package imgRef, err := transform.ParseImageRef(img) if err != nil { - return nil, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + return pkg, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) } // Only include new images or images that have a commonly overwritten tag @@ -105,7 +109,7 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types // Split the remote url and the zarf reference _, refPlain, err := transform.GitURLSplitRef(repoURL) if err != nil { - return nil, err + return pkg, err } var ref plumbing.ReferenceName @@ -131,8 +135,8 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types // Update the package with the new component list for idx, component := range componentMap { - pkg.Components[idx] = component + diffPkg.Components[idx] = component } - return pkg, nil + return diffPkg, nil } diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index 536f143e67..0d26d04d34 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -12,25 +12,26 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func processExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { +func processExtensions(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions, layout *layout.PackagePaths) (extendedPkg types.ZarfPackage, err error) { + extendedPkg = pkg components := []types.ZarfComponent{} // Create component paths and process extensions for each component. for _, c := range pkg.Components { componentPaths, err := layout.Components.Create(c) if err != nil { - return nil, err + return pkg, err } // Big Bang if c.Extensions.BigBang != nil { if createOpts.IsSkeleton { if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + return pkg, fmt.Errorf("unable to process bigbang extension: %w", err) } } else { if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { - return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + return pkg, fmt.Errorf("unable to process bigbang extension: %w", err) } } } @@ -38,9 +39,9 @@ func processExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio components = append(components, c) } - // Update the parent package config with the expanded sub components. + // Update the package config with the expanded sub components. // This is important when the deploy package is created. - pkg.Components = components + extendedPkg.Components = components - return pkg, nil + return extendedPkg, nil } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index ad0e3936a8..56e73be18a 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -11,9 +11,9 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(dst *layout.PackagePaths) (*types.ZarfPackage, []string, error) - Assemble(dst *layout.PackagePaths) error - Output(dst *layout.PackagePaths) error + LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (types.ZarfPackage, []string, error) + Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error + Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e4af16f1fd..fb5159dc8d 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -43,54 +43,54 @@ type PackageCreator struct { } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(&pc.cfg.Pkg, &pc.cfg.CreateOpts) +func (pc *PackageCreator) LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (loadedPkg types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(pkg, pc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &pc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.cfg.CreateOpts) if err != nil { - return nil, nil, err + return pkg, nil, err } warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) + templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, pc.cfg.CreateOpts) if err != nil { - return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) + return pkg, nil, fmt.Errorf("unable to fill values in template: %w", err) } warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := processExtensions(templatedPkg, &pc.cfg.CreateOpts, dst) + extendedPkg, err := processExtensions(templatedPkg, pc.cfg.CreateOpts, dst) if err != nil { - return nil, nil, err + return pkg, nil, err } // If we are creating a differential package, remove duplicate images and repos. if extendedPkg.Build.Differential { - diffData, err := loadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) + diffData, err := loadDifferentialData(pc.cfg.CreateOpts.DifferentialData) if err != nil { - return nil, nil, err + return pkg, nil, err } versionsMatch := diffData.DifferentialPackageVersion == extendedPkg.Metadata.Version if versionsMatch { - return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) + return pkg, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } noVersionSet := diffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" if noVersionSet { - return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) + return pkg, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, diffData) if err != nil { - return nil, nil, err + return pkg, nil, err } return diffPkg, nil, nil } @@ -98,13 +98,13 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * return extendedPkg, warnings, nil } -func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { +func (pc *PackageCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error { var imageList []transform.Image skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM componentSBOMs := map[string]*layout.ComponentSBOM{} - for _, component := range pc.cfg.Pkg.Components { + for _, component := range pkg.Components { onCreate := component.Actions.OnCreate onFailure := func() { @@ -160,7 +160,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { ImagesPath: dst.Images.Base, ImageList: imageList, Insecure: config.CommonOptions.Insecure, - Architectures: []string{pc.cfg.Pkg.Metadata.Architecture, pc.cfg.Pkg.Build.Architecture}, + Architectures: []string{pkg.Metadata.Architecture, pkg.Build.Architecture}, RegistryOverrides: pc.cfg.CreateOpts.RegistryOverrides, } @@ -196,10 +196,10 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { } // Output assumes it is running from cwd, not the build directory -func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { +func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range pc.cfg.Pkg.Components { + for _, component := range pkg.Components { // Make the component a tar archive if err := dst.Components.Archive(component, true); err != nil { return fmt.Errorf("unable to archive component: %s", err.Error()) @@ -211,16 +211,16 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } - pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + pkg.Metadata.AggregateChecksum = checksumChecksum // Record the migrations that will be ran on the package. - pc.cfg.Pkg.Build.Migrations = []string{ + pkg.Build.Migrations = []string{ deprecated.ScriptsToActionsMigrated, deprecated.PluralizeSetVariable, } // Save the transformed config. - if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } @@ -234,7 +234,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) + ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pkg.Metadata, &pkg.Build) if err != nil { return err } @@ -243,7 +243,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { return err } - err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(&pkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -258,7 +258,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) + packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pkg, pc.cfg.CreateOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) @@ -279,7 +279,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index a42a7e3d70..a2ae38097f 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -33,30 +33,32 @@ type SkeletonCreator struct { } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) +func (sc *SkeletonCreator) LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (loadedPkg types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(pkg, sc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.cfg.CreateOpts) if err != nil { - return nil, nil, err + return pkg, nil, err } warnings = append(warnings, composeWarnings...) - pkg, err = processExtensions(composedPkg, &sc.cfg.CreateOpts, dst) + extendedPkg, err := processExtensions(composedPkg, sc.cfg.CreateOpts, dst) if err != nil { - return nil, nil, err + return pkg, nil, err } - return pkg, warnings, nil + loadedPkg = extendedPkg + + return loadedPkg, warnings, nil } // TODO: print warnings somewhere else in the skeleton create flow. -func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { - for idx, component := range sc.cfg.Pkg.Components { +func (sc *SkeletonCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error { + for idx, component := range pkg.Components { if err := sc.addComponent(idx, component, dst); err != nil { return err } @@ -64,8 +66,8 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { return nil } -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { - for _, component := range sc.cfg.Pkg.Components { +func (sc *SkeletonCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error { + for _, component := range pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } @@ -75,9 +77,9 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } - sc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + pkg.Metadata.AggregateChecksum = checksumChecksum - return utils.WriteYaml(dst.ZarfYAML, sc.cfg.Pkg, 0400) + return utils.WriteYaml(dst.ZarfYAML, pkg, 0400) } func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 3b55230e77..174fa3e35d 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,11 +14,12 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (templatedPkg *types.ZarfPackage, warnings []string, err error) { +func FillActiveTemplate(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (templatedPkg types.ZarfPackage, warnings []string, err error) { + templatedPkg = pkg templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(pkg, templatePrefix, "###") + yamlTemplates, err := utils.FindYamlTemplates(templatedPkg, templatePrefix, "###") if err != nil { return err } @@ -51,27 +52,25 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti } // update the component templates on the package - if err := ReloadComponentTemplatesInPackage(pkg); err != nil { - return nil, nil, err + if err := ReloadComponentTemplatesInPackage(&templatedPkg); err != nil { + return pkg, nil, err } if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return nil, nil, err + return pkg, nil, err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return nil, nil, err + return pkg, nil, err } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = pkg.Build.Architecture + templateMap[types.ZarfPackageArch] = templatedPkg.Build.Architecture - if err := utils.ReloadYamlTemplate(pkg, templateMap); err != nil { - return nil, nil, err + if err := utils.ReloadYamlTemplate(&templatedPkg, templateMap); err != nil { + return pkg, nil, err } - templatedPkg = pkg - return templatedPkg, warnings, nil } diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index cb4dfdd5f7..4a26bccb9e 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -16,56 +16,58 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (*types.ZarfPackage, error) { +func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (configuredPkg types.ZarfPackage, err error) { + configuredPkg = pkg + now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. // Record the name of the user creating the package. if runtime.GOOS == "windows" { - pkg.Build.User = os.Getenv("USERNAME") + configuredPkg.Build.User = os.Getenv("USERNAME") } else { - pkg.Build.User = os.Getenv("USER") + configuredPkg.Build.User = os.Getenv("USER") } hostname, err := os.Hostname() if err != nil { - return nil, err + return configuredPkg, err } - if utils.IsInitConfig(*pkg) { - pkg.Metadata.Version = config.CLIVersion + if utils.IsInitConfig(pkg) { + configuredPkg.Metadata.Version = config.CLIVersion } // Set package architecture if createOpts.IsSkeleton { - pkg.Metadata.Architecture = "skeleton" + configuredPkg.Metadata.Architecture = "skeleton" } - if pkg.Metadata.Architecture == "" { - pkg.Metadata.Architecture = config.GetArch() + if configuredPkg.Metadata.Architecture == "" { + configuredPkg.Metadata.Architecture = config.GetArch() } - pkg.Build.Architecture = pkg.Metadata.Architecture + configuredPkg.Build.Architecture = configuredPkg.Metadata.Architecture // Record the time of package creation. - pkg.Build.Timestamp = now.Format(time.RFC1123Z) + configuredPkg.Build.Timestamp = now.Format(time.RFC1123Z) // Record the Zarf Version the CLI was built with. - pkg.Build.Version = config.CLIVersion + configuredPkg.Build.Version = config.CLIVersion // Record the hostname of the package creation terminal. - pkg.Build.Terminal = hostname + configuredPkg.Build.Terminal = hostname // If the --differential flag was used, record that this is a differential package. if createOpts.DifferentialData.DifferentialPackagePath != "" { - pkg.Build.Differential = true + configuredPkg.Build.Differential = true } // Record the flavor of Zarf used to build this package (if any). - pkg.Build.Flavor = createOpts.Flavor + configuredPkg.Build.Flavor = createOpts.Flavor - pkg.Build.RegistryOverrides = createOpts.RegistryOverrides + configuredPkg.Build.RegistryOverrides = createOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. - pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + configuredPkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return pkg, nil + return configuredPkg, nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b63cfbef16..3fd5a934a0 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -37,10 +37,11 @@ func (p *Packager) DevDeploy() error { c := creator.New(p.cfg) - pkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) // Filter out components that are not compatible with this system @@ -51,25 +52,25 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - pkg.Components = p.getSelectedComponents() + loadedPkg.Components = p.getSelectedComponents() - if err := validate.Run(*pkg); err != nil { + if err := validate.Run(loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } // If building in yolo mode, strip out all images and repos if !p.cfg.CreateOpts.NoYOLO { for idx := range p.cfg.Pkg.Components { - p.cfg.Pkg.Components[idx].Images = []string{} - p.cfg.Pkg.Components[idx].Repos = []string{} + loadedPkg.Components[idx].Images = []string{} + loadedPkg.Components[idx].Repos = []string{} } } - if err := c.Assemble(p.layout); err != nil { + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } - message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) + message.HeaderInfof("📦 PACKAGE DEPLOY %s", loadedPkg.Metadata.Name) // Set variables and prompt if --confirm is not set if err := variables.SetVariableMapInConfig(p.cfg); err != nil { @@ -79,7 +80,7 @@ func (p *Packager) DevDeploy() error { p.connectStrings = make(types.ConnectStrings) if !p.cfg.CreateOpts.NoYOLO { - p.cfg.Pkg.Metadata.YOLO = true + loadedPkg.Metadata.YOLO = true } else { p.hpaModified = false // Reset registry HPA scale down whether an error occurs or not @@ -101,7 +102,7 @@ func (p *Packager) DevDeploy() error { message.HorizontalRule() message.Title("Next steps:", "") - message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) + message.ZarfCommand("package inspect %s", loadedPkg.Metadata.Name) // cd back return os.Chdir(cwd) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index f2d908fd1d..d5810c84f7 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -54,7 +54,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) + composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, &p.cfg.CreateOpts) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index bc8241218d..a8cb9ff02e 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -90,22 +90,28 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } + c := creator.New(p.cfg) - _, warnings, err := c.LoadPackageDefinition(p.layout) + + loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) - if err := c.Assemble(p.layout); err != nil { + + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } - if err := c.Output(p.layout); err != nil { + + if err := c.Output(loadedPkg, p.layout); err != nil { return err } } else { From 03a55b45ae820206bf63a776e73e6088dcf6778e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 29 Jan 2024 22:21:43 -0600 Subject: [PATCH 061/172] Revert "Use ZarfPackage struct instead of pointers in creator functions" This reverts commit 05010a400252a024cf0c9e16fb053198b30a6667. --- src/pkg/packager/create.go | 9 ++--- src/pkg/packager/creator/compose.go | 21 +++++----- src/pkg/packager/creator/differential.go | 38 ++++++++---------- src/pkg/packager/creator/extensions.go | 15 ++++--- src/pkg/packager/creator/new.go | 6 +-- src/pkg/packager/creator/normal.go | 50 ++++++++++++------------ src/pkg/packager/creator/skeleton.go | 28 ++++++------- src/pkg/packager/creator/template.go | 21 +++++----- src/pkg/packager/creator/utils.go | 38 +++++++++--------- src/pkg/packager/dev.go | 19 +++++---- src/pkg/packager/prepare.go | 4 +- src/pkg/packager/publish.go | 12 ++---- 12 files changed, 123 insertions(+), 138 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 9e0453e27d..0c45ee4f93 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -21,7 +21,6 @@ func (p *Packager) Create() (err error) { if err != nil { return err } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } @@ -31,7 +30,7 @@ func (p *Packager) Create() (err error) { c := creator.New(p.cfg) - loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) + pkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -39,7 +38,7 @@ func (p *Packager) Create() (err error) { p.warnings = append(p.warnings, warnings...) // Perform early package validation. - if err := validate.Run(loadedPkg); err != nil { + if err := validate.Run(*pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } @@ -47,7 +46,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } @@ -56,5 +55,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output(loadedPkg, p.layout) + return c.Output(p.layout) } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index aa977fcf08..f1214e15fd 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,12 +10,11 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func ComposeComponents(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (composedPkg types.ZarfPackage, warnings []string, err error) { - composedPkg = pkg - +func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { components := []types.ZarfComponent{} - pkgVars := []types.ZarfPackageVariable{} - pkgConsts := []types.ZarfPackageConstant{} + + pkgVars := pkg.Variables + pkgConsts := pkg.Constants for i, component := range pkg.Components { arch := pkg.Metadata.Architecture @@ -31,7 +30,7 @@ func ComposeComponents(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions // build the import chain chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, createOpts.Flavor) if err != nil { - return pkg, nil, err + return nil, nil, err } message.Debugf("%s", chain) @@ -42,7 +41,7 @@ func ComposeComponents(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions // get the composed component composed, err := chain.Compose() if err != nil { - return pkg, nil, err + return nil, nil, err } components = append(components, *composed) @@ -52,10 +51,12 @@ func ComposeComponents(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions } // set the filtered + composed components - composedPkg.Components = components + pkg.Components = components - composedPkg.Variables = pkgVars - composedPkg.Constants = pkgConsts + pkg.Variables = pkgVars + pkg.Constants = pkgConsts + + composedPkg = pkg return composedPkg, warnings, nil } diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 8e82e62cb1..3fa65451e2 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -23,12 +23,10 @@ import ( ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func loadDifferentialData(diffData types.DifferentialData) (loadedDiffData types.DifferentialData, err error) { - loadedDiffData = diffData - +func loadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - return diffData, err + return nil, err } defer os.RemoveAll(tmpDir) @@ -37,25 +35,25 @@ func loadDifferentialData(diffData types.DifferentialData) (loadedDiffData types if helpers.IsOCIURL(diffData.DifferentialPackagePath) { remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) if err != nil { - return diffData, err + return nil, err } pkg, err := remote.FetchZarfYAML() if err != nil { - return diffData, err + return nil, err } err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) if err != nil { - return diffData, err + return nil, err } } else { if err := archiver.Extract(diffData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return diffData, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + return nil, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) } } var differentialZarfConfig types.ZarfPackage if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return diffData, fmt.Errorf("unable to load the differential zarf package spec: %w", err) + return nil, fmt.Errorf("unable to load the differential zarf package spec: %w", err) } allIncludedImagesMap := map[string]bool{} @@ -70,20 +68,18 @@ func loadDifferentialData(diffData types.DifferentialData) (loadedDiffData types } } - loadedDiffData.DifferentialImages = allIncludedImagesMap - loadedDiffData.DifferentialRepos = allIncludedReposMap - loadedDiffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + diffData.DifferentialImages = allIncludedImagesMap + diffData.DifferentialRepos = allIncludedReposMap + diffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - return loadedDiffData, nil + return diffData, nil } // removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func removeCopiesFromDifferentialPackage(pkg types.ZarfPackage, diffData types.DifferentialData) (diffPkg types.ZarfPackage, err error) { - diffPkg = pkg - +func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { // Loop through all of the components to determine if any of them are using already included images or repos componentMap := make(map[int]types.ZarfComponent) - for idx, component := range diffPkg.Components { + for idx, component := range pkg.Components { newImageList := []string{} newRepoList := []string{} // Generate a list of all unique images for this component @@ -91,7 +87,7 @@ func removeCopiesFromDifferentialPackage(pkg types.ZarfPackage, diffData types.D // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package imgRef, err := transform.ParseImageRef(img) if err != nil { - return pkg, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + return nil, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) } // Only include new images or images that have a commonly overwritten tag @@ -109,7 +105,7 @@ func removeCopiesFromDifferentialPackage(pkg types.ZarfPackage, diffData types.D // Split the remote url and the zarf reference _, refPlain, err := transform.GitURLSplitRef(repoURL) if err != nil { - return pkg, err + return nil, err } var ref plumbing.ReferenceName @@ -135,8 +131,8 @@ func removeCopiesFromDifferentialPackage(pkg types.ZarfPackage, diffData types.D // Update the package with the new component list for idx, component := range componentMap { - diffPkg.Components[idx] = component + pkg.Components[idx] = component } - return diffPkg, nil + return pkg, nil } diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go index 0d26d04d34..536f143e67 100644 --- a/src/pkg/packager/creator/extensions.go +++ b/src/pkg/packager/creator/extensions.go @@ -12,26 +12,25 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func processExtensions(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions, layout *layout.PackagePaths) (extendedPkg types.ZarfPackage, err error) { - extendedPkg = pkg +func processExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. for _, c := range pkg.Components { componentPaths, err := layout.Components.Create(c) if err != nil { - return pkg, err + return nil, err } // Big Bang if c.Extensions.BigBang != nil { if createOpts.IsSkeleton { if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return pkg, fmt.Errorf("unable to process bigbang extension: %w", err) + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) } } else { if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { - return pkg, fmt.Errorf("unable to process bigbang extension: %w", err) + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) } } } @@ -39,9 +38,9 @@ func processExtensions(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions components = append(components, c) } - // Update the package config with the expanded sub components. + // Update the parent package config with the expanded sub components. // This is important when the deploy package is created. - extendedPkg.Components = components + pkg.Components = components - return extendedPkg, nil + return pkg, nil } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 56e73be18a..ad0e3936a8 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -11,9 +11,9 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (types.ZarfPackage, []string, error) - Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error - Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error + LoadPackageDefinition(dst *layout.PackagePaths) (*types.ZarfPackage, []string, error) + Assemble(dst *layout.PackagePaths) error + Output(dst *layout.PackagePaths) error } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index fb5159dc8d..e4af16f1fd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -43,54 +43,54 @@ type PackageCreator struct { } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (loadedPkg types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(pkg, pc.cfg.CreateOpts) +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(&pc.cfg.Pkg, &pc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &pc.cfg.CreateOpts) if err != nil { - return pkg, nil, err + return nil, nil, err } warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, pc.cfg.CreateOpts) + templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) if err != nil { - return pkg, nil, fmt.Errorf("unable to fill values in template: %w", err) + return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := processExtensions(templatedPkg, pc.cfg.CreateOpts, dst) + extendedPkg, err := processExtensions(templatedPkg, &pc.cfg.CreateOpts, dst) if err != nil { - return pkg, nil, err + return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. if extendedPkg.Build.Differential { - diffData, err := loadDifferentialData(pc.cfg.CreateOpts.DifferentialData) + diffData, err := loadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) if err != nil { - return pkg, nil, err + return nil, nil, err } versionsMatch := diffData.DifferentialPackageVersion == extendedPkg.Metadata.Version if versionsMatch { - return pkg, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) + return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } noVersionSet := diffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" if noVersionSet { - return pkg, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) + return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, diffData) if err != nil { - return pkg, nil, err + return nil, nil, err } return diffPkg, nil, nil } @@ -98,13 +98,13 @@ func (pc *PackageCreator) LoadPackageDefinition(pkg types.ZarfPackage, dst *layo return extendedPkg, warnings, nil } -func (pc *PackageCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error { +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { var imageList []transform.Image skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM componentSBOMs := map[string]*layout.ComponentSBOM{} - for _, component := range pkg.Components { + for _, component := range pc.cfg.Pkg.Components { onCreate := component.Actions.OnCreate onFailure := func() { @@ -160,7 +160,7 @@ func (pc *PackageCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePat ImagesPath: dst.Images.Base, ImageList: imageList, Insecure: config.CommonOptions.Insecure, - Architectures: []string{pkg.Metadata.Architecture, pkg.Build.Architecture}, + Architectures: []string{pc.cfg.Pkg.Metadata.Architecture, pc.cfg.Pkg.Build.Architecture}, RegistryOverrides: pc.cfg.CreateOpts.RegistryOverrides, } @@ -196,10 +196,10 @@ func (pc *PackageCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePat } // Output assumes it is running from cwd, not the build directory -func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range pkg.Components { + for _, component := range pc.cfg.Pkg.Components { // Make the component a tar archive if err := dst.Components.Archive(component, true); err != nil { return fmt.Errorf("unable to archive component: %s", err.Error()) @@ -211,16 +211,16 @@ func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } - pkg.Metadata.AggregateChecksum = checksumChecksum + pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum // Record the migrations that will be ran on the package. - pkg.Build.Migrations = []string{ + pc.cfg.Pkg.Build.Migrations = []string{ deprecated.ScriptsToActionsMigrated, deprecated.PluralizeSetVariable, } // Save the transformed config. - if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } @@ -234,7 +234,7 @@ func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pkg.Metadata, &pkg.Build) + ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) if err != nil { return err } @@ -243,7 +243,7 @@ func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths return err } - err = remote.PublishPackage(&pkg, dst, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -258,7 +258,7 @@ func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pkg, pc.cfg.CreateOpts.DifferentialData)) + packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) @@ -279,7 +279,7 @@ func (pc *PackageCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pkg.Metadata.Name) + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index a2ae38097f..a42a7e3d70 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -33,32 +33,30 @@ type SkeletonCreator struct { } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(pkg types.ZarfPackage, dst *layout.PackagePaths) (loadedPkg types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(pkg, sc.cfg.CreateOpts) +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { + configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) if err != nil { - return pkg, nil, err + return nil, nil, err } warnings = append(warnings, composeWarnings...) - extendedPkg, err := processExtensions(composedPkg, sc.cfg.CreateOpts, dst) + pkg, err = processExtensions(composedPkg, &sc.cfg.CreateOpts, dst) if err != nil { - return pkg, nil, err + return nil, nil, err } - loadedPkg = extendedPkg - - return loadedPkg, warnings, nil + return pkg, warnings, nil } // TODO: print warnings somewhere else in the skeleton create flow. -func (sc *SkeletonCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePaths) error { - for idx, component := range pkg.Components { +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { + for idx, component := range sc.cfg.Pkg.Components { if err := sc.addComponent(idx, component, dst); err != nil { return err } @@ -66,8 +64,8 @@ func (sc *SkeletonCreator) Assemble(pkg types.ZarfPackage, dst *layout.PackagePa return nil } -func (sc *SkeletonCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePaths) error { - for _, component := range pkg.Components { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { + for _, component := range sc.cfg.Pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } @@ -77,9 +75,9 @@ func (sc *SkeletonCreator) Output(pkg types.ZarfPackage, dst *layout.PackagePath if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } - pkg.Metadata.AggregateChecksum = checksumChecksum + sc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - return utils.WriteYaml(dst.ZarfYAML, pkg, 0400) + return utils.WriteYaml(dst.ZarfYAML, sc.cfg.Pkg, 0400) } func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 174fa3e35d..3b55230e77 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,12 +14,11 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func FillActiveTemplate(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (templatedPkg types.ZarfPackage, warnings []string, err error) { - templatedPkg = pkg +func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (templatedPkg *types.ZarfPackage, warnings []string, err error) { templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(templatedPkg, templatePrefix, "###") + yamlTemplates, err := utils.FindYamlTemplates(pkg, templatePrefix, "###") if err != nil { return err } @@ -52,25 +51,27 @@ func FillActiveTemplate(pkg types.ZarfPackage, createOpts types.ZarfCreateOption } // update the component templates on the package - if err := ReloadComponentTemplatesInPackage(&templatedPkg); err != nil { - return pkg, nil, err + if err := ReloadComponentTemplatesInPackage(pkg); err != nil { + return nil, nil, err } if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return pkg, nil, err + return nil, nil, err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return pkg, nil, err + return nil, nil, err } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = templatedPkg.Build.Architecture + templateMap[types.ZarfPackageArch] = pkg.Build.Architecture - if err := utils.ReloadYamlTemplate(&templatedPkg, templateMap); err != nil { - return pkg, nil, err + if err := utils.ReloadYamlTemplate(pkg, templateMap); err != nil { + return nil, nil, err } + templatedPkg = pkg + return templatedPkg, warnings, nil } diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 4a26bccb9e..cb4dfdd5f7 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -16,58 +16,56 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (configuredPkg types.ZarfPackage, err error) { - configuredPkg = pkg - +func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (*types.ZarfPackage, error) { now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. // Record the name of the user creating the package. if runtime.GOOS == "windows" { - configuredPkg.Build.User = os.Getenv("USERNAME") + pkg.Build.User = os.Getenv("USERNAME") } else { - configuredPkg.Build.User = os.Getenv("USER") + pkg.Build.User = os.Getenv("USER") } hostname, err := os.Hostname() if err != nil { - return configuredPkg, err + return nil, err } - if utils.IsInitConfig(pkg) { - configuredPkg.Metadata.Version = config.CLIVersion + if utils.IsInitConfig(*pkg) { + pkg.Metadata.Version = config.CLIVersion } // Set package architecture if createOpts.IsSkeleton { - configuredPkg.Metadata.Architecture = "skeleton" + pkg.Metadata.Architecture = "skeleton" } - if configuredPkg.Metadata.Architecture == "" { - configuredPkg.Metadata.Architecture = config.GetArch() + if pkg.Metadata.Architecture == "" { + pkg.Metadata.Architecture = config.GetArch() } - configuredPkg.Build.Architecture = configuredPkg.Metadata.Architecture + pkg.Build.Architecture = pkg.Metadata.Architecture // Record the time of package creation. - configuredPkg.Build.Timestamp = now.Format(time.RFC1123Z) + pkg.Build.Timestamp = now.Format(time.RFC1123Z) // Record the Zarf Version the CLI was built with. - configuredPkg.Build.Version = config.CLIVersion + pkg.Build.Version = config.CLIVersion // Record the hostname of the package creation terminal. - configuredPkg.Build.Terminal = hostname + pkg.Build.Terminal = hostname // If the --differential flag was used, record that this is a differential package. if createOpts.DifferentialData.DifferentialPackagePath != "" { - configuredPkg.Build.Differential = true + pkg.Build.Differential = true } // Record the flavor of Zarf used to build this package (if any). - configuredPkg.Build.Flavor = createOpts.Flavor + pkg.Build.Flavor = createOpts.Flavor - configuredPkg.Build.RegistryOverrides = createOpts.RegistryOverrides + pkg.Build.RegistryOverrides = createOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. - configuredPkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return configuredPkg, nil + return pkg, nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 3fd5a934a0..b63cfbef16 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -37,11 +37,10 @@ func (p *Packager) DevDeploy() error { c := creator.New(p.cfg) - loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) + pkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } - p.warnings = append(p.warnings, warnings...) // Filter out components that are not compatible with this system @@ -52,25 +51,25 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - loadedPkg.Components = p.getSelectedComponents() + pkg.Components = p.getSelectedComponents() - if err := validate.Run(loadedPkg); err != nil { + if err := validate.Run(*pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } // If building in yolo mode, strip out all images and repos if !p.cfg.CreateOpts.NoYOLO { for idx := range p.cfg.Pkg.Components { - loadedPkg.Components[idx].Images = []string{} - loadedPkg.Components[idx].Repos = []string{} + p.cfg.Pkg.Components[idx].Images = []string{} + p.cfg.Pkg.Components[idx].Repos = []string{} } } - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } - message.HeaderInfof("📦 PACKAGE DEPLOY %s", loadedPkg.Metadata.Name) + message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) // Set variables and prompt if --confirm is not set if err := variables.SetVariableMapInConfig(p.cfg); err != nil { @@ -80,7 +79,7 @@ func (p *Packager) DevDeploy() error { p.connectStrings = make(types.ConnectStrings) if !p.cfg.CreateOpts.NoYOLO { - loadedPkg.Metadata.YOLO = true + p.cfg.Pkg.Metadata.YOLO = true } else { p.hpaModified = false // Reset registry HPA scale down whether an error occurs or not @@ -102,7 +101,7 @@ func (p *Packager) DevDeploy() error { message.HorizontalRule() message.Title("Next steps:", "") - message.ZarfCommand("package inspect %s", loadedPkg.Metadata.Name) + message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) // cd back return os.Chdir(cwd) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index d5810c84f7..f2d908fd1d 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -54,7 +54,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts) + composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, &p.cfg.CreateOpts) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index a8cb9ff02e..bc8241218d 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -90,28 +90,22 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { return fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - c := creator.New(p.cfg) - - loadedPkg, warnings, err := c.LoadPackageDefinition(p.cfg.Pkg, p.layout) + _, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } - p.warnings = append(p.warnings, warnings...) - - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(p.layout); err != nil { return err } - - if err := c.Output(loadedPkg, p.layout); err != nil { + if err := c.Output(p.layout); err != nil { return err } } else { From a378a848b7064693fc73c1e135b965658b430220 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 30 Jan 2024 15:57:58 -0600 Subject: [PATCH 062/172] Return warnings for diff pkgs in LoadPackageDefinition() --- src/pkg/packager/creator/normal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e4af16f1fd..cc1e90a8e3 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -92,7 +92,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * if err != nil { return nil, nil, err } - return diffPkg, nil, nil + return diffPkg, warnings, nil } return extendedPkg, warnings, nil From fb187a60d2bf04f5a38026034c9e1eecbc4bb604 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 14:50:53 -0600 Subject: [PATCH 063/172] Reduce dependency on PackagerConfig in Creator implementations Move logic for reading zarf.yaml into memory to LoadPackageDefinition() --- src/pkg/packager/create.go | 15 +-- src/pkg/packager/creator/compose.go | 6 +- src/pkg/packager/creator/extensions.go | 46 --------- src/pkg/packager/creator/new.go | 14 +-- src/pkg/packager/creator/normal.go | 93 ++++++++++++------ src/pkg/packager/creator/skeleton.go | 127 +++++++++++++++++-------- src/pkg/packager/creator/template.go | 8 +- src/pkg/packager/creator/utils.go | 36 +++---- src/pkg/packager/dev.go | 15 +-- src/pkg/packager/prepare.go | 4 +- src/pkg/packager/publish.go | 19 ++-- src/test/e2e/24_variables_test.go | 2 +- 12 files changed, 209 insertions(+), 176 deletions(-) delete mode 100644 src/pkg/packager/creator/extensions.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 0c45ee4f93..3b6254852b 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,9 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" - "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -24,13 +22,10 @@ func (p *Packager) Create() (err error) { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %w", err) - } - c := creator.New(p.cfg) + c := creator.New(p.cfg.CreateOpts) - pkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -38,7 +33,7 @@ func (p *Packager) Create() (err error) { p.warnings = append(p.warnings, warnings...) // Perform early package validation. - if err := validate.Run(*pkg); err != nil { + if err := validate.Run(*loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } @@ -46,7 +41,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(p.layout); err != nil { + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } @@ -55,5 +50,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output(p.layout) + return c.Output(loadedPkg, p.layout) } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index f1214e15fd..ce91a467dd 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,7 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (composedPkg *types.ZarfPackage, warnings []string, err error) { +func ComposeComponents(pkg *types.ZarfPackage, flavor string) (composedPkg *types.ZarfPackage, warnings []string, err error) { components := []types.ZarfComponent{} pkgVars := pkg.Variables @@ -19,7 +19,7 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio for i, component := range pkg.Components { arch := pkg.Metadata.Architecture // filter by architecture - if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { + if !composer.CompatibleComponent(component, arch, flavor) { continue } @@ -28,7 +28,7 @@ func ComposeComponents(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptio component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, createOpts.Flavor) + chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, flavor) if err != nil { return nil, nil, err } diff --git a/src/pkg/packager/creator/extensions.go b/src/pkg/packager/creator/extensions.go deleted file mode 100644 index 536f143e67..0000000000 --- a/src/pkg/packager/creator/extensions.go +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/types" -) - -func processExtensions(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions, layout *layout.PackagePaths) (*types.ZarfPackage, error) { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range pkg.Components { - componentPaths, err := layout.Components.Create(c) - if err != nil { - return nil, err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if createOpts.IsSkeleton { - if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return nil, fmt.Errorf("unable to process bigbang extension: %w", err) - } - } else { - if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { - return nil, fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - pkg.Components = components - - return pkg, nil -} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index ad0e3936a8..230492270f 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -11,15 +11,15 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(dst *layout.PackagePaths) (*types.ZarfPackage, []string, error) - Assemble(dst *layout.PackagePaths) error - Output(dst *layout.PackagePaths) error + LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) + Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error + Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error } // New returns a new Creator based on the provided create options. -func New(cfg *types.PackagerConfig) Creator { - if cfg.CreateOpts.IsSkeleton { - return &SkeletonCreator{cfg} +func New(createOpts types.ZarfCreateOptions) Creator { + if createOpts.IsSkeleton { + return &SkeletonCreator{createOpts: createOpts} } - return &PackageCreator{cfg} + return &PackageCreator{createOpts: createOpts} } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index cc1e90a8e3..21cbc3a5be 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/images" @@ -39,18 +40,25 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { - cfg *types.PackagerConfig + createOpts types.ZarfCreateOptions + cfg *types.PackagerConfig } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(&pc.cfg.Pkg, &pc.cfg.CreateOpts) +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + var pkg types.ZarfPackage + + if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { + return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + } + + configuredPkg, err := setPackageMetadata(pkg, pc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &pc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.createOpts.Flavor) if err != nil { return nil, nil, err } @@ -58,7 +66,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, &pc.cfg.CreateOpts) + templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, pc.createOpts.SetVariables) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -66,14 +74,14 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := processExtensions(templatedPkg, &pc.cfg.CreateOpts, dst) + extendedPkg, err := pc.processExtensions(templatedPkg, dst) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. if extendedPkg.Build.Differential { - diffData, err := loadDifferentialData(&pc.cfg.CreateOpts.DifferentialData) + diffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) if err != nil { return nil, nil, err } @@ -98,13 +106,13 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg * return extendedPkg, warnings, nil } -func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { +func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { var imageList []transform.Image - skipSBOMFlagUsed := pc.cfg.CreateOpts.SkipSBOM + skipSBOMFlagUsed := pc.createOpts.SkipSBOM componentSBOMs := map[string]*layout.ComponentSBOM{} - for _, component := range pc.cfg.Pkg.Components { + for _, component := range loadedPkg.Components { onCreate := component.Actions.OnCreate onFailure := func() { @@ -160,8 +168,8 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { ImagesPath: dst.Images.Base, ImageList: imageList, Insecure: config.CommonOptions.Insecure, - Architectures: []string{pc.cfg.Pkg.Metadata.Architecture, pc.cfg.Pkg.Build.Architecture}, - RegistryOverrides: pc.cfg.CreateOpts.RegistryOverrides, + Architectures: []string{loadedPkg.Metadata.Architecture, loadedPkg.Build.Architecture}, + RegistryOverrides: pc.createOpts.RegistryOverrides, } pulled, err = imgConfig.PullAll() @@ -196,10 +204,10 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths) error { } // Output assumes it is running from cwd, not the build directory -func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { +func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range pc.cfg.Pkg.Components { + for _, component := range loadedPkg.Components { // Make the component a tar archive if err := dst.Components.Archive(component, true); err != nil { return fmt.Errorf("unable to archive component: %s", err.Error()) @@ -211,30 +219,30 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } - pc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + loadedPkg.Metadata.AggregateChecksum = checksumChecksum // Record the migrations that will be ran on the package. - pc.cfg.Pkg.Build.Migrations = []string{ + loadedPkg.Build.Migrations = []string{ deprecated.ScriptsToActionsMigrated, deprecated.PluralizeSetVariable, } // Save the transformed config. - if err := utils.WriteYaml(dst.ZarfYAML, pc.cfg.Pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } // Sign the config file if a key was provided - if pc.cfg.CreateOpts.SigningKeyPath != "" { - if err := dst.SignPackage(pc.cfg.CreateOpts.SigningKeyPath, pc.cfg.CreateOpts.SigningKeyPassword); err != nil { + if pc.createOpts.SigningKeyPath != "" { + if err := dst.SignPackage(pc.createOpts.SigningKeyPath, pc.createOpts.SigningKeyPassword); err != nil { return err } } // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. - if helpers.IsOCIURL(pc.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(pc.cfg.CreateOpts.Output, &pc.cfg.Pkg.Metadata, &pc.cfg.Pkg.Build) + if helpers.IsOCIURL(pc.createOpts.Output) { + ref, err := oci.ReferenceFromMetadata(pc.createOpts.Output, &loadedPkg.Metadata, &loadedPkg.Build) if err != nil { return err } @@ -243,7 +251,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { return err } - err = remote.PublishPackage(&pc.cfg.Pkg, dst, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(loadedPkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -258,20 +266,20 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.cfg.CreateOpts.Output, utils.GetPackageName(pc.cfg.Pkg, pc.cfg.CreateOpts.DifferentialData)) + packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(*loadedPkg, pc.createOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) // Create the package tarball. - if err := dst.ArchivePackage(packageName, pc.cfg.CreateOpts.MaxPackageSizeMB); err != nil { + if err := dst.ArchivePackage(packageName, pc.createOpts.MaxPackageSizeMB); err != nil { return fmt.Errorf("unable to archive package: %w", err) } } // Output the SBOM files into a directory if specified. - if pc.cfg.CreateOpts.ViewSBOM || pc.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := pc.cfg.CreateOpts.SBOMOutputDir + if pc.createOpts.ViewSBOM || pc.createOpts.SBOMOutputDir != "" { + outputSBOM := pc.createOpts.SBOMOutputDir var sbomDir string if err := dst.SBOMs.Unarchive(); err != nil { return fmt.Errorf("unable to unarchive SBOMs: %w", err) @@ -279,20 +287,49 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths) error { sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, pc.cfg.Pkg.Metadata.Name) + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, loadedPkg.Metadata.Name) if err != nil { return err } sbomDir = out } - if pc.cfg.CreateOpts.ViewSBOM { + if pc.createOpts.ViewSBOM { sbom.ViewSBOMFiles(sbomDir) } } return nil } +func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range pkg.Components { + componentPaths, err := layout.Components.Create(c) + if err != nil { + return nil, err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + pkg.Components = components + + extendedPkg = pkg + + return extendedPkg, nil +} + func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index a42a7e3d70..62a77a2c3b 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/layout" @@ -29,43 +30,58 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - cfg *types.PackagerConfig + createOpts types.ZarfCreateOptions } // LoadPackageDefinition loads and configure a zarf.yaml file during package create. -func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg *types.ZarfPackage, warnings []string, err error) { - configuredPkg, err := setPackageMetadata(&sc.cfg.Pkg, &sc.cfg.CreateOpts) +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { + var pkg types.ZarfPackage + + if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { + return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + } + + configuredPkg, err := setPackageMetadata(pkg, sc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, &sc.cfg.CreateOpts) + composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.createOpts.Flavor) if err != nil { return nil, nil, err } warnings = append(warnings, composeWarnings...) - pkg, err = processExtensions(composedPkg, &sc.cfg.CreateOpts, dst) + extendedPkg, err := sc.processExtensions(composedPkg, dst) if err != nil { return nil, nil, err } - return pkg, warnings, nil + loadedPkg = extendedPkg + + return loadedPkg, warnings, nil } // TODO: print warnings somewhere else in the skeleton create flow. -func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths) error { - for idx, component := range sc.cfg.Pkg.Components { - if err := sc.addComponent(idx, component, dst); err != nil { +func (sc *SkeletonCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { + var updatedComponents []types.ZarfComponent + + for _, component := range loadedPkg.Components { + c, err := sc.addComponent(component, dst) + if err != nil { return err } + updatedComponents = append(updatedComponents, *c) } + + loadedPkg.Components = updatedComponents + return nil } -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { - for _, component := range sc.cfg.Pkg.Components { +func (sc *SkeletonCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { + for _, component := range loadedPkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } @@ -75,30 +91,61 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths) error { if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } - sc.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + loadedPkg.Metadata.AggregateChecksum = checksumChecksum + + return utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400) +} + +func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range pkg.Components { + componentPaths, err := layout.Components.Create(c) + if err != nil { + return nil, err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + pkg.Components = components + + extendedPkg = pkg - return utils.WriteYaml(dst.ZarfYAML, sc.cfg.Pkg, 0400) + return extendedPkg, nil } -func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent, dst *layout.PackagePaths) error { +func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + updatedComponent = &component + componentPaths, err := dst.Components.Create(component) if err != nil { - return err + return nil, err } if component.DeprecatedCosignKeyPath != "" { dst := filepath.Join(componentPaths.Base, "cosign.pub") err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) if err != nil { - return err + return nil, err } - sc.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" + updatedComponent.DeprecatedCosignKeyPath = "cosign.pub" } // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - component.Actions.OnCreate.Defaults.Dir = "" + updatedComponent.Actions.OnCreate.Defaults.Dir = "" resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { for idx := range actions { @@ -107,10 +154,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent return actions } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + updatedComponent.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + updatedComponent.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + updatedComponent.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + updatedComponent.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) // If any helm charts are defined, process them. for chartIdx, chart := range component.Charts { @@ -121,10 +168,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent err := utils.CreatePathAndCopy(chart.LocalPath, dst) if err != nil { - return err + return nil, err } - sc.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel + updatedComponent.Charts[chartIdx].LocalPath = rel } for valuesIdx, path := range chart.ValuesFiles { @@ -133,10 +180,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent } rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) - sc.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel + updatedComponent.Charts[chartIdx].ValuesFiles[valuesIdx] = rel if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) + return nil, fmt.Errorf("unable to copy chart values file %s: %w", path, err) } } } @@ -154,32 +201,32 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent if file.ExtractPath != "" { if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + return nil, fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) } // Make sure dst reflects the actual file or directory. updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) if updatedExtractedFileOrDir != dst { if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) + return nil, fmt.Errorf(lang.ErrWritingFile, dst, err) } } } else { if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + return nil, fmt.Errorf("unable to copy file %s: %w", file.Source, err) } } // Change the source to the new relative source directory (any remote files will have been skipped above) - sc.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + updatedComponent.Files[filesIdx].Source = rel // Remove the extractPath from a skeleton since it will already extract it - sc.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" + updatedComponent.Files[filesIdx].ExtractPath = "" // Abort packaging on invalid shasum (if one is specified). if file.Shasum != "" { if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err + return nil, err } } @@ -201,10 +248,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent dst := filepath.Join(componentPaths.Base, rel) if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + return nil, fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) } - sc.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel + updatedComponent.DataInjections[dataIdx].Source = rel } spinner.Success() @@ -232,10 +279,10 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent spinner.Updatef("Copying manifest %s", path) if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) + return nil, fmt.Errorf("unable to copy manifest %s: %w", path, err) } - sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel + updatedComponent.Manifests[manifestIdx].Files[fileIdx] = rel } for kustomizeIdx, path := range manifest.Kustomizations { @@ -247,17 +294,17 @@ func (sc *SkeletonCreator) addComponent(index int, component types.ZarfComponent dst := filepath.Join(componentPaths.Base, rel) if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) + return nil, fmt.Errorf("unable to build kustomization %s: %w", path, err) } - sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) + updatedComponent.Manifests[manifestIdx].Files = append(updatedComponent.Manifests[manifestIdx].Files, rel) } // remove kustomizations - sc.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil + updatedComponent.Manifests[manifestIdx].Kustomizations = nil } spinner.Success() } - return nil + return updatedComponent, nil } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 3b55230e77..75068ad3b4 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,7 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (templatedPkg *types.ZarfPackage, warnings []string, err error) { +func FillActiveTemplate(pkg *types.ZarfPackage, setVariables map[string]string) (templatedPkg *types.ZarfPackage, warnings []string, err error) { templateMap := map[string]string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { @@ -28,7 +28,7 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti warnings = append(warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) } - _, present := createOpts.SetVariables[key] + _, present := setVariables[key] if !present && !config.CommonOptions.Confirm { setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ Name: key, @@ -36,14 +36,14 @@ func FillActiveTemplate(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti if err != nil { return err } - createOpts.SetVariables[key] = setVal + setVariables[key] = setVal } else if !present { // erroring out here return fmt.Errorf("template %q must be '--set' when using the '--confirm' flag", key) } } - for key, value := range createOpts.SetVariables { + for key, value := range setVariables { templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value } diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index cb4dfdd5f7..5ff19fa562 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -16,15 +16,17 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOptions) (*types.ZarfPackage, error) { +func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (configuredPkg *types.ZarfPackage, err error) { + configuredPkg = &pkg + now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. // Record the name of the user creating the package. if runtime.GOOS == "windows" { - pkg.Build.User = os.Getenv("USERNAME") + configuredPkg.Build.User = os.Getenv("USERNAME") } else { - pkg.Build.User = os.Getenv("USER") + configuredPkg.Build.User = os.Getenv("USER") } hostname, err := os.Hostname() @@ -32,40 +34,40 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts *types.ZarfCreateOpti return nil, err } - if utils.IsInitConfig(*pkg) { - pkg.Metadata.Version = config.CLIVersion + if utils.IsInitConfig(pkg) { + configuredPkg.Metadata.Version = config.CLIVersion } // Set package architecture if createOpts.IsSkeleton { - pkg.Metadata.Architecture = "skeleton" + configuredPkg.Metadata.Architecture = "skeleton" } - if pkg.Metadata.Architecture == "" { - pkg.Metadata.Architecture = config.GetArch() + if configuredPkg.Metadata.Architecture == "" { + configuredPkg.Metadata.Architecture = config.GetArch() } - pkg.Build.Architecture = pkg.Metadata.Architecture + configuredPkg.Build.Architecture = configuredPkg.Metadata.Architecture // Record the time of package creation. - pkg.Build.Timestamp = now.Format(time.RFC1123Z) + configuredPkg.Build.Timestamp = now.Format(time.RFC1123Z) // Record the Zarf Version the CLI was built with. - pkg.Build.Version = config.CLIVersion + configuredPkg.Build.Version = config.CLIVersion // Record the hostname of the package creation terminal. - pkg.Build.Terminal = hostname + configuredPkg.Build.Terminal = hostname // If the --differential flag was used, record that this is a differential package. if createOpts.DifferentialData.DifferentialPackagePath != "" { - pkg.Build.Differential = true + configuredPkg.Build.Differential = true } // Record the flavor of Zarf used to build this package (if any). - pkg.Build.Flavor = createOpts.Flavor + configuredPkg.Build.Flavor = createOpts.Flavor - pkg.Build.RegistryOverrides = createOpts.RegistryOverrides + configuredPkg.Build.RegistryOverrides = createOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. - pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + configuredPkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return pkg, nil + return configuredPkg, nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b63cfbef16..b971b39cee 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -10,11 +10,9 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/variables" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -31,13 +29,10 @@ func (p *Packager) DevDeploy() error { if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %w", err) - } - c := creator.New(p.cfg) + c := creator.New(p.cfg.CreateOpts) - pkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -51,9 +46,9 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - pkg.Components = p.getSelectedComponents() + loadedPkg.Components = p.getSelectedComponents() - if err := validate.Run(*pkg); err != nil { + if err := validate.Run(*loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } @@ -65,7 +60,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(p.layout); err != nil { + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index f2d908fd1d..e8c341f742 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -54,7 +54,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, &p.cfg.CreateOpts) + composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, p.cfg.CreateOpts.Flavor) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, &p.cfg.CreateOpts) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts.SetVariables) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index bc8241218d..6b95efcac3 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" @@ -90,24 +89,28 @@ func (p *Packager) Publish() (err error) { if err != nil { return err } + if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { return err } - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %w", err) - } - c := creator.New(p.cfg) - _, warnings, err := c.LoadPackageDefinition(p.layout) + + c := creator.New(p.cfg.CreateOpts) + + loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } p.warnings = append(p.warnings, warnings...) - if err := c.Assemble(p.layout); err != nil { + + if err := c.Assemble(loadedPkg, p.layout); err != nil { return err } - if err := c.Output(p.layout); err != nil { + + if err := c.Output(loadedPkg, p.layout); err != nil { return err } + + p.cfg.Pkg = *loadedPkg } else { if err := p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) diff --git a/src/test/e2e/24_variables_test.go b/src/test/e2e/24_variables_test.go index eacd883f41..c0f546ad83 100644 --- a/src/test/e2e/24_variables_test.go +++ b/src/test/e2e/24_variables_test.go @@ -64,7 +64,7 @@ func TestVariables(t *testing.T) { // Verify that the sensitive variable 'unicorn-land' was not printed to the screen require.NotContains(t, stdErr, "unicorn-land") - logText := e2e.GetLogFileContents(t, stdErr) + logText := e2e.GetLogFileContents(t, e2e.StripMessageFormatting(stdErr)) // Verify that the sensitive variable 'unicorn-land' was not included in the log require.NotContains(t, logText, "unicorn-land") From def07e28be7daea9733ca41721f5e93434256b10 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 15:36:43 -0600 Subject: [PATCH 064/172] Move confirm action logic to a utility function --- src/internal/packager/sbom/tools.go | 12 -- src/pkg/packager/create.go | 4 +- src/pkg/packager/deploy.go | 2 +- src/pkg/packager/interactive.go | 202 +++++++++++++--------------- src/pkg/packager/mirror.go | 3 +- src/pkg/utils/package.go | 48 ------- src/pkg/utils/zarf_package.go | 159 ++++++++++++++++++++++ 7 files changed, 259 insertions(+), 171 deletions(-) delete mode 100644 src/pkg/utils/package.go create mode 100644 src/pkg/utils/zarf_package.go diff --git a/src/internal/packager/sbom/tools.go b/src/internal/packager/sbom/tools.go index d9ef4ef1f0..bd46cc5d07 100644 --- a/src/internal/packager/sbom/tools.go +++ b/src/internal/packager/sbom/tools.go @@ -13,7 +13,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/defenseunicorns/zarf/src/types" ) // ViewSBOMFiles opens a browser to view the SBOM files and pauses for user input. @@ -55,14 +54,3 @@ func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { return packagePath, utils.CreatePathAndCopy(sourceDir, packagePath) } - -// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections) -func IsSBOMAble(pkg types.ZarfPackage) bool { - for _, c := range pkg.Components { - if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { - return true - } - } - - return false -} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 3b6254852b..016135dabb 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,7 +10,9 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -37,7 +39,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to validate package: %w", err) } - if !p.confirmAction(config.ZarfCreateStage) { + if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, *loadedPkg, p.cfg.PkgOpts) { return fmt.Errorf("package creation canceled") } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 120124b760..747df4c63f 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -62,7 +62,7 @@ func (p *Packager) Deploy() (err error) { } // Confirm the overall package deployment - if !p.confirmAction(config.ZarfDeployStage) { + if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { return fmt.Errorf("deployment cancelled") } diff --git a/src/pkg/packager/interactive.go b/src/pkg/packager/interactive.go index f338b00b90..8382bd05c2 100644 --- a/src/pkg/packager/interactive.go +++ b/src/pkg/packager/interactive.go @@ -4,111 +4,97 @@ // Package packager contains functions for interacting with, managing and deploying Zarf packages. package packager -import ( - "fmt" - "os" - "path/filepath" - - "github.com/AlecAivazis/survey/v2" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/pterm/pterm" -) - -func (p *Packager) confirmAction(stage string) (confirm bool) { - - pterm.Println() - message.HeaderInfof("📦 PACKAGE DEFINITION") - utils.ColorPrintYAML(p.cfg.Pkg, p.getPackageYAMLHints(stage), true) - - // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package - if stage == config.ZarfDeployStage { - if sbom.IsSBOMAble(p.cfg.Pkg) { - // Print the location that the user can view the package SBOMs from - message.HorizontalRule() - message.Title("Software Bill of Materials", "an inventory of all software contained in this package") - - if len(p.sbomViewFiles) > 0 { - cwd, _ := os.Getwd() - link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, layout.SBOMDir, filepath.Base(p.sbomViewFiles[0])))) - inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", p.cfg.PkgOpts.PackageSource))) - - artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(p.sbomViewFiles)) + " to be reviewed. These are" - if len(p.sbomViewFiles) == 1 { - artifactMsg = pterm.Bold.Sprintf("%d artifact", len(p.sbomViewFiles)) + " to be reviewed. This is" - } - - msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) - viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) - viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) - - message.Note(msg) - pterm.Println(viewNow) - pterm.Println(viewLater) - } else { - message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") - } - } - } - - if len(p.warnings) > 0 { - message.HorizontalRule() - message.Title("Package Warnings", "the following warnings were flagged while reading the package") - for _, warning := range p.warnings { - message.Warn(warning) - } - } - - message.HorizontalRule() - - // Display prompt if not auto-confirmed - if config.CommonOptions.Confirm { - pterm.Println() - message.Successf("%s Zarf package confirmed", stage) - return config.CommonOptions.Confirm - } - - prompt := &survey.Confirm{ - Message: stage + " this Zarf package?", - } - - pterm.Println() - - // Prompt the user for confirmation, on abort return false - if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { - // User aborted or declined, cancel the action - return false - } - - return true -} - -func (p *Packager) getPackageYAMLHints(stage string) map[string]string { - hints := map[string]string{} - - if stage == config.ZarfDeployStage { - for _, variable := range p.cfg.Pkg.Variables { - value, present := p.cfg.PkgOpts.SetVariables[variable.Name] - if !present { - value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) - } else { - value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) - } - if variable.Sensitive { - value = "'**sanitized**'" - } - hints = utils.AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) - } - } - - hints = utils.AddRootHint(hints, "metadata", "information about this package\n") - hints = utils.AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") - hints = utils.AddRootHint(hints, "components", "definition of capabilities this package deploys") - hints = utils.AddRootHint(hints, "constants", "static values set by the package author") - hints = utils.AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") - - return hints -} +// func (p *Packager) confirmAction(stage string) (confirm bool) { + +// pterm.Println() +// message.HeaderInfof("📦 PACKAGE DEFINITION") +// utils.ColorPrintYAML(p.cfg.Pkg, p.getPackageYAMLHints(stage), true) + +// // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package +// if stage == config.ZarfDeployStage { +// if utils.IsSBOMAble(p.cfg.Pkg) { +// // Print the location that the user can view the package SBOMs from +// message.HorizontalRule() +// message.Title("Software Bill of Materials", "an inventory of all software contained in this package") + +// if len(p.sbomViewFiles) > 0 { +// cwd, _ := os.Getwd() +// link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, layout.SBOMDir, filepath.Base(p.sbomViewFiles[0])))) +// inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", p.cfg.PkgOpts.PackageSource))) + +// artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(p.sbomViewFiles)) + " to be reviewed. These are" +// if len(p.sbomViewFiles) == 1 { +// artifactMsg = pterm.Bold.Sprintf("%d artifact", len(p.sbomViewFiles)) + " to be reviewed. This is" +// } + +// msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) +// viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) +// viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) + +// message.Note(msg) +// pterm.Println(viewNow) +// pterm.Println(viewLater) +// } else { +// message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") +// } +// } +// } + +// if len(p.warnings) > 0 { +// message.HorizontalRule() +// message.Title("Package Warnings", "the following warnings were flagged while reading the package") +// for _, warning := range p.warnings { +// message.Warn(warning) +// } +// } + +// message.HorizontalRule() + +// // Display prompt if not auto-confirmed +// if config.CommonOptions.Confirm { +// pterm.Println() +// message.Successf("%s Zarf package confirmed", stage) +// return config.CommonOptions.Confirm +// } + +// prompt := &survey.Confirm{ +// Message: stage + " this Zarf package?", +// } + +// pterm.Println() + +// // Prompt the user for confirmation, on abort return false +// if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { +// // User aborted or declined, cancel the action +// return false +// } + +// return true +// } + +// func (p *Packager) getPackageYAMLHints(stage string) map[string]string { +// hints := map[string]string{} + +// if stage == config.ZarfDeployStage { +// for _, variable := range p.cfg.Pkg.Variables { +// value, present := p.cfg.PkgOpts.SetVariables[variable.Name] +// if !present { +// value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) +// } else { +// value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) +// } +// if variable.Sensitive { +// value = "'**sanitized**'" +// } +// hints = utils.AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) +// } +// } + +// hints = utils.AddRootHint(hints, "metadata", "information about this package\n") +// hints = utils.AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") +// hints = utils.AddRootHint(hints, "components", "definition of capabilities this package deploys") +// hints = utils.AddRootHint(hints, "constants", "static values set by the package author") +// hints = utils.AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") + +// return hints +// } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index ea210a5846..7819a01617 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -30,7 +31,7 @@ func (p *Packager) Mirror() (err error) { } // Confirm the overall package mirror - if !p.confirmAction(config.ZarfMirrorStage) { + if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { return fmt.Errorf("mirror cancelled") } diff --git a/src/pkg/utils/package.go b/src/pkg/utils/package.go deleted file mode 100644 index 982c4e5793..0000000000 --- a/src/pkg/utils/package.go +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/types" -) - -// IsInitConfig returns whether the provided Zarf package is an init config. -func IsInitConfig(pkg types.ZarfPackage) bool { - return pkg.Kind == types.ZarfInitConfig -} - -// GetInitPackageName returns the formatted name of the init package. -func GetInitPackageName(arch string) string { - if arch == "" { - // No package has been loaded yet so lookup GetArch() with no package info - arch = config.GetArch() - } - return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) -} - -// GetPackageName returns the formatted name of the package. -func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { - if IsInitConfig(pkg) { - return GetInitPackageName(pkg.Metadata.Architecture) - } - - packageName := pkg.Metadata.Name - suffix := "tar.zst" - if pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) - if pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) - } else if pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) - } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) -} diff --git a/src/pkg/utils/zarf_package.go b/src/pkg/utils/zarf_package.go new file mode 100644 index 0000000000..b1af40d2dd --- /dev/null +++ b/src/pkg/utils/zarf_package.go @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic utility functions. +package utils + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" + "github.com/pterm/pterm" +) + +// IsInitConfig returns whether the provided Zarf package is an init config. +func IsInitConfig(pkg types.ZarfPackage) bool { + return pkg.Kind == types.ZarfInitConfig +} + +// GetInitPackageName returns the formatted name of the init package. +func GetInitPackageName(arch string) string { + if arch == "" { + // No package has been loaded yet so lookup GetArch() with no package info + arch = config.GetArch() + } + return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) +} + +// GetPackageName returns the formatted name of the package. +func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { + if IsInitConfig(pkg) { + return GetInitPackageName(pkg.Metadata.Architecture) + } + + packageName := pkg.Metadata.Name + suffix := "tar.zst" + if pkg.Metadata.Uncompressed { + suffix = "tar" + } + + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) + if pkg.Build.Differential { + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) + } else if pkg.Metadata.Version != "" { + packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) + } + + return fmt.Sprintf("%s.%s", packageFileName, suffix) +} + +// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections) +func IsSBOMAble(pkg types.ZarfPackage) bool { + for _, c := range pkg.Components { + if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { + return true + } + } + + return false +} + +func ConfirmAction(stage, sbomDir string, sbomViewFiles, warnings []string, pkg types.ZarfPackage, pkgOpts types.ZarfPackageOptions) (confirm bool) { + + pterm.Println() + message.HeaderInfof("📦 PACKAGE DEFINITION") + ColorPrintYAML(pkg, getPackageYAMLHints(stage, pkg.Variables, pkgOpts.SetVariables), true) + + // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package + if stage == config.ZarfDeployStage { + if IsSBOMAble(pkg) { + // Print the location that the user can view the package SBOMs from + message.HorizontalRule() + message.Title("Software Bill of Materials", "an inventory of all software contained in this package") + + if len(sbomViewFiles) > 0 { + cwd, _ := os.Getwd() + link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, sbomDir, filepath.Base(sbomViewFiles[0])))) + inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", pkgOpts.PackageSource))) + + artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(sbomViewFiles)) + " to be reviewed. These are" + if len(sbomViewFiles) == 1 { + artifactMsg = pterm.Bold.Sprintf("%d artifact", len(sbomViewFiles)) + " to be reviewed. This is" + } + + msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) + viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) + viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) + + message.Note(msg) + pterm.Println(viewNow) + pterm.Println(viewLater) + } else { + message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") + } + } + } + + if len(warnings) > 0 { + message.HorizontalRule() + message.Title("Package Warnings", "the following warnings were flagged while reading the package") + for _, warning := range warnings { + message.Warn(warning) + } + } + + message.HorizontalRule() + + // Display prompt if not auto-confirmed + if config.CommonOptions.Confirm { + pterm.Println() + message.Successf("%s Zarf package confirmed", stage) + return config.CommonOptions.Confirm + } + + prompt := &survey.Confirm{ + Message: stage + " this Zarf package?", + } + + pterm.Println() + + // Prompt the user for confirmation, on abort return false + if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { + // User aborted or declined, cancel the action + return false + } + + return true +} + +func getPackageYAMLHints(stage string, pkgVars []types.ZarfPackageVariable, setVars map[string]string) map[string]string { + hints := map[string]string{} + + if stage == config.ZarfDeployStage { + for _, variable := range pkgVars { + value, present := setVars[variable.Name] + if !present { + value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) + } else { + value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) + } + if variable.Sensitive { + value = "'**sanitized**'" + } + hints = AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) + } + } + + hints = AddRootHint(hints, "metadata", "information about this package\n") + hints = AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") + hints = AddRootHint(hints, "components", "definition of capabilities this package deploys") + hints = AddRootHint(hints, "constants", "static values set by the package author") + hints = AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") + + return hints +} From 2524bcc0056fbe9f3e43de5a113796963b2afcf4 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 16:38:36 -0600 Subject: [PATCH 065/172] Move stage SBOM files logic to utils --- src/internal/packager/sbom/tools.go | 17 ----------------- src/pkg/layout/sbom.go | 29 ++++++++++++++++++++++------- src/pkg/packager/common.go | 21 --------------------- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/deploy.go | 7 +++++-- src/pkg/packager/inspect.go | 2 +- src/pkg/packager/mirror.go | 9 +++++++-- src/pkg/utils/sbom.go | 25 +++++++++++++++++++++++++ src/test/e2e/20_zarf_init_test.go | 2 +- 10 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 src/pkg/utils/sbom.go diff --git a/src/internal/packager/sbom/tools.go b/src/internal/packager/sbom/tools.go index bd46cc5d07..860b50e879 100644 --- a/src/internal/packager/sbom/tools.go +++ b/src/internal/packager/sbom/tools.go @@ -6,12 +6,10 @@ package sbom import ( "fmt" - "os" "path/filepath" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" ) @@ -39,18 +37,3 @@ func ViewSBOMFiles(directory string) { message.Note("There were no images with software bill-of-materials (SBOM) included.") } } - -// OutputSBOMFiles outputs the sbom files into a specified directory. -func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { - packagePath := filepath.Join(outputDir, packageName) - - if err := os.RemoveAll(packagePath); err != nil { - return "", err - } - - if err := utils.CreateDirectory(packagePath, 0700); err != nil { - return "", err - } - - return packagePath, utils.CreatePathAndCopy(sourceDir, packagePath) -} diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 13f7ee0fc1..5c16330a87 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -5,6 +5,7 @@ package layout import ( + "fmt" "io/fs" "os" "path/filepath" @@ -67,12 +68,26 @@ func (s *SBOMs) Archive() (err error) { return os.RemoveAll(dir) } -// IsDir returns true if the SBOMs are a directory. -func (s SBOMs) IsDir() bool { - return utils.IsDir(s.Path) -} +func (s *SBOMs) StageSBOMViewFiles() (sbomViewFiles, warnings []string, err error) { + isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" + if isTarball { + return nil, nil, fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", s.Path) + } + + // If SBOMs were loaded, temporarily place them in the deploy directory + if !utils.InvalidPath(s.Path) { + sbomViewFiles, err = filepath.Glob(filepath.Join(s.Path, "sbom-viewer-*")) + if err != nil { + return nil, nil, err + } + + _, err := utils.OutputSBOMFiles(s.Path, SBOMDir, "") + if err != nil { + // Don't stop the deployment, let the user decide if they want to continue the deployment + warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) + warnings = append(warnings, warning) + } + } -// IsTarball returns true if the SBOMs are a tarball. -func (s SBOMs) IsTarball() bool { - return !s.IsDir() && filepath.Ext(s.Path) == ".tar" + return sbomViewFiles, warnings, nil } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index bf162f7a1d..f3fa8019e8 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "os" - "path/filepath" "regexp" "strings" "time" @@ -17,7 +16,6 @@ import ( "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/types" @@ -40,7 +38,6 @@ type Packager struct { valueTemplate *template.Values hpaModified bool connectStrings types.ConnectStrings - sbomViewFiles []string source sources.PackageSource generation int } @@ -281,21 +278,3 @@ func (p *Packager) validateLastNonBreakingVersion() (err error) { return nil } - -func (p *Packager) stageSBOMViewFiles() error { - if p.layout.SBOMs.IsTarball() { - return fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", p.layout.SBOMs.Path) - } - // If SBOMs were loaded, temporarily place them in the deploy directory - sbomDir := p.layout.SBOMs.Path - if !utils.InvalidPath(sbomDir) { - p.sbomViewFiles, _ = filepath.Glob(filepath.Join(sbomDir, "sbom-viewer-*")) - _, err := sbom.OutputSBOMFiles(sbomDir, layout.SBOMDir, "") - if err != nil { - // Don't stop the deployment, let the user decide if they want to continue the deployment - warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) - p.warnings = append(p.warnings, warning) - } - } - return nil -} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 016135dabb..566d1f08ef 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -39,7 +39,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to validate package: %w", err) } - if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, *loadedPkg, p.cfg.PkgOpts) { + if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, []string{}, p.warnings, *loadedPkg, p.cfg.PkgOpts) { return fmt.Errorf("package creation canceled") } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 21cbc3a5be..769b5ae795 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -287,7 +287,7 @@ func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.Packa sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, loadedPkg.Metadata.Name) + out, err := utils.OutputSBOMFiles(sbomDir, outputSBOM, loadedPkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 747df4c63f..e61c48b9e1 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -57,12 +57,15 @@ func (p *Packager) Deploy() (err error) { return err } - if err := p.stageSBOMViewFiles(); err != nil { + sbomViewFiles, sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + if err != nil { return err } + p.warnings = append(p.warnings, sbomWarnings...) + // Confirm the overall package deployment - if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { + if !utils.ConfirmAction(config.ZarfDeployStage, layout.SBOMDir, sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { return fmt.Errorf("deployment cancelled") } diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index e666434eb3..6f9f7898fa 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -26,7 +26,7 @@ func (p *Packager) Inspect() (err error) { sbomDir := p.layout.SBOMs.Path if p.cfg.InspectOpts.SBOMOutputDir != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) + out, err := utils.OutputSBOMFiles(sbomDir, p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 7819a01617..93f0b9f944 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -23,15 +23,20 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } + if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } - if err := p.stageSBOMViewFiles(); err != nil { + + sbomViewFiles, sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + if err != nil { return err } + p.warnings = append(p.warnings, sbomWarnings...) + // Confirm the overall package mirror - if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, p.sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { + if !utils.ConfirmAction(config.ZarfMirrorStage, layout.SBOMDir, sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { return fmt.Errorf("mirror cancelled") } diff --git a/src/pkg/utils/sbom.go b/src/pkg/utils/sbom.go new file mode 100644 index 0000000000..7c5e2a1252 --- /dev/null +++ b/src/pkg/utils/sbom.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic helper functions. +package utils + +import ( + "os" + "path/filepath" +) + +// OutputSBOMFiles outputs the sbom files into a specified directory. +func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { + packagePath := filepath.Join(outputDir, packageName) + + if err := os.RemoveAll(packagePath); err != nil { + return "", err + } + + if err := CreateDirectory(packagePath, 0700); err != nil { + return "", err + } + + return packagePath, CreatePathAndCopy(sourceDir, packagePath) +} diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 86f127fc83..13c9dd4f48 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -73,7 +73,7 @@ func TestZarfInit(t *testing.T) { require.NoError(t, err) require.Contains(t, initStdErr, "an inventory of all software contained in this package") - logText := e2e.GetLogFileContents(t, initStdErr) + logText := e2e.GetLogFileContents(t, e2e.StripMessageFormatting(initStdErr)) // Verify that any state secrets were not included in the log state := types.ZarfState{} From 1867cea4b561e3c8334d663121b0cdacaebfc41c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 20:11:40 -0600 Subject: [PATCH 066/172] Fix dev deploy --- src/pkg/packager/dev.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index b971b39cee..d32202ead1 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -38,6 +38,8 @@ func (p *Packager) DevDeploy() error { } p.warnings = append(p.warnings, warnings...) + p.cfg.Pkg = *loadedPkg + // Filter out components that are not compatible with this system p.filterComponents() @@ -46,7 +48,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - loadedPkg.Components = p.getSelectedComponents() + p.getSelectedComponents() if err := validate.Run(*loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) @@ -60,7 +62,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(&p.cfg.Pkg, p.layout); err != nil { return err } From fb789d737f41cb6a97874967f620efdbd4e9ce84 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 20:16:46 -0600 Subject: [PATCH 067/172] Remove creator.CdToBaseDir Set differential package path in creator.New instead --- src/pkg/packager/create.go | 12 ++++++++---- src/pkg/packager/creator/io.go | 28 ---------------------------- src/pkg/packager/creator/new.go | 10 +++++++++- src/pkg/packager/dev.go | 10 +++++----- src/pkg/packager/publish.go | 10 +++++----- 5 files changed, 27 insertions(+), 43 deletions(-) delete mode 100644 src/pkg/packager/creator/io.go diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 566d1f08ef..e39dc077db 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -11,21 +11,25 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) + } + + message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) + cwd, err := os.Getwd() if err != nil { return err } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { - return err - } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg.CreateOpts, cwd) loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { diff --git a/src/pkg/packager/creator/io.go b/src/pkg/packager/creator/io.go deleted file mode 100644 index 0ed6bebc0f..0000000000 --- a/src/pkg/packager/creator/io.go +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" -) - -// CdToBaseDir changes into the specified base directory. -func CdToBaseDir(createOpts *types.ZarfCreateOptions, cwd string) error { - if err := os.Chdir(createOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory %q: %w", createOpts.BaseDir, err) - } - message.Note(fmt.Sprintf("Using build directory %s", createOpts.BaseDir)) - - // differentials are relative to the current working directory - if createOpts.DifferentialData.DifferentialPackagePath != "" { - createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 230492270f..24af91e4f1 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -5,6 +5,8 @@ package creator import ( + "path/filepath" + "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ) @@ -17,9 +19,15 @@ type Creator interface { } // New returns a new Creator based on the provided create options. -func New(createOpts types.ZarfCreateOptions) Creator { +func New(createOpts types.ZarfCreateOptions, cwd string) Creator { if createOpts.IsSkeleton { return &SkeletonCreator{createOpts: createOpts} } + + // differentials are relative to the current working directory + if createOpts.DifferentialData.DifferentialPackagePath != "" { + createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + } + return &PackageCreator{createOpts: createOpts} } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index d32202ead1..dc1b5eba1b 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -21,16 +21,16 @@ func (p *Packager) DevDeploy() error { config.CommonOptions.Confirm = true p.cfg.CreateOpts.SkipSBOM = !p.cfg.CreateOpts.NoYOLO - cwd, err := os.Getwd() - if err != nil { - return err + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + cwd, err := os.Getwd() + if err != nil { return err } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg.CreateOpts, cwd) loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6b95efcac3..43974d5aa8 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -85,16 +85,16 @@ func (p *Packager) Publish() (err error) { } if p.cfg.CreateOpts.IsSkeleton { - cwd, err := os.Getwd() - if err != nil { - return err + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - if err := creator.CdToBaseDir(&p.cfg.CreateOpts, cwd); err != nil { + cwd, err := os.Getwd() + if err != nil { return err } - c := creator.New(p.cfg.CreateOpts) + c := creator.New(p.cfg.CreateOpts, cwd) loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { From f36dda8cfbb01b56377df7ded9476cda08a3a1c5 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 20:53:26 -0600 Subject: [PATCH 068/172] Get cwd before cd into create dir --- src/pkg/packager/create.go | 10 +++++----- src/pkg/packager/dev.go | 9 +++++---- src/pkg/packager/publish.go | 8 ++------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index e39dc077db..a56cf6df89 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -18,17 +18,17 @@ import ( // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { + cwd, err := os.Getwd() + if err != nil { + return err + } + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - cwd, err := os.Getwd() - if err != nil { - return err - } - c := creator.New(p.cfg.CreateOpts, cwd) loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index dc1b5eba1b..607493543d 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -21,21 +21,22 @@ func (p *Packager) DevDeploy() error { config.CommonOptions.Confirm = true p.cfg.CreateOpts.SkipSBOM = !p.cfg.CreateOpts.NoYOLO - if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) - } - cwd, err := os.Getwd() if err != nil { return err } + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) + } + c := creator.New(p.cfg.CreateOpts, cwd) loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) p.cfg.Pkg = *loadedPkg diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 43974d5aa8..57a9013278 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -89,17 +89,13 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - cwd, err := os.Getwd() - if err != nil { - return err - } - - c := creator.New(p.cfg.CreateOpts, cwd) + c := creator.New(p.cfg.CreateOpts, "") loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) if err != nil { return err } + p.warnings = append(p.warnings, warnings...) if err := c.Assemble(loadedPkg, p.layout); err != nil { From 10c7bb1ec3857ffcff5e04b7296e40a81385109b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 21:19:14 -0600 Subject: [PATCH 069/172] Move GenerateChecksums to *layout.PackagePaths receiver --- src/pkg/layout/package.go | 31 +++++++++++++++++++ src/pkg/packager/creator/checksums.go | 43 --------------------------- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/creator/skeleton.go | 2 +- 4 files changed, 33 insertions(+), 45 deletions(-) delete mode 100644 src/pkg/packager/creator/checksums.go diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 8e2dab301b..fe72ff1584 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "github.com/Masterminds/semver/v3" @@ -168,6 +169,36 @@ func (pp *PackagePaths) addSignature(keyPath string) *PackagePaths { return pp } +// GenerateChecksums walks through all of the files starting at the base path and generates a checksum file. +// +// Each file within the basePath represents a layer within the Zarf package. +// +// Returns a SHA256 checksum of the checksums.txt file. +func (pp *PackagePaths) GenerateChecksums() (string, error) { + var checksumsData = []string{} + + for rel, abs := range pp.Files() { + if rel == ZarfYAML || rel == Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) + } + slices.Sort(checksumsData) + + // Create the checksums file + if err := utils.WriteFile(pp.Checksums, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(pp.Checksums) +} + func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSizeMB int) error { spinner := message.NewProgressSpinner("Writing %s to %s", pp.Base, destinationTarball) defer spinner.Stop() diff --git a/src/pkg/packager/creator/checksums.go b/src/pkg/packager/creator/checksums.go deleted file mode 100644 index ddfde66486..0000000000 --- a/src/pkg/packager/creator/checksums.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package creator contains functions for creating Zarf packages. -package creator - -import ( - "fmt" - "slices" - "strings" - - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/utils" -) - -// generateChecksums walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksums returns a SHA256 checksum of the checksums.txt file. -func generateChecksums(dst *layout.PackagePaths) (string, error) { - // Loop over the "loaded" files - var checksumsData = []string{} - for rel, abs := range dst.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) - } - slices.Sort(checksumsData) - - // Create the checksums file - checksumsFilePath := dst.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 769b5ae795..60407ae11a 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -215,7 +215,7 @@ func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.Packa } // Calculate all the checksums - checksumChecksum, err := generateChecksums(dst) + checksumChecksum, err := dst.GenerateChecksums() if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 62a77a2c3b..ccbc0796cc 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -87,7 +87,7 @@ func (sc *SkeletonCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.Pack } } - checksumChecksum, err := generateChecksums(dst) + checksumChecksum, err := dst.GenerateChecksums() if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } From 1768143459de82ea80522a54ad7079d8475a901c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 31 Jan 2024 21:46:13 -0600 Subject: [PATCH 070/172] Delete interactive.go file in packager --- src/pkg/packager/interactive.go | 100 -------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 src/pkg/packager/interactive.go diff --git a/src/pkg/packager/interactive.go b/src/pkg/packager/interactive.go deleted file mode 100644 index 8382bd05c2..0000000000 --- a/src/pkg/packager/interactive.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -// func (p *Packager) confirmAction(stage string) (confirm bool) { - -// pterm.Println() -// message.HeaderInfof("📦 PACKAGE DEFINITION") -// utils.ColorPrintYAML(p.cfg.Pkg, p.getPackageYAMLHints(stage), true) - -// // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package -// if stage == config.ZarfDeployStage { -// if utils.IsSBOMAble(p.cfg.Pkg) { -// // Print the location that the user can view the package SBOMs from -// message.HorizontalRule() -// message.Title("Software Bill of Materials", "an inventory of all software contained in this package") - -// if len(p.sbomViewFiles) > 0 { -// cwd, _ := os.Getwd() -// link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, layout.SBOMDir, filepath.Base(p.sbomViewFiles[0])))) -// inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", p.cfg.PkgOpts.PackageSource))) - -// artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(p.sbomViewFiles)) + " to be reviewed. These are" -// if len(p.sbomViewFiles) == 1 { -// artifactMsg = pterm.Bold.Sprintf("%d artifact", len(p.sbomViewFiles)) + " to be reviewed. This is" -// } - -// msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) -// viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) -// viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) - -// message.Note(msg) -// pterm.Println(viewNow) -// pterm.Println(viewLater) -// } else { -// message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") -// } -// } -// } - -// if len(p.warnings) > 0 { -// message.HorizontalRule() -// message.Title("Package Warnings", "the following warnings were flagged while reading the package") -// for _, warning := range p.warnings { -// message.Warn(warning) -// } -// } - -// message.HorizontalRule() - -// // Display prompt if not auto-confirmed -// if config.CommonOptions.Confirm { -// pterm.Println() -// message.Successf("%s Zarf package confirmed", stage) -// return config.CommonOptions.Confirm -// } - -// prompt := &survey.Confirm{ -// Message: stage + " this Zarf package?", -// } - -// pterm.Println() - -// // Prompt the user for confirmation, on abort return false -// if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { -// // User aborted or declined, cancel the action -// return false -// } - -// return true -// } - -// func (p *Packager) getPackageYAMLHints(stage string) map[string]string { -// hints := map[string]string{} - -// if stage == config.ZarfDeployStage { -// for _, variable := range p.cfg.Pkg.Variables { -// value, present := p.cfg.PkgOpts.SetVariables[variable.Name] -// if !present { -// value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) -// } else { -// value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) -// } -// if variable.Sensitive { -// value = "'**sanitized**'" -// } -// hints = utils.AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) -// } -// } - -// hints = utils.AddRootHint(hints, "metadata", "information about this package\n") -// hints = utils.AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") -// hints = utils.AddRootHint(hints, "components", "definition of capabilities this package deploys") -// hints = utils.AddRootHint(hints, "constants", "static values set by the package author") -// hints = utils.AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") - -// return hints -// } From 068101af706d0e96feb03186b23d4705a32e8230 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 12:25:16 -0600 Subject: [PATCH 071/172] Add comment to ComposeComponents() --- src/pkg/packager/creator/compose.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index ce91a467dd..dabe285c6e 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,6 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// ComposeComponents composes components and their dependencies into a single zarf.yaml using an import chain. func ComposeComponents(pkg *types.ZarfPackage, flavor string) (composedPkg *types.ZarfPackage, warnings []string, err error) { components := []types.ZarfComponent{} From e84dd48718fbcc1874c48e96160c93be2e3aad90 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 13:17:51 -0600 Subject: [PATCH 072/172] Add named returns to differential functions --- src/pkg/packager/creator/differential.go | 16 ++++++++++------ src/pkg/packager/creator/normal.go | 8 ++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 3fa65451e2..f9d50be513 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -23,7 +23,7 @@ import ( ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func loadDifferentialData(diffData *types.DifferentialData) (*types.DifferentialData, error) { +func loadDifferentialData(diffData *types.DifferentialData) (loadedDiffData *types.DifferentialData, err error) { tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return nil, err @@ -72,11 +72,13 @@ func loadDifferentialData(diffData *types.DifferentialData) (*types.Differential diffData.DifferentialRepos = allIncludedReposMap diffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - return diffData, nil + loadedDiffData = diffData + + return loadedDiffData, nil } // removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types.DifferentialData) (*types.ZarfPackage, error) { +func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, loadedDiffData *types.DifferentialData) (diffPkg *types.ZarfPackage, err error) { // Loop through all of the components to determine if any of them are using already included images or repos componentMap := make(map[int]types.ZarfComponent) for idx, component := range pkg.Components { @@ -93,7 +95,7 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types // Only include new images or images that have a commonly overwritten tag imgTag := imgRef.TagOrDigest useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !diffData.DifferentialImages[img] { + if useImgAnyways || !loadedDiffData.DifferentialImages[img] { newImageList = append(newImageList, img) } else { message.Debugf("Image %s is already included in the differential package", img) @@ -116,7 +118,7 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types // Only include new repos or repos that were not referenced by a specific commit sha or tag useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !diffData.DifferentialRepos[repoURL] { + if useRepoAnyways || !loadedDiffData.DifferentialRepos[repoURL] { newRepoList = append(newRepoList, repoURL) } else { message.Debugf("Repo %s is already included in the differential package", repoURL) @@ -134,5 +136,7 @@ func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, diffData *types pkg.Components[idx] = component } - return pkg, nil + diffPkg = pkg + + return diffPkg, nil } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 60407ae11a..3707ac643e 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -81,22 +81,22 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade // If we are creating a differential package, remove duplicate images and repos. if extendedPkg.Build.Differential { - diffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) + loadedDiffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) if err != nil { return nil, nil, err } - versionsMatch := diffData.DifferentialPackageVersion == extendedPkg.Metadata.Version + versionsMatch := loadedDiffData.DifferentialPackageVersion == extendedPkg.Metadata.Version if versionsMatch { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := diffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" + noVersionSet := loadedDiffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" if noVersionSet { return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } - diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, diffData) + diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, loadedDiffData) if err != nil { return nil, nil, err } From ec0dc95b7eec8bace2113729a3f6342d7a1558ba Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 13:26:27 -0600 Subject: [PATCH 073/172] Add comments to exported functions in normal.go --- src/pkg/packager/creator/normal.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 3707ac643e..02a7428296 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -106,6 +106,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade return extendedPkg, warnings, nil } +// Assemble assembles all of the package assets into Zarf's tmp directory layout. func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { var imageList []transform.Image @@ -203,7 +204,8 @@ func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.Pac return nil } -// Output assumes it is running from cwd, not the build directory +// Output writes the Zarf package as a tarball to a local directory, +// or an OCI registry based on the --output flag. func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging From 5e5cccf3d96ab69e9839942113e1c5b04faeca09 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 13:45:39 -0600 Subject: [PATCH 074/172] Change SetBaseDirectory() to take in createOpts instead of pkgerConfig --- src/cmd/common/utils.go | 6 +++--- src/cmd/dev.go | 6 +++--- src/cmd/package.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go index 01a6b104d6..6b8eba9e0b 100644 --- a/src/cmd/common/utils.go +++ b/src/cmd/common/utils.go @@ -9,10 +9,10 @@ import ( ) // SetBaseDirectory sets base directory on package config when given in args -func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { +func SetBaseDirectory(args []string, createOpts *types.ZarfCreateOptions) { if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] + createOpts.BaseDir = args[0] } else { - pkgConfig.CreateOpts.BaseDir = "." + createOpts.BaseDir = "." } } diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 63426692ea..8e483ba2b5 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -40,7 +40,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig.CreateOpts) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -186,7 +186,7 @@ var devFindImagesCmd = &cobra.Command{ Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - common.SetBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig.CreateOpts) // Ensure uppercase keys from viper v := common.GetViper() @@ -232,7 +232,7 @@ var devLintCmd = &cobra.Command{ Short: lang.CmdDevLintShort, Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig.CreateOpts) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) diff --git a/src/cmd/package.go b/src/cmd/package.go index 1ccd3e3061..de1aefe3f3 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -40,7 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + common.SetBaseDirectory(args, &pkgConfig.CreateOpts) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { From b92d045a3603af12069d64047183bff844db4d1d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 14:31:42 -0600 Subject: [PATCH 075/172] Add comments to exported functions in skeleton.go --- src/pkg/packager/creator/normal.go | 16 +++++++-- src/pkg/packager/creator/skeleton.go | 50 +++++++++++++++++----------- src/pkg/packager/publish.go | 14 ++++---- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 02a7428296..5cb601e227 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -204,8 +204,18 @@ func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.Pac return nil } -// Output writes the Zarf package as a tarball to a local directory, -// or an OCI registry based on the --output flag. +// Output does the following: +// +// - archives components +// +// - generates checksums for all package files +// +// - writes the loaded zarf.yaml to disk +// +// - signs the package +// +// - writes the Zarf package as a tarball to a local directory, +// or an OCI registry based on the --output flag func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging @@ -234,7 +244,7 @@ func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.Packa return fmt.Errorf("unable to write zarf.yaml: %w", err) } - // Sign the config file if a key was provided + // Sign the package if a key has been provided if pc.createOpts.SigningKeyPath != "" { if err := dst.SignPackage(pc.createOpts.SigningKeyPath, pc.createOpts.SigningKeyPassword); err != nil { return err diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index ccbc0796cc..7e115c3d60 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/extensions/bigbang" "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -30,10 +29,11 @@ var ( // SkeletonCreator provides methods for creating skeleton Zarf packages. type SkeletonCreator struct { - createOpts types.ZarfCreateOptions + createOpts types.ZarfCreateOptions + publishOpts types.ZarfPublishOptions } -// LoadPackageDefinition loads and configure a zarf.yaml file during package create. +// LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { var pkg types.ZarfPackage @@ -58,12 +58,18 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load return nil, nil, err } + for _, warning := range warnings { + message.Warn(warning) + } + loadedPkg = extendedPkg return loadedPkg, warnings, nil } -// TODO: print warnings somewhere else in the skeleton create flow. +// Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. +// +// It processes each component to ensure correct structure and resource locations. func (sc *SkeletonCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { var updatedComponents []types.ZarfComponent @@ -80,6 +86,15 @@ func (sc *SkeletonCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.Pa return nil } +// Output does the following: +// +// - archives components +// +// - generates checksums for all package files +// +// - writes the loaded zarf.yaml to disk +// +// - signs the package func (sc *SkeletonCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { for _, component := range loadedPkg.Components { if err := dst.Components.Archive(component, false); err != nil { @@ -93,7 +108,18 @@ func (sc *SkeletonCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.Pack } loadedPkg.Metadata.AggregateChecksum = checksumChecksum - return utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400) + if err := utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400); err != nil { + return err + } + + // Sign the package if a key has been provided + if sc.publishOpts.SigningKeyPath != "" { + if err := dst.SignPackage(sc.publishOpts.SigningKeyPath, sc.publishOpts.SigningKeyPassword); err != nil { + return err + } + } + + return nil } func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { @@ -285,20 +311,6 @@ func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layo updatedComponent.Manifests[manifestIdx].Files[fileIdx] = rel } - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return nil, fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - - updatedComponent.Manifests[manifestIdx].Files = append(updatedComponent.Manifests[manifestIdx].Files, rel) - } // remove kustomizations updatedComponent.Manifests[manifestIdx].Kustomizations = nil } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 57a9013278..54b8072337 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -114,6 +114,13 @@ func (p *Packager) Publish() (err error) { if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { return err } + + // Sign the package if a key has been provided + if p.cfg.PublishOpts.SigningKeyPath != "" { + if err := p.layout.SignPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { + return err + } + } } // Get a reference to the registry for this package @@ -127,13 +134,6 @@ func (p *Packager) Publish() (err error) { return err } - // Sign the package if a key has been provided - if p.cfg.PublishOpts.SigningKeyPath != "" { - if err := p.layout.SignPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { - return err - } - } - message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref) // Publish the package/skeleton to the registry From bb658fcea4b41a865ea8fc792d1ee74bd3ab08ca Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 14:50:11 -0600 Subject: [PATCH 076/172] Add comment to FillActiveTemplate() --- src/pkg/packager/creator/template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 75068ad3b4..f840280dfc 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// FillActiveTemplate merges user-specified variables into the configuration templates of a zarf.yaml. func FillActiveTemplate(pkg *types.ZarfPackage, setVariables map[string]string) (templatedPkg *types.ZarfPackage, warnings []string, err error) { templateMap := map[string]string{} From db32f78e30eaf4f9276ed40a5448d08f0f0a5c20 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 14:56:56 -0600 Subject: [PATCH 077/172] Add comment to SignPackage and ArchivePackage --- src/pkg/layout/package.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index fe72ff1584..0e8ac4eb79 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -144,6 +144,7 @@ func (pp *PackagePaths) IsLegacyLayout() bool { return pp.isLegacyLayout } +// SignPackage signs the zarf.yaml in a Zarf package. func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { pp.addSignature(signingKeyPath) @@ -199,6 +200,7 @@ func (pp *PackagePaths) GenerateChecksums() (string, error) { return utils.GetSHA256OfFile(pp.Checksums) } +// ArchivePackage creates an archive for a Zarf package. func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSizeMB int) error { spinner := message.NewProgressSpinner("Writing %s to %s", pp.Base, destinationTarball) defer spinner.Stop() From 72a33186821cc2e1fffe934e5902640471717337 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 15:03:17 -0600 Subject: [PATCH 078/172] Add comment to StageSBOMViewFiles() --- src/pkg/layout/sbom.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 5c16330a87..ddd14a0577 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -68,6 +68,7 @@ func (s *SBOMs) Archive() (err error) { return os.RemoveAll(dir) } +// StageSBOMViewFiles writes SBOM files to disk. func (s *SBOMs) StageSBOMViewFiles() (sbomViewFiles, warnings []string, err error) { isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" if isTarball { From ee81fc4ba7051afb2fa3f94264f9af1662720137 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 15:08:22 -0600 Subject: [PATCH 079/172] Catch return value of getSelectedComponents in dev deploy --- src/pkg/packager/dev.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 607493543d..90101853d5 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -49,7 +49,7 @@ func (p *Packager) DevDeploy() error { // the user's selection and the component's `required` field // This is also different from regular package creation, where we still assemble and package up // all components and their dependencies, regardless of whether they are required or not - p.getSelectedComponents() + p.cfg.Pkg.Components = p.getSelectedComponents() if err := validate.Run(*loadedPkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) From b79329c2c8eb927c3b13c06a0676935f896f7489 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 2 Feb 2024 15:11:01 -0600 Subject: [PATCH 080/172] Use wrapped error in FindImages() --- src/pkg/packager/prepare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index e8c341f742..67d22b3e46 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -49,7 +49,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { - return nil, err + return nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) From 1b22b640b9a3c5061820033301ca760e3099c61b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 5 Feb 2024 16:31:41 -0600 Subject: [PATCH 081/172] Update comment for StageSBOMViewFiles --- src/pkg/layout/sbom.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index ddd14a0577..4fe8c75a78 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -68,7 +68,7 @@ func (s *SBOMs) Archive() (err error) { return os.RemoveAll(dir) } -// StageSBOMViewFiles writes SBOM files to disk. +// StageSBOMViewFiles writes SBOM viewer HTML files to disk. func (s *SBOMs) StageSBOMViewFiles() (sbomViewFiles, warnings []string, err error) { isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" if isTarball { From 90d7d7f71ebb98e7d88c3b7d1382cc2080228cfd Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 5 Feb 2024 16:52:53 -0600 Subject: [PATCH 082/172] Update comment for StageSBOMViewFiles --- src/pkg/layout/sbom.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 4fe8c75a78..23dbeb79fa 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -68,7 +68,7 @@ func (s *SBOMs) Archive() (err error) { return os.RemoveAll(dir) } -// StageSBOMViewFiles writes SBOM viewer HTML files to disk. +// StageSBOMViewFiles copies SBOM viewer HTML files to the Zarf SBOM directory. func (s *SBOMs) StageSBOMViewFiles() (sbomViewFiles, warnings []string, err error) { isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" if isTarball { From 0cbd40fb49be9188b4fe9cf9b593c21205d59b5b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 5 Feb 2024 17:40:38 -0600 Subject: [PATCH 083/172] Make PackagePaths first arg in all Creator methods --- src/pkg/packager/create.go | 4 ++-- src/pkg/packager/creator/new.go | 4 ++-- src/pkg/packager/creator/normal.go | 4 ++-- src/pkg/packager/creator/skeleton.go | 4 ++-- src/pkg/packager/dev.go | 2 +- src/pkg/packager/publish.go | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index a56cf6df89..468006356d 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -47,7 +47,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(p.layout, loadedPkg); err != nil { return err } @@ -56,5 +56,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output(loadedPkg, p.layout) + return c.Output(p.layout, loadedPkg) } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index 24af91e4f1..f5d04d56ba 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -14,8 +14,8 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) - Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error - Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error + Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error + Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error } // New returns a new Creator based on the provided create options. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 5cb601e227..e6cbf2fe06 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -107,7 +107,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade } // Assemble assembles all of the package assets into Zarf's tmp directory layout. -func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var imageList []transform.Image skipSBOMFlagUsed := pc.createOpts.SkipSBOM @@ -216,7 +216,7 @@ func (pc *PackageCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.Pac // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *PackageCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range loadedPkg.Components { diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 7e115c3d60..f3e17dfe8a 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -70,7 +70,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load // Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. // // It processes each component to ensure correct structure and resource locations. -func (sc *SkeletonCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var updatedComponents []types.ZarfComponent for _, component := range loadedPkg.Components { @@ -95,7 +95,7 @@ func (sc *SkeletonCreator) Assemble(loadedPkg *types.ZarfPackage, dst *layout.Pa // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *SkeletonCreator) Output(loadedPkg *types.ZarfPackage, dst *layout.PackagePaths) error { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { for _, component := range loadedPkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 90101853d5..762ef1d45a 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -63,7 +63,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(&p.cfg.Pkg, p.layout); err != nil { + if err := c.Assemble(p.layout, &p.cfg.Pkg); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 54b8072337..a3e0e26e2f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -98,11 +98,11 @@ func (p *Packager) Publish() (err error) { p.warnings = append(p.warnings, warnings...) - if err := c.Assemble(loadedPkg, p.layout); err != nil { + if err := c.Assemble(p.layout, loadedPkg); err != nil { return err } - if err := c.Output(loadedPkg, p.layout); err != nil { + if err := c.Output(p.layout, loadedPkg); err != nil { return err } From 20d435a86d88e8b3e7e1ea96700c943c4709c57b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 5 Feb 2024 17:46:51 -0600 Subject: [PATCH 084/172] Move filterComponents() to components.go and delete yaml.go --- src/pkg/packager/components.go | 32 +++++++++++++++++++++++++ src/pkg/packager/yaml.go | 43 ---------------------------------- 2 files changed, 32 insertions(+), 43 deletions(-) delete mode 100644 src/pkg/packager/yaml.go diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index d393eb3e59..c9faccf869 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -6,6 +6,7 @@ package packager import ( "path" + "runtime" "slices" "strings" @@ -24,6 +25,37 @@ const ( excluded ) +// filterComponents removes components not matching the current OS if filterByOS is set. +func (p *Packager) filterComponents() { + // Filter each component to only compatible platforms. + filteredComponents := []types.ZarfComponent{} + for _, component := range p.cfg.Pkg.Components { + // Ignore only filters that are empty + var validArch, validOS bool + + // Test for valid architecture + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.arch { + validArch = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.arch) + } + + // Test for a valid OS + if component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS { + validOS = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS) + } + + // If both the OS and architecture are valid, add the component to the filtered list + if validArch && validOS { + filteredComponents = append(filteredComponents, component) + } + } + // Update the active package with the filtered components. + p.cfg.Pkg.Components = filteredComponents +} + func (p *Packager) getSelectedComponents() []types.ZarfComponent { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go deleted file mode 100644 index 1d8573ebed..0000000000 --- a/src/pkg/packager/yaml.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "runtime" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" -) - -// filterComponents removes components not matching the current OS if filterByOS is set. -func (p *Packager) filterComponents() { - // Filter each component to only compatible platforms. - filteredComponents := []types.ZarfComponent{} - for _, component := range p.cfg.Pkg.Components { - // Ignore only filters that are empty - var validArch, validOS bool - - // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.arch { - validArch = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.arch) - } - - // Test for a valid OS - if component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS { - validOS = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS) - } - - // If both the OS and architecture are valid, add the component to the filtered list - if validArch && validOS { - filteredComponents = append(filteredComponents, component) - } - } - // Update the active package with the filtered components. - p.cfg.Pkg.Components = filteredComponents -} From 92b794249f455abe33f0d8d9c12d5451d9f05213 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 5 Feb 2024 18:02:36 -0600 Subject: [PATCH 085/172] Move confirm action logic back to packager --- src/pkg/layout/sbom.go | 14 ++-- src/pkg/packager/common.go | 1 + src/pkg/packager/create.go | 4 +- src/pkg/packager/deploy.go | 4 +- src/pkg/packager/interactive.go | 113 ++++++++++++++++++++++++++++++++ src/pkg/packager/mirror.go | 5 +- src/pkg/utils/zarf_package.go | 100 ---------------------------- 7 files changed, 125 insertions(+), 116 deletions(-) create mode 100644 src/pkg/packager/interactive.go diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 23dbeb79fa..20c77a440c 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -69,26 +69,24 @@ func (s *SBOMs) Archive() (err error) { } // StageSBOMViewFiles copies SBOM viewer HTML files to the Zarf SBOM directory. -func (s *SBOMs) StageSBOMViewFiles() (sbomViewFiles, warnings []string, err error) { +func (s *SBOMs) StageSBOMViewFiles() (warnings []string, err error) { isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" if isTarball { - return nil, nil, fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", s.Path) + return nil, fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", s.Path) } // If SBOMs were loaded, temporarily place them in the deploy directory if !utils.InvalidPath(s.Path) { - sbomViewFiles, err = filepath.Glob(filepath.Join(s.Path, "sbom-viewer-*")) - if err != nil { - return nil, nil, err + if _, err := filepath.Glob(filepath.Join(s.Path, "sbom-viewer-*")); err != nil { + return nil, err } - _, err := utils.OutputSBOMFiles(s.Path, SBOMDir, "") - if err != nil { + if _, err := utils.OutputSBOMFiles(s.Path, SBOMDir, ""); err != nil { // Don't stop the deployment, let the user decide if they want to continue the deployment warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) warnings = append(warnings, warning) } } - return sbomViewFiles, warnings, nil + return warnings, nil } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index f3fa8019e8..3a53528ccc 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -38,6 +38,7 @@ type Packager struct { valueTemplate *template.Values hpaModified bool connectStrings types.ConnectStrings + sbomViewFiles []string source sources.PackageSource generation int } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 468006356d..abef71b30f 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,10 +10,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" - "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. @@ -43,7 +41,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("unable to validate package: %w", err) } - if !utils.ConfirmAction(config.ZarfCreateStage, layout.SBOMDir, []string{}, p.warnings, *loadedPkg, p.cfg.PkgOpts) { + if !p.confirmAction(config.ZarfCreateStage) { return fmt.Errorf("package creation canceled") } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 380357bae9..54db488724 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -57,7 +57,7 @@ func (p *Packager) Deploy() (err error) { return err } - sbomViewFiles, sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() if err != nil { return err } @@ -65,7 +65,7 @@ func (p *Packager) Deploy() (err error) { p.warnings = append(p.warnings, sbomWarnings...) // Confirm the overall package deployment - if !utils.ConfirmAction(config.ZarfDeployStage, layout.SBOMDir, sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { + if !p.confirmAction(config.ZarfDeployStage) { return fmt.Errorf("deployment cancelled") } diff --git a/src/pkg/packager/interactive.go b/src/pkg/packager/interactive.go new file mode 100644 index 0000000000..dca7d3a648 --- /dev/null +++ b/src/pkg/packager/interactive.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/pterm/pterm" +) + +func (p *Packager) confirmAction(stage string) (confirm bool) { + + pterm.Println() + message.HeaderInfof("📦 PACKAGE DEFINITION") + utils.ColorPrintYAML(p.cfg.Pkg, p.getPackageYAMLHints(stage), true) + + // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package + if stage == config.ZarfDeployStage { + if utils.IsSBOMAble(p.cfg.Pkg) { + // Print the location that the user can view the package SBOMs from + message.HorizontalRule() + message.Title("Software Bill of Materials", "an inventory of all software contained in this package") + + if len(p.sbomViewFiles) > 0 { + cwd, _ := os.Getwd() + link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, layout.SBOMDir, filepath.Base(p.sbomViewFiles[0])))) + inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", p.cfg.PkgOpts.PackageSource))) + + artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(p.sbomViewFiles)) + " to be reviewed. These are" + if len(p.sbomViewFiles) == 1 { + artifactMsg = pterm.Bold.Sprintf("%d artifact", len(p.sbomViewFiles)) + " to be reviewed. This is" + } + + msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) + viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) + viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) + + message.Note(msg) + pterm.Println(viewNow) + pterm.Println(viewLater) + } else { + message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") + } + } + } + + if len(p.warnings) > 0 { + message.HorizontalRule() + message.Title("Package Warnings", "the following warnings were flagged while reading the package") + for _, warning := range p.warnings { + message.Warn(warning) + } + } + + message.HorizontalRule() + + // Display prompt if not auto-confirmed + if config.CommonOptions.Confirm { + pterm.Println() + message.Successf("%s Zarf package confirmed", stage) + return config.CommonOptions.Confirm + } + + prompt := &survey.Confirm{ + Message: stage + " this Zarf package?", + } + + pterm.Println() + + // Prompt the user for confirmation, on abort return false + if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { + // User aborted or declined, cancel the action + return false + } + + return true +} + +func (p *Packager) getPackageYAMLHints(stage string) map[string]string { + hints := map[string]string{} + + if stage == config.ZarfDeployStage { + for _, variable := range p.cfg.Pkg.Variables { + value, present := p.cfg.PkgOpts.SetVariables[variable.Name] + if !present { + value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) + } else { + value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) + } + if variable.Sensitive { + value = "'**sanitized**'" + } + hints = utils.AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) + } + } + + hints = utils.AddRootHint(hints, "metadata", "information about this package\n") + hints = utils.AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") + hints = utils.AddRootHint(hints, "components", "definition of capabilities this package deploys") + hints = utils.AddRootHint(hints, "constants", "static values set by the package author") + hints = utils.AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") + + return hints +} diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 93f0b9f944..972b033ed3 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -28,7 +27,7 @@ func (p *Packager) Mirror() (err error) { return err } - sbomViewFiles, sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() if err != nil { return err } @@ -36,7 +35,7 @@ func (p *Packager) Mirror() (err error) { p.warnings = append(p.warnings, sbomWarnings...) // Confirm the overall package mirror - if !utils.ConfirmAction(config.ZarfMirrorStage, layout.SBOMDir, sbomViewFiles, p.warnings, p.cfg.Pkg, p.cfg.PkgOpts) { + if !p.confirmAction(config.ZarfMirrorStage) { return fmt.Errorf("mirror cancelled") } diff --git a/src/pkg/utils/zarf_package.go b/src/pkg/utils/zarf_package.go index b1af40d2dd..95c9cf9084 100644 --- a/src/pkg/utils/zarf_package.go +++ b/src/pkg/utils/zarf_package.go @@ -6,14 +6,9 @@ package utils import ( "fmt" - "os" - "path/filepath" - "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" - "github.com/pterm/pterm" ) // IsInitConfig returns whether the provided Zarf package is an init config. @@ -62,98 +57,3 @@ func IsSBOMAble(pkg types.ZarfPackage) bool { return false } - -func ConfirmAction(stage, sbomDir string, sbomViewFiles, warnings []string, pkg types.ZarfPackage, pkgOpts types.ZarfPackageOptions) (confirm bool) { - - pterm.Println() - message.HeaderInfof("📦 PACKAGE DEFINITION") - ColorPrintYAML(pkg, getPackageYAMLHints(stage, pkg.Variables, pkgOpts.SetVariables), true) - - // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package - if stage == config.ZarfDeployStage { - if IsSBOMAble(pkg) { - // Print the location that the user can view the package SBOMs from - message.HorizontalRule() - message.Title("Software Bill of Materials", "an inventory of all software contained in this package") - - if len(sbomViewFiles) > 0 { - cwd, _ := os.Getwd() - link := pterm.FgLightCyan.Sprint(pterm.Bold.Sprint(filepath.Join(cwd, sbomDir, filepath.Base(sbomViewFiles[0])))) - inspect := pterm.BgBlack.Sprint(pterm.FgWhite.Sprint(pterm.Bold.Sprintf("$ zarf package inspect %s", pkgOpts.PackageSource))) - - artifactMsg := pterm.Bold.Sprintf("%d artifacts", len(sbomViewFiles)) + " to be reviewed. These are" - if len(sbomViewFiles) == 1 { - artifactMsg = pterm.Bold.Sprintf("%d artifact", len(sbomViewFiles)) + " to be reviewed. This is" - } - - msg := fmt.Sprintf("This package has %s available in a temporary '%s' folder in this directory and will be removed upon deployment.\n", artifactMsg, pterm.Bold.Sprint("zarf-sbom")) - viewNow := fmt.Sprintf("\n- View SBOMs %s by navigating to the '%s' folder or copying this link into a browser:\n%s", pterm.Bold.Sprint("now"), pterm.Bold.Sprint("zarf-sbom"), link) - viewLater := fmt.Sprintf("\n- View SBOMs %s deployment with this command:\n%s", pterm.Bold.Sprint("after"), inspect) - - message.Note(msg) - pterm.Println(viewNow) - pterm.Println(viewLater) - } else { - message.Warn("This package does NOT contain an SBOM. If you require an SBOM, please contact the creator of this package to request a version that includes an SBOM.") - } - } - } - - if len(warnings) > 0 { - message.HorizontalRule() - message.Title("Package Warnings", "the following warnings were flagged while reading the package") - for _, warning := range warnings { - message.Warn(warning) - } - } - - message.HorizontalRule() - - // Display prompt if not auto-confirmed - if config.CommonOptions.Confirm { - pterm.Println() - message.Successf("%s Zarf package confirmed", stage) - return config.CommonOptions.Confirm - } - - prompt := &survey.Confirm{ - Message: stage + " this Zarf package?", - } - - pterm.Println() - - // Prompt the user for confirmation, on abort return false - if err := survey.AskOne(prompt, &confirm); err != nil || !confirm { - // User aborted or declined, cancel the action - return false - } - - return true -} - -func getPackageYAMLHints(stage string, pkgVars []types.ZarfPackageVariable, setVars map[string]string) map[string]string { - hints := map[string]string{} - - if stage == config.ZarfDeployStage { - for _, variable := range pkgVars { - value, present := setVars[variable.Name] - if !present { - value = fmt.Sprintf("'%s' (default)", message.Truncate(variable.Default, 20, false)) - } else { - value = fmt.Sprintf("'%s'", message.Truncate(value, 20, false)) - } - if variable.Sensitive { - value = "'**sanitized**'" - } - hints = AddRootListHint(hints, "name", variable.Name, fmt.Sprintf("currently set to %s", value)) - } - } - - hints = AddRootHint(hints, "metadata", "information about this package\n") - hints = AddRootHint(hints, "build", "info about the machine, zarf version, and user that created this package\n") - hints = AddRootHint(hints, "components", "definition of capabilities this package deploys") - hints = AddRootHint(hints, "constants", "static values set by the package author") - hints = AddRootHint(hints, "variables", "deployment-specific values that are set on each package deployment") - - return hints -} From 6c6935c5bc5fd974a391cfc2869f70304f8817a8 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 6 Feb 2024 12:04:52 -0600 Subject: [PATCH 086/172] Assign p.cfg.Pkg to loadedPkg to populate packager before confirmAction is called --- src/pkg/packager/create.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index abef71b30f..d800b713e4 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -34,6 +34,7 @@ func (p *Packager) Create() (err error) { return err } + p.cfg.Pkg = *loadedPkg p.warnings = append(p.warnings, warnings...) // Perform early package validation. From 086ec1e2ef901525e10f43acc36a066343e98ccc Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 6 Feb 2024 12:13:56 -0600 Subject: [PATCH 087/172] Add comment to actions.Run() --- src/pkg/packager/actions/actions.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/actions/actions.go b/src/pkg/packager/actions/actions.go index 329db5cf6d..416c5aafd7 100644 --- a/src/pkg/packager/actions/actions.go +++ b/src/pkg/packager/actions/actions.go @@ -21,6 +21,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// Run runs all provided actions. func Run(cfg *types.PackagerConfig, defaultCfg types.ZarfComponentActionDefaults, actions []types.ZarfComponentAction, valueTemplate *template.Values) error { for _, a := range actions { if err := runAction(cfg, defaultCfg, a, valueTemplate); err != nil { From 7b97d157df4f41845bc30f52731f7faf08300a45 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 6 Feb 2024 14:09:01 -0600 Subject: [PATCH 088/172] Remove outdated comments from processExtensions --- src/pkg/packager/creator/normal.go | 3 --- src/pkg/packager/creator/skeleton.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e6cbf2fe06..f975e34d22 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -333,10 +333,7 @@ func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layo components = append(components, c) } - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. pkg.Components = components - extendedPkg = pkg return extendedPkg, nil diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index f3e17dfe8a..7f346a54e1 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -142,10 +142,7 @@ func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *lay components = append(components, c) } - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. pkg.Components = components - extendedPkg = pkg return extendedPkg, nil From 44e663ae18353ea785b9776d4fe6cd9da8db228b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 6 Feb 2024 14:18:22 -0600 Subject: [PATCH 089/172] Make Creator implementations private --- src/pkg/packager/creator/new.go | 4 ++-- src/pkg/packager/creator/normal.go | 22 ++++++++++++---------- src/pkg/packager/creator/skeleton.go | 18 +++++++++--------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/new.go index f5d04d56ba..b3970aeadf 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/new.go @@ -21,7 +21,7 @@ type Creator interface { // New returns a new Creator based on the provided create options. func New(createOpts types.ZarfCreateOptions, cwd string) Creator { if createOpts.IsSkeleton { - return &SkeletonCreator{createOpts: createOpts} + return &skeletonCreator{createOpts: createOpts} } // differentials are relative to the current working directory @@ -29,5 +29,5 @@ func New(createOpts types.ZarfCreateOptions, cwd string) Creator { createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) } - return &PackageCreator{createOpts: createOpts} + return &packageCreator{createOpts: createOpts} } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index f975e34d22..40dba6f256 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -34,18 +34,20 @@ import ( ) var ( - // veryify that PackageCreator implements Creator - _ Creator = (*PackageCreator)(nil) + // veryify that packageCreator implements Creator + _ Creator = (*packageCreator)(nil) ) // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. -type PackageCreator struct { +type packageCreator struct { createOpts types.ZarfCreateOptions - cfg *types.PackagerConfig + + // TODO: (@lucasrod16) remove PackagerConfig once actions do not depend on it: https://github.com/defenseunicorns/zarf/pull/2276 + cfg *types.PackagerConfig } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { +func (pc *packageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { var pkg types.ZarfPackage if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { @@ -107,7 +109,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade } // Assemble assembles all of the package assets into Zarf's tmp directory layout. -func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *packageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var imageList []transform.Image skipSBOMFlagUsed := pc.createOpts.SkipSBOM @@ -216,7 +218,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.Za // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *packageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range loadedPkg.Components { @@ -313,7 +315,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf return nil } -func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { +func (pc *packageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. @@ -339,7 +341,7 @@ func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layo return extendedPkg, nil } -func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { +func (pc *packageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) componentPaths, err := dst.Components.Create(component) @@ -519,7 +521,7 @@ func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layou return nil } -func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { +func (pc *packageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { componentPaths, err := dst.Components.Create(component) if err != nil { return nil, err diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 7f346a54e1..016c3b6dce 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -23,18 +23,18 @@ import ( ) var ( - // veryify that SkeletonCreator implements Creator - _ Creator = (*SkeletonCreator)(nil) + // veryify that skeletonCreator implements Creator + _ Creator = (*skeletonCreator)(nil) ) -// SkeletonCreator provides methods for creating skeleton Zarf packages. -type SkeletonCreator struct { +// skeletonCreator provides methods for creating skeleton Zarf packages. +type skeletonCreator struct { createOpts types.ZarfCreateOptions publishOpts types.ZarfPublishOptions } // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. -func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { +func (sc *skeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { var pkg types.ZarfPackage if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { @@ -70,7 +70,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load // Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. // // It processes each component to ensure correct structure and resource locations. -func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (sc *skeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var updatedComponents []types.ZarfComponent for _, component := range loadedPkg.Components { @@ -95,7 +95,7 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.Z // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (sc *skeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { for _, component := range loadedPkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err @@ -122,7 +122,7 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zar return nil } -func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { +func (sc *skeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. @@ -148,7 +148,7 @@ func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *lay return extendedPkg, nil } -func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { +func (sc *skeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) updatedComponent = &component From c2978b5420d538b21f8d620c273ca1d1a1260c3e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 6 Feb 2024 14:20:49 -0600 Subject: [PATCH 090/172] Update packageCreator comment --- src/pkg/packager/creator/normal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 40dba6f256..6c2a9c7c73 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -38,7 +38,7 @@ var ( _ Creator = (*packageCreator)(nil) ) -// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. +// packageCreator provides methods for creating normal (not skeleton) Zarf packages. type packageCreator struct { createOpts types.ZarfCreateOptions From 25021615e7f9c8524430b6967108fc8dc4b5873f Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 12:47:31 -0600 Subject: [PATCH 091/172] Move cfg.SetVariableMap functions to receiver methods on PackagerConfig --- src/pkg/packager/actions/actions.go | 5 ++-- src/pkg/packager/variables/variables.go | 32 ++++--------------------- src/types/packager.go | 24 +++++++++++++++++++ 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/pkg/packager/actions/actions.go b/src/pkg/packager/actions/actions.go index 416c5aafd7..4f41416d20 100644 --- a/src/pkg/packager/actions/actions.go +++ b/src/pkg/packager/actions/actions.go @@ -14,7 +14,6 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -113,8 +112,8 @@ retryCmd: // If an output variable is defined, set it. for _, v := range action.SetVariables { - variables.SetVariableInConfig(cfg, v.Name, out, v.Sensitive, v.AutoIndent, v.Type) - if err := variables.CheckVariablePattern(cfg, v.Name, v.Pattern); err != nil { + cfg.SetVariable(v.Name, out, v.Sensitive, v.AutoIndent, v.Type) + if err := cfg.CheckVariablePattern(v.Name, v.Pattern); err != nil { message.WarnErr(err, err.Error()) return err } diff --git a/src/pkg/packager/variables/variables.go b/src/pkg/packager/variables/variables.go index 3acc33d59f..a2ab07a2e3 100644 --- a/src/pkg/packager/variables/variables.go +++ b/src/pkg/packager/variables/variables.go @@ -5,9 +5,6 @@ package variables import ( - "fmt" - "regexp" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/types" @@ -16,7 +13,7 @@ import ( // SetVariableMapInConfig handles setting the active variables used to template component files. func SetVariableMapInConfig(cfg *types.PackagerConfig) error { for name, value := range cfg.PkgOpts.SetVariables { - SetVariableInConfig(cfg, name, value, false, false, "") + cfg.SetVariable(name, value, false, false, "") } for _, variable := range cfg.Pkg.Variables { @@ -27,14 +24,14 @@ func SetVariableMapInConfig(cfg *types.PackagerConfig) error { cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent cfg.SetVariableMap[variable.Name].Type = variable.Type - if err := CheckVariablePattern(cfg, variable.Name, variable.Pattern); err != nil { + if err := cfg.CheckVariablePattern(variable.Name, variable.Pattern); err != nil { return err } continue } // First set default (may be overridden by prompt) - SetVariableInConfig(cfg, variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) + cfg.SetVariable(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) // Variable is set to prompt the user if variable.Prompt && !config.CommonOptions.Confirm { @@ -45,32 +42,13 @@ func SetVariableMapInConfig(cfg *types.PackagerConfig) error { return err } - SetVariableInConfig(cfg, variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) + cfg.SetVariable(variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) } - if err := CheckVariablePattern(cfg, variable.Name, variable.Pattern); err != nil { + if err := cfg.CheckVariablePattern(variable.Name, variable.Pattern); err != nil { return err } } return nil } - -func SetVariableInConfig(cfg *types.PackagerConfig, name, value string, sensitive bool, autoIndent bool, varType types.VariableType) { - cfg.SetVariableMap[name] = &types.ZarfSetVariable{ - Name: name, - Value: value, - Sensitive: sensitive, - AutoIndent: autoIndent, - Type: varType, - } -} - -// CheckVariablePattern checks to see if a current variable is set to a value that matches its pattern -func CheckVariablePattern(cfg *types.PackagerConfig, name, pattern string) error { - if regexp.MustCompile(pattern).MatchString(cfg.SetVariableMap[name].Value) { - return nil - } - - return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) -} diff --git a/src/types/packager.go b/src/types/packager.go index d61ef87292..7c943ca313 100644 --- a/src/types/packager.go +++ b/src/types/packager.go @@ -4,6 +4,11 @@ // Package types contains all the types used by Zarf. package types +import ( + "fmt" + "regexp" +) + // PackagerConfig is the main struct that the packager uses to hold high-level options. type PackagerConfig struct { // CreateOpts tracks the user-defined options used to create the package @@ -42,3 +47,22 @@ type PackagerConfig struct { // Variables set by the user SetVariableMap map[string]*ZarfSetVariable } + +// SetVariable sets a value for a variable in PackagerConfig.SetVariableMap. +func (cfg *PackagerConfig) SetVariable(name, value string, sensitive bool, autoIndent bool, varType VariableType) { + cfg.SetVariableMap[name] = &ZarfSetVariable{ + Name: name, + Value: value, + Sensitive: sensitive, + AutoIndent: autoIndent, + Type: varType, + } +} + +// CheckVariablePattern checks to see if a variable is set to a value that matches its pattern. +func (cfg PackagerConfig) CheckVariablePattern(name, pattern string) error { + if regexp.MustCompile(pattern).MatchString(cfg.SetVariableMap[name].Value) { + return nil + } + return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) +} From f0c6ccc05897d4fd3c0216158289813bde0e7151 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 13:23:18 -0600 Subject: [PATCH 092/172] Move IsInitConfig() and IsSBOMAble() to receiver methods on ZarfPackage{} --- src/pkg/packager/creator/utils.go | 3 +-- src/pkg/packager/deploy.go | 6 +++--- src/pkg/packager/interactive.go | 2 +- src/pkg/utils/zarf_package.go | 18 +----------------- src/types/package.go | 15 +++++++++++++++ 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 5ff19fa562..bf4a5bd50a 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -34,7 +33,7 @@ func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOption return nil, err } - if utils.IsInitConfig(pkg) { + if pkg.IsInitConfig() { configuredPkg.Metadata.Version = config.CLIVersion } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 54db488724..a3376bf8ed 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -125,7 +125,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // If this component requires a cluster, connect to one if requiresCluster(component) { timeout := cluster.DefaultTimeout - if utils.IsInitConfig(p.cfg.Pkg) { + if p.cfg.Pkg.IsInitConfig() { timeout = 5 * time.Minute } @@ -155,7 +155,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // Deploy the component var charts []types.InstalledChart var deployErr error - if utils.IsInitConfig(p.cfg.Pkg) { + if p.cfg.Pkg.IsInitConfig() { charts, deployErr = p.deployInitComponent(component) } else { charts, deployErr = p.deployComponent(component, false /* keep img checksum */, false /* always push images */) @@ -630,7 +630,7 @@ func (p *Packager) installChartAndManifests(componentPaths *layout.ComponentPath func (p *Packager) printTablesForDeployment(componentsToDeploy []types.DeployedComponent) { // If not init config, print the application connection table - if !utils.IsInitConfig(p.cfg.Pkg) { + if !p.cfg.Pkg.IsInitConfig() { message.PrintConnectStringTable(p.connectStrings) } else { if p.cluster != nil { diff --git a/src/pkg/packager/interactive.go b/src/pkg/packager/interactive.go index dca7d3a648..cdd46bf069 100644 --- a/src/pkg/packager/interactive.go +++ b/src/pkg/packager/interactive.go @@ -25,7 +25,7 @@ func (p *Packager) confirmAction(stage string) (confirm bool) { // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package if stage == config.ZarfDeployStage { - if utils.IsSBOMAble(p.cfg.Pkg) { + if p.cfg.Pkg.IsSBOMAble() { // Print the location that the user can view the package SBOMs from message.HorizontalRule() message.Title("Software Bill of Materials", "an inventory of all software contained in this package") diff --git a/src/pkg/utils/zarf_package.go b/src/pkg/utils/zarf_package.go index 95c9cf9084..84c4f614d7 100644 --- a/src/pkg/utils/zarf_package.go +++ b/src/pkg/utils/zarf_package.go @@ -11,11 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// IsInitConfig returns whether the provided Zarf package is an init config. -func IsInitConfig(pkg types.ZarfPackage) bool { - return pkg.Kind == types.ZarfInitConfig -} - // GetInitPackageName returns the formatted name of the init package. func GetInitPackageName(arch string) string { if arch == "" { @@ -27,7 +22,7 @@ func GetInitPackageName(arch string) string { // GetPackageName returns the formatted name of the package. func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { - if IsInitConfig(pkg) { + if pkg.IsInitConfig() { return GetInitPackageName(pkg.Metadata.Architecture) } @@ -46,14 +41,3 @@ func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) stri return fmt.Sprintf("%s.%s", packageFileName, suffix) } - -// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections) -func IsSBOMAble(pkg types.ZarfPackage) bool { - for _, c := range pkg.Components { - if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { - return true - } - } - - return false -} diff --git a/src/types/package.go b/src/types/package.go index 0ec8f58a23..cee57b3ef8 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -24,6 +24,21 @@ type ZarfPackage struct { Variables []ZarfPackageVariable `json:"variables,omitempty" jsonschema:"description=Variable template values applied on deploy for K8s resources"` } +// IsInitConfig returns whether a Zarf package is an init config. +func (pkg ZarfPackage) IsInitConfig() bool { + return pkg.Kind == ZarfInitConfig +} + +// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections). +func (pkg ZarfPackage) IsSBOMAble() bool { + for _, c := range pkg.Components { + if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { + return true + } + } + return false +} + // ZarfMetadata lists information about the current ZarfPackage. type ZarfMetadata struct { Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]*[a-z0-9]$"` From f36eeb491c34334ec8daf095ffca288c80737807 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 14:18:52 -0600 Subject: [PATCH 093/172] Move pp.addSignature logic to pp.SignPackage Refactor TestPackageFiles() to use subtests and run in parallel --- src/pkg/layout/package.go | 12 +- src/pkg/layout/package_test.go | 252 ++++++++++++++++++--------------- 2 files changed, 138 insertions(+), 126 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 0e8ac4eb79..f7e827607e 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -146,7 +146,9 @@ func (pp *PackagePaths) IsLegacyLayout() bool { // SignPackage signs the zarf.yaml in a Zarf package. func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { - pp.addSignature(signingKeyPath) + if signingKeyPath != "" { + pp.Signature = filepath.Join(pp.Base, Signature) + } passwordFunc := func(_ bool) ([]byte, error) { if signingKeyPassword != "" { @@ -162,14 +164,6 @@ func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) e return nil } -// addSignature sets the signature path if the keyPath is not empty. -func (pp *PackagePaths) addSignature(keyPath string) *PackagePaths { - if keyPath != "" { - pp.Signature = filepath.Join(pp.Base, Signature) - } - return pp -} - // GenerateChecksums walks through all of the files starting at the base path and generates a checksum file. // // Each file within the basePath represents a layer within the Zarf package. diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index c3e543642b..1a3d4c5aa0 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -5,6 +5,7 @@ package layout import ( + "path/filepath" "runtime" "strings" "testing" @@ -14,128 +15,145 @@ import ( ) func TestPackageFiles(t *testing.T) { - pp := New("test") - - raw := &PackagePaths{ - Base: "test", - ZarfYAML: normalizePath("test/zarf.yaml"), - Checksums: normalizePath("test/checksums.txt"), - Components: Components{ - Base: normalizePath("test/components"), - }, - } - - require.Equal(t, raw, pp) - - files := pp.Files() - - expected := map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - } - - require.Equal(t, expected, files) - - pp = pp.addSignature("") - - files = pp.Files() - - // addSignature will only add the signature if it is not empty - require.Equal(t, expected, files) - - pp = pp.addSignature("key.priv") - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), - } - - require.Equal(t, expected, files) - pp = pp.AddImages() - - files = pp.Files() - - // Note that the map key will always be the forward "Slash" (/) version of the file path (never \) - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - } - - require.Equal(t, expected, files) + t.Parallel() - pp = pp.AddSBOMs() + t.Run("Verify New()", func(t *testing.T) { + t.Parallel() - files = pp.Files() + pp := New("test") - // AddSBOMs adds the SBOMs directory, and files will only cares about files - require.Equal(t, expected, files) - - paths := []string{ - "zarf.yaml", - "checksums.txt", - "sboms.tar", - normalizePath("components/c1.tar"), - normalizePath("images/index.json"), - normalizePath("images/oci-layout"), - normalizePath("images/blobs/sha256/" + strings.Repeat("1", 64)), - } - - pp = New("test") - - pp.SetFromPaths(paths) - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "sboms.tar": normalizePath("test/sboms.tar"), - "components/c1.tar": normalizePath("test/components/c1.tar"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), - } - - require.Len(t, pp.Images.Blobs, 1) - - require.Equal(t, expected, files) - - descs := []ocispec.Descriptor{ - { - Annotations: map[string]string{ - ocispec.AnnotationTitle: "components/c2.tar", + raw := &PackagePaths{ + Base: "test", + ZarfYAML: normalizePath("test/zarf.yaml"), + Checksums: normalizePath("test/checksums.txt"), + Components: Components{ + Base: normalizePath("test/components"), }, - }, - { - Annotations: map[string]string{ - ocispec.AnnotationTitle: "images/blobs/sha256/" + strings.Repeat("2", 64), + } + require.Equal(t, raw, pp) + }) + + t.Run("Verify Files()", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with signature", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp.Signature = filepath.Join(pp.Base, Signature) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with images", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp = pp.AddImages() + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + } + require.Equal(t, expected, files) + }) + + // AddSBOMs sets the SBOMs path, so Files() should not return new files. + t.Run("Verify Files() with SBOMs", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp = pp.AddSBOMs() + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with paths mapped to package paths", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + paths := []string{ + "zarf.yaml", + "checksums.txt", + "sboms.tar", + normalizePath("components/c1.tar"), + normalizePath("images/index.json"), + normalizePath("images/oci-layout"), + normalizePath("images/blobs/sha256/" + strings.Repeat("1", 64)), + } + pp.SetFromPaths(paths) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "sboms.tar": normalizePath("test/sboms.tar"), + "components/c1.tar": normalizePath("test/components/c1.tar"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), + } + + require.Len(t, pp.Images.Blobs, 1) + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with image layers mapped to package paths", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + descs := []ocispec.Descriptor{ + { + Annotations: map[string]string{ + ocispec.AnnotationTitle: "components/c2.tar", + }, }, - }, - } - - pp.SetFromLayers(descs) - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "sboms.tar": normalizePath("test/sboms.tar"), - "components/c1.tar": normalizePath("test/components/c1.tar"), - "components/c2.tar": normalizePath("test/components/c2.tar"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), - "images/blobs/sha256/" + strings.Repeat("2", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("2", 64)), - } - - require.Equal(t, expected, files) + { + Annotations: map[string]string{ + ocispec.AnnotationTitle: "images/blobs/sha256/" + strings.Repeat("1", 64), + }, + }, + } + pp.AddImages() + pp.SetFromLayers(descs) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "components/c2.tar": normalizePath("test/components/c2.tar"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), + } + require.Equal(t, expected, files) + }) } // normalizePath ensures that the filepaths being generated are normalized to the host OS. From aa5d107b1ac94db29854394e2b606e560a73d458 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 14:43:52 -0600 Subject: [PATCH 094/172] Move OutputSBOMFiles() to a SBOMs receiver --- src/pkg/layout/sbom.go | 17 ++++++++++++++++- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/inspect.go | 2 +- src/pkg/utils/sbom.go | 25 ------------------------- 4 files changed, 18 insertions(+), 28 deletions(-) delete mode 100644 src/pkg/utils/sbom.go diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 20c77a440c..47a94dac47 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -81,7 +81,7 @@ func (s *SBOMs) StageSBOMViewFiles() (warnings []string, err error) { return nil, err } - if _, err := utils.OutputSBOMFiles(s.Path, SBOMDir, ""); err != nil { + if _, err := s.OutputSBOMFiles(SBOMDir, ""); err != nil { // Don't stop the deployment, let the user decide if they want to continue the deployment warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) warnings = append(warnings, warning) @@ -90,3 +90,18 @@ func (s *SBOMs) StageSBOMViewFiles() (warnings []string, err error) { return warnings, nil } + +// OutputSBOMFiles outputs SBOM files into outputDir. +func (s *SBOMs) OutputSBOMFiles(outputDir, packageName string) (string, error) { + packagePath := filepath.Join(outputDir, packageName) + + if err := os.RemoveAll(packagePath); err != nil { + return "", err + } + + if err := utils.CreateDirectory(packagePath, 0700); err != nil { + return "", err + } + + return packagePath, utils.CreatePathAndCopy(s.Path, packagePath) +} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 6c2a9c7c73..0629e9b525 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -301,7 +301,7 @@ func (pc *packageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := utils.OutputSBOMFiles(sbomDir, outputSBOM, loadedPkg.Metadata.Name) + out, err := dst.SBOMs.OutputSBOMFiles(outputSBOM, loadedPkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 6f9f7898fa..b73f71c765 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -26,7 +26,7 @@ func (p *Packager) Inspect() (err error) { sbomDir := p.layout.SBOMs.Path if p.cfg.InspectOpts.SBOMOutputDir != "" { - out, err := utils.OutputSBOMFiles(sbomDir, p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) + out, err := p.layout.SBOMs.OutputSBOMFiles(p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/utils/sbom.go b/src/pkg/utils/sbom.go deleted file mode 100644 index 7c5e2a1252..0000000000 --- a/src/pkg/utils/sbom.go +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic helper functions. -package utils - -import ( - "os" - "path/filepath" -) - -// OutputSBOMFiles outputs the sbom files into a specified directory. -func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { - packagePath := filepath.Join(outputDir, packageName) - - if err := os.RemoveAll(packagePath); err != nil { - return "", err - } - - if err := CreateDirectory(packagePath, 0700); err != nil { - return "", err - } - - return packagePath, CreatePathAndCopy(sourceDir, packagePath) -} From 8a02c39cf7a12218ece95032e475b19c27a51d9d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 15:11:04 -0600 Subject: [PATCH 095/172] Create factory methods for creator implementations rather than returning the interface --- src/pkg/packager/create.go | 8 ++--- .../packager/creator/{new.go => creator.go} | 16 ---------- src/pkg/packager/creator/normal.go | 30 ++++++++++++------- src/pkg/packager/creator/skeleton.go | 23 ++++++++------ src/pkg/packager/dev.go | 6 ++-- src/pkg/packager/publish.go | 8 ++--- 6 files changed, 45 insertions(+), 46 deletions(-) rename src/pkg/packager/creator/{new.go => creator.go} (53%) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index d800b713e4..7c93aab5b9 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,9 +27,9 @@ func (p *Packager) Create() (err error) { message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - c := creator.New(p.cfg.CreateOpts, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) - loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -46,7 +46,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := c.Assemble(p.layout, loadedPkg); err != nil { + if err := pc.Assemble(p.layout, loadedPkg); err != nil { return err } @@ -55,5 +55,5 @@ func (p *Packager) Create() (err error) { return err } - return c.Output(p.layout, loadedPkg) + return pc.Output(p.layout, loadedPkg) } diff --git a/src/pkg/packager/creator/new.go b/src/pkg/packager/creator/creator.go similarity index 53% rename from src/pkg/packager/creator/new.go rename to src/pkg/packager/creator/creator.go index b3970aeadf..663152478f 100644 --- a/src/pkg/packager/creator/new.go +++ b/src/pkg/packager/creator/creator.go @@ -5,8 +5,6 @@ package creator import ( - "path/filepath" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ) @@ -17,17 +15,3 @@ type Creator interface { Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error } - -// New returns a new Creator based on the provided create options. -func New(createOpts types.ZarfCreateOptions, cwd string) Creator { - if createOpts.IsSkeleton { - return &skeletonCreator{createOpts: createOpts} - } - - // differentials are relative to the current working directory - if createOpts.DifferentialData.DifferentialPackagePath != "" { - createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) - } - - return &packageCreator{createOpts: createOpts} -} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 0629e9b525..772d3bbcc2 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -34,20 +34,30 @@ import ( ) var ( - // veryify that packageCreator implements Creator - _ Creator = (*packageCreator)(nil) + // veryify that PackageCreator implements Creator + _ Creator = (*PackageCreator)(nil) ) -// packageCreator provides methods for creating normal (not skeleton) Zarf packages. -type packageCreator struct { +// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. +type PackageCreator struct { createOpts types.ZarfCreateOptions // TODO: (@lucasrod16) remove PackagerConfig once actions do not depend on it: https://github.com/defenseunicorns/zarf/pull/2276 cfg *types.PackagerConfig } +// NewPackageCreator returns a new PackageCreator. +func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { + // differentials are relative to the current working directory + if createOpts.DifferentialData.DifferentialPackagePath != "" { + createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + } + + return &PackageCreator{createOpts: createOpts, cfg: cfg} +} + // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *packageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { var pkg types.ZarfPackage if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { @@ -109,7 +119,7 @@ func (pc *packageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade } // Assemble assembles all of the package assets into Zarf's tmp directory layout. -func (pc *packageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var imageList []transform.Image skipSBOMFlagUsed := pc.createOpts.SkipSBOM @@ -218,7 +228,7 @@ func (pc *packageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.Za // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *packageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range loadedPkg.Components { @@ -315,7 +325,7 @@ func (pc *packageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf return nil } -func (pc *packageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { +func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. @@ -341,7 +351,7 @@ func (pc *packageCreator) processExtensions(pkg *types.ZarfPackage, layout *layo return extendedPkg, nil } -func (pc *packageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { +func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) componentPaths, err := dst.Components.Create(component) @@ -521,7 +531,7 @@ func (pc *packageCreator) addComponent(component types.ZarfComponent, dst *layou return nil } -func (pc *packageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { +func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { componentPaths, err := dst.Components.Create(component) if err != nil { return nil, err diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 016c3b6dce..29fb806ed4 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -23,18 +23,23 @@ import ( ) var ( - // veryify that skeletonCreator implements Creator - _ Creator = (*skeletonCreator)(nil) + // veryify that SkeletonCreator implements Creator + _ Creator = (*SkeletonCreator)(nil) ) -// skeletonCreator provides methods for creating skeleton Zarf packages. -type skeletonCreator struct { +// SkeletonCreator provides methods for creating skeleton Zarf packages. +type SkeletonCreator struct { createOpts types.ZarfCreateOptions publishOpts types.ZarfPublishOptions } +// NewSkeletonCreator returns a new SkeletonCreator. +func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.ZarfPublishOptions) *SkeletonCreator { + return &SkeletonCreator{createOpts: createOpts, publishOpts: publishOpts} +} + // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. -func (sc *skeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { var pkg types.ZarfPackage if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { @@ -70,7 +75,7 @@ func (sc *skeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load // Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. // // It processes each component to ensure correct structure and resource locations. -func (sc *skeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { var updatedComponents []types.ZarfComponent for _, component := range loadedPkg.Components { @@ -95,7 +100,7 @@ func (sc *skeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.Z // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *skeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { for _, component := range loadedPkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err @@ -122,7 +127,7 @@ func (sc *skeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zar return nil } -func (sc *skeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { +func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { components := []types.ZarfComponent{} // Create component paths and process extensions for each component. @@ -148,7 +153,7 @@ func (sc *skeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *lay return extendedPkg, nil } -func (sc *skeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { +func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) updatedComponent = &component diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 762ef1d45a..1acb985565 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -30,9 +30,9 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - c := creator.New(p.cfg.CreateOpts, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) - loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) if err != nil { return err } @@ -63,7 +63,7 @@ func (p *Packager) DevDeploy() error { } } - if err := c.Assemble(p.layout, &p.cfg.Pkg); err != nil { + if err := pc.Assemble(p.layout, &p.cfg.Pkg); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index a3e0e26e2f..9f677ec90f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -89,20 +89,20 @@ func (p *Packager) Publish() (err error) { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - c := creator.New(p.cfg.CreateOpts, "") + sc := creator.NewSkeletonCreator(p.cfg.CreateOpts, p.cfg.PublishOpts) - loadedPkg, warnings, err := c.LoadPackageDefinition(p.layout) + loadedPkg, warnings, err := sc.LoadPackageDefinition(p.layout) if err != nil { return err } p.warnings = append(p.warnings, warnings...) - if err := c.Assemble(p.layout, loadedPkg); err != nil { + if err := sc.Assemble(p.layout, loadedPkg); err != nil { return err } - if err := c.Output(p.layout, loadedPkg); err != nil { + if err := sc.Output(p.layout, loadedPkg); err != nil { return err } From 2d6ea9ceadd8d0e7c9d2ac7554e7db0df0e6b99c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 15:30:05 -0600 Subject: [PATCH 096/172] Make pkg variable naming less weird and confusing --- src/pkg/packager/creator/normal.go | 19 +++++++++---------- src/pkg/packager/creator/skeleton.go | 8 +++----- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 772d3bbcc2..6c67545174 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -64,13 +64,13 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - configuredPkg, err := setPackageMetadata(pkg, pc.createOpts) + loadedPkg, err = setPackageMetadata(pkg, pc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, pc.createOpts.Flavor) + loadedPkg, composeWarnings, err := ComposeComponents(loadedPkg, pc.createOpts.Flavor) if err != nil { return nil, nil, err } @@ -78,7 +78,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - templatedPkg, templateWarnings, err := FillActiveTemplate(composedPkg, pc.createOpts.SetVariables) + loadedPkg, templateWarnings, err := FillActiveTemplate(loadedPkg, pc.createOpts.SetVariables) if err != nil { return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -86,36 +86,35 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - extendedPkg, err := pc.processExtensions(templatedPkg, dst) + loadedPkg, err = pc.processExtensions(loadedPkg, dst) if err != nil { return nil, nil, err } // If we are creating a differential package, remove duplicate images and repos. - if extendedPkg.Build.Differential { + if loadedPkg.Build.Differential { loadedDiffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) if err != nil { return nil, nil, err } - versionsMatch := loadedDiffData.DifferentialPackageVersion == extendedPkg.Metadata.Version + versionsMatch := loadedDiffData.DifferentialPackageVersion == loadedPkg.Metadata.Version if versionsMatch { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := loadedDiffData.DifferentialPackageVersion == "" || extendedPkg.Metadata.Version == "" + noVersionSet := loadedDiffData.DifferentialPackageVersion == "" || loadedPkg.Metadata.Version == "" if noVersionSet { return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } - diffPkg, err := removeCopiesFromDifferentialPackage(extendedPkg, loadedDiffData) + loadedPkg, err = removeCopiesFromDifferentialPackage(loadedPkg, loadedDiffData) if err != nil { return nil, nil, err } - return diffPkg, warnings, nil } - return extendedPkg, warnings, nil + return loadedPkg, warnings, nil } // Assemble assembles all of the package assets into Zarf's tmp directory layout. diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 29fb806ed4..5ed316ec4b 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -46,19 +46,19 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - configuredPkg, err := setPackageMetadata(pkg, sc.createOpts) + loadedPkg, err = setPackageMetadata(pkg, sc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - composedPkg, composeWarnings, err := ComposeComponents(configuredPkg, sc.createOpts.Flavor) + loadedPkg, composeWarnings, err := ComposeComponents(loadedPkg, sc.createOpts.Flavor) if err != nil { return nil, nil, err } warnings = append(warnings, composeWarnings...) - extendedPkg, err := sc.processExtensions(composedPkg, dst) + loadedPkg, err = sc.processExtensions(loadedPkg, dst) if err != nil { return nil, nil, err } @@ -67,8 +67,6 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load message.Warn(warning) } - loadedPkg = extendedPkg - return loadedPkg, warnings, nil } From 8d281defb44a95b959bf0d735718e03ea37e47b8 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 15:48:51 -0600 Subject: [PATCH 097/172] Update processExtensions to work with components directly --- src/pkg/packager/creator/normal.go | 17 ++++++----------- src/pkg/packager/creator/skeleton.go | 15 +++++---------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 6c67545174..4d62cf0dcd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -86,7 +86,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - loadedPkg, err = pc.processExtensions(loadedPkg, dst) + loadedPkg.Components, err = pc.processExtensions(loadedPkg.Components, dst, loadedPkg.Metadata.YOLO) if err != nil { return nil, nil, err } @@ -324,11 +324,9 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf return nil } -func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { - components := []types.ZarfComponent{} - +func (pc *PackageCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths, isYOLO bool) (processedComponents []types.ZarfComponent, err error) { // Create component paths and process extensions for each component. - for _, c := range pkg.Components { + for _, c := range components { componentPaths, err := layout.Components.Create(c) if err != nil { return nil, err @@ -336,18 +334,15 @@ func (pc *PackageCreator) processExtensions(pkg *types.ZarfPackage, layout *layo // Big Bang if c.Extensions.BigBang != nil { - if c, err = bigbang.Run(pkg.Metadata.YOLO, componentPaths, c); err != nil { + if c, err = bigbang.Run(isYOLO, componentPaths, c); err != nil { return nil, fmt.Errorf("unable to process bigbang extension: %w", err) } } - components = append(components, c) + processedComponents = append(processedComponents, c) } - pkg.Components = components - extendedPkg = pkg - - return extendedPkg, nil + return processedComponents, nil } func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 5ed316ec4b..7334897bb8 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -58,7 +58,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load } warnings = append(warnings, composeWarnings...) - loadedPkg, err = sc.processExtensions(loadedPkg, dst) + loadedPkg.Components, err = sc.processExtensions(loadedPkg.Components, dst) if err != nil { return nil, nil, err } @@ -125,11 +125,9 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zar return nil } -func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *layout.PackagePaths) (extendedPkg *types.ZarfPackage, err error) { - components := []types.ZarfComponent{} - +func (pc *SkeletonCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths) (processedComponents []types.ZarfComponent, err error) { // Create component paths and process extensions for each component. - for _, c := range pkg.Components { + for _, c := range components { componentPaths, err := layout.Components.Create(c) if err != nil { return nil, err @@ -142,13 +140,10 @@ func (sc *SkeletonCreator) processExtensions(pkg *types.ZarfPackage, layout *lay } } - components = append(components, c) + processedComponents = append(processedComponents, c) } - pkg.Components = components - extendedPkg = pkg - - return extendedPkg, nil + return processedComponents, nil } func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { From 0bca3e09fb79c1c722c1a6f3729e052ca6c32338 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 7 Feb 2024 16:23:58 -0600 Subject: [PATCH 098/172] Change Assemble() to accept components instead of a ZarfPackage --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/creator.go | 2 +- src/pkg/packager/creator/normal.go | 6 +++--- src/pkg/packager/creator/skeleton.go | 10 +++------- src/pkg/packager/dev.go | 2 +- src/pkg/packager/publish.go | 2 +- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 7c93aab5b9..bab63c3404 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -46,7 +46,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := pc.Assemble(p.layout, loadedPkg); err != nil { + if err := pc.Assemble(p.layout, loadedPkg.Components, loadedPkg.Metadata.Architecture); err != nil { return err } diff --git a/src/pkg/packager/creator/creator.go b/src/pkg/packager/creator/creator.go index 663152478f..16cc1d5b77 100644 --- a/src/pkg/packager/creator/creator.go +++ b/src/pkg/packager/creator/creator.go @@ -12,6 +12,6 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) - Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error + Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 4d62cf0dcd..b324165f09 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -118,13 +118,13 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade } // Assemble assembles all of the package assets into Zarf's tmp directory layout. -func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error { var imageList []transform.Image skipSBOMFlagUsed := pc.createOpts.SkipSBOM componentSBOMs := map[string]*layout.ComponentSBOM{} - for _, component := range loadedPkg.Components { + for _, component := range components { onCreate := component.Actions.OnCreate onFailure := func() { @@ -180,7 +180,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.Za ImagesPath: dst.Images.Base, ImageList: imageList, Insecure: config.CommonOptions.Insecure, - Architectures: []string{loadedPkg.Metadata.Architecture, loadedPkg.Build.Architecture}, + Architectures: []string{arch}, RegistryOverrides: pc.createOpts.RegistryOverrides, } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 7334897bb8..c9f733bf4a 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -73,19 +73,15 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (load // Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. // // It processes each component to ensure correct structure and resource locations. -func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { - var updatedComponents []types.ZarfComponent - - for _, component := range loadedPkg.Components { +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, _ string) error { + for _, component := range components { c, err := sc.addComponent(component, dst) if err != nil { return err } - updatedComponents = append(updatedComponents, *c) + components = append(components, *c) } - loadedPkg.Components = updatedComponents - return nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 1acb985565..7ffd54c209 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -63,7 +63,7 @@ func (p *Packager) DevDeploy() error { } } - if err := pc.Assemble(p.layout, &p.cfg.Pkg); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 9f677ec90f..d40173873f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -98,7 +98,7 @@ func (p *Packager) Publish() (err error) { p.warnings = append(p.warnings, warnings...) - if err := sc.Assemble(p.layout, loadedPkg); err != nil { + if err := sc.Assemble(p.layout, loadedPkg.Components, ""); err != nil { return err } From b642690c00d001745d8762dd9b68bc7d68f51c4d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 10:00:19 -0600 Subject: [PATCH 099/172] Refactor removeCopiesFromDifferentialPackage to work with components directly --- src/pkg/packager/creator/differential.go | 36 +++++++----------------- src/pkg/packager/creator/normal.go | 2 +- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index f9d50be513..2948e831c4 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -77,66 +77,50 @@ func loadDifferentialData(diffData *types.DifferentialData) (loadedDiffData *typ return loadedDiffData, nil } -// removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package. -func removeCopiesFromDifferentialPackage(pkg *types.ZarfPackage, loadedDiffData *types.DifferentialData) (diffPkg *types.ZarfPackage, err error) { - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range pkg.Components { +// removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package components. +func removeCopiesFromDifferentialPackage(components []types.ZarfComponent, loadedDiffData *types.DifferentialData) (diffComponents []types.ZarfComponent, err error) { + for _, component := range components { newImageList := []string{} newRepoList := []string{} - // Generate a list of all unique images for this component + for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package imgRef, err := transform.ParseImageRef(img) if err != nil { return nil, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) } - // Only include new images or images that have a commonly overwritten tag imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !loadedDiffData.DifferentialImages[img] { + includeImage := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if includeImage || !loadedDiffData.DifferentialImages[img] { newImageList = append(newImageList, img) } else { message.Debugf("Image %s is already included in the differential package", img) } } - // Generate a list of all unique repos for this component for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference _, refPlain, err := transform.GitURLSplitRef(repoURL) if err != nil { return nil, err } var ref plumbing.ReferenceName - // Parse the ref from the git URL. if refPlain != "" { ref = git.ParseRef(refPlain) } - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !loadedDiffData.DifferentialRepos[repoURL] { + includeRepo := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if includeRepo || !loadedDiffData.DifferentialRepos[repoURL] { newRepoList = append(newRepoList, repoURL) } else { message.Debugf("Repo %s is already included in the differential package", repoURL) } } - // Update the component with the unique lists of repos and images component.Images = newImageList component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - pkg.Components[idx] = component + diffComponents = append(diffComponents, component) } - diffPkg = pkg - - return diffPkg, nil + return diffComponents, nil } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index b324165f09..89323739c3 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -108,7 +108,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } - loadedPkg, err = removeCopiesFromDifferentialPackage(loadedPkg, loadedDiffData) + loadedPkg.Components, err = removeCopiesFromDifferentialPackage(loadedPkg.Components, loadedDiffData) if err != nil { return nil, nil, err } From e958ad9f2665cadea4139de5353d77219dd213ce Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 14:43:30 -0600 Subject: [PATCH 100/172] Refactor loadDifferentialData to use sources and layout to load pkg --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/differential.go | 60 +++++++++--------------- src/pkg/packager/creator/normal.go | 13 ++--- src/pkg/packager/dev.go | 2 +- 4 files changed, 30 insertions(+), 47 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index bab63c3404..58df6a0177 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,7 +27,7 @@ func (p *Packager) Create() (err error) { message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) if err != nil { diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 2948e831c4..1dade5fcbc 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -6,60 +6,42 @@ package creator import ( "fmt" - "os" - "path/filepath" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func loadDifferentialData(diffData *types.DifferentialData) (loadedDiffData *types.DifferentialData, err error) { - tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) +func (pc *PackageCreator) loadDifferentialData(dst *layout.PackagePaths) (diffData *types.DifferentialData, err error) { + diffPkgPath := pc.createOpts.DifferentialData.DifferentialPackagePath + if diffPkgPath != "" { + pc.pkgOpts.PackageSource = diffPkgPath + } + + src, err := sources.New(pc.pkgOpts) if err != nil { return nil, err } - defer os.RemoveAll(tmpDir) - - // Load the reference package. - if helpers.IsOCIURL(diffData.DifferentialPackagePath) { - remote, err := oci.NewOrasRemote(diffData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) - if err != nil { - return nil, err - } - pkg, err := remote.FetchZarfYAML() - if err != nil { - return nil, err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return nil, err - } - } else { - if err := archiver.Extract(diffData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return nil, fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } + if err := src.LoadPackage(dst, true); err != nil { + return nil, err } - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return nil, fmt.Errorf("unable to load the differential zarf package spec: %w", err) + var diffPkg types.ZarfPackage + if err := utils.ReadYaml(dst.ZarfYAML, &diffPkg); err != nil { + return nil, fmt.Errorf("error reading the differential Zarf package: %w", err) } allIncludedImagesMap := map[string]bool{} allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { + for _, component := range diffPkg.Components { for _, image := range component.Images { allIncludedImagesMap[image] = true } @@ -68,17 +50,17 @@ func loadDifferentialData(diffData *types.DifferentialData) (loadedDiffData *typ } } - diffData.DifferentialImages = allIncludedImagesMap - diffData.DifferentialRepos = allIncludedReposMap - diffData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + pc.createOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + pc.createOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + pc.createOpts.DifferentialData.DifferentialPackageVersion = diffPkg.Metadata.Version - loadedDiffData = diffData + diffData = &pc.createOpts.DifferentialData - return loadedDiffData, nil + return diffData, nil } -// removeCopiesFromDifferentialPackage removes any images and repos already present in the reference package components. -func removeCopiesFromDifferentialPackage(components []types.ZarfComponent, loadedDiffData *types.DifferentialData) (diffComponents []types.ZarfComponent, err error) { +// removeCopiesFromComponents removes any images and repos already present in the reference package components. +func removeCopiesFromComponents(components []types.ZarfComponent, loadedDiffData *types.DifferentialData) (diffComponents []types.ZarfComponent, err error) { for _, component := range components { newImageList := []string{} newRepoList := []string{} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 89323739c3..23b09ef935 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -41,19 +41,20 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { createOpts types.ZarfCreateOptions + pkgOpts *types.ZarfPackageOptions // TODO: (@lucasrod16) remove PackagerConfig once actions do not depend on it: https://github.com/defenseunicorns/zarf/pull/2276 cfg *types.PackagerConfig } // NewPackageCreator returns a new PackageCreator. -func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { +func NewPackageCreator(createOpts types.ZarfCreateOptions, pkgOpts *types.ZarfPackageOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { // differentials are relative to the current working directory if createOpts.DifferentialData.DifferentialPackagePath != "" { createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) } - return &PackageCreator{createOpts: createOpts, cfg: cfg} + return &PackageCreator{createOpts: createOpts, pkgOpts: pkgOpts, cfg: cfg} } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. @@ -93,22 +94,22 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade // If we are creating a differential package, remove duplicate images and repos. if loadedPkg.Build.Differential { - loadedDiffData, err := loadDifferentialData(&pc.createOpts.DifferentialData) + diffData, err := pc.loadDifferentialData(dst) if err != nil { return nil, nil, err } - versionsMatch := loadedDiffData.DifferentialPackageVersion == loadedPkg.Metadata.Version + versionsMatch := diffData.DifferentialPackageVersion == loadedPkg.Metadata.Version if versionsMatch { return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := loadedDiffData.DifferentialPackageVersion == "" || loadedPkg.Metadata.Version == "" + noVersionSet := diffData.DifferentialPackageVersion == "" || loadedPkg.Metadata.Version == "" if noVersionSet { return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } - loadedPkg.Components, err = removeCopiesFromDifferentialPackage(loadedPkg.Components, loadedDiffData) + loadedPkg.Components, err = removeCopiesFromComponents(loadedPkg.Components, diffData) if err != nil { return nil, nil, err } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 7ffd54c209..2e32403807 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -30,7 +30,7 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) if err != nil { From f182d25533e6e2fa80519a7dbaf132d6339ae503 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 17:03:08 -0600 Subject: [PATCH 101/172] Use a tmpdir and isolated layout for loading diff pkg --- src/pkg/packager/creator/differential.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 1dade5fcbc..af3801a43c 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -6,7 +6,9 @@ package creator import ( "fmt" + "os" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -24,17 +26,25 @@ func (pc *PackageCreator) loadDifferentialData(dst *layout.PackagePaths) (diffDa pc.pkgOpts.PackageSource = diffPkgPath } + tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + + diffLayout := layout.New(tmpdir) + defer os.RemoveAll(diffLayout.Base) + src, err := sources.New(pc.pkgOpts) if err != nil { return nil, err } - if err := src.LoadPackage(dst, true); err != nil { + if err := src.LoadPackageMetadata(diffLayout, false, false); err != nil { return nil, err } var diffPkg types.ZarfPackage - if err := utils.ReadYaml(dst.ZarfYAML, &diffPkg); err != nil { + if err := utils.ReadYaml(diffLayout.ZarfYAML, &diffPkg); err != nil { return nil, fmt.Errorf("error reading the differential Zarf package: %w", err) } From 501dcdfdec920e6b33c44ac6f3dad015d72da795 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 17:16:22 -0600 Subject: [PATCH 102/172] Remove unused PackagePaths param from loadDifferentialData() --- src/pkg/packager/creator/differential.go | 2 +- src/pkg/packager/creator/normal.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index af3801a43c..8e50a2984e 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -20,7 +20,7 @@ import ( ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func (pc *PackageCreator) loadDifferentialData(dst *layout.PackagePaths) (diffData *types.DifferentialData, err error) { +func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialData, err error) { diffPkgPath := pc.createOpts.DifferentialData.DifferentialPackagePath if diffPkgPath != "" { pc.pkgOpts.PackageSource = diffPkgPath diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 23b09ef935..e96498bae1 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -94,7 +94,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loade // If we are creating a differential package, remove duplicate images and repos. if loadedPkg.Build.Differential { - diffData, err := pc.loadDifferentialData(dst) + diffData, err := pc.loadDifferentialData() if err != nil { return nil, nil, err } From 75e516b6dc75d893de3280fcf5640ff6c70d3ab4 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 22:55:59 -0600 Subject: [PATCH 103/172] Use ZarfPackage struct rather than pointer in Creator operations --- src/pkg/packager/create.go | 11 +++--- src/pkg/packager/creator/compose.go | 14 +++---- src/pkg/packager/creator/creator.go | 4 +- src/pkg/packager/creator/normal.go | 56 ++++++++++++++-------------- src/pkg/packager/creator/skeleton.go | 26 ++++++------- src/pkg/packager/creator/template.go | 21 +++++------ src/pkg/packager/creator/utils.go | 36 +++++++++--------- src/pkg/packager/dev.go | 7 ++-- src/pkg/packager/prepare.go | 2 +- src/pkg/packager/publish.go | 9 ++--- 10 files changed, 87 insertions(+), 99 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 58df6a0177..013db17ed4 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -29,16 +29,15 @@ func (p *Packager) Create() (err error) { pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) - loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) + var warnings []string + p.cfg.Pkg, warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { return err } - - p.cfg.Pkg = *loadedPkg p.warnings = append(p.warnings, warnings...) // Perform early package validation. - if err := validate.Run(*loadedPkg); err != nil { + if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } @@ -46,7 +45,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := pc.Assemble(p.layout, loadedPkg.Components, loadedPkg.Metadata.Architecture); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } @@ -55,5 +54,5 @@ func (p *Packager) Create() (err error) { return err } - return pc.Output(p.layout, loadedPkg) + return pc.Output(p.layout, p.cfg.Pkg) } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index dabe285c6e..a0ce1a4a63 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -11,8 +11,9 @@ import ( ) // ComposeComponents composes components and their dependencies into a single zarf.yaml using an import chain. -func ComposeComponents(pkg *types.ZarfPackage, flavor string) (composedPkg *types.ZarfPackage, warnings []string, err error) { +func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, []string, error) { components := []types.ZarfComponent{} + warnings := []string{} pkgVars := pkg.Variables pkgConsts := pkg.Constants @@ -31,18 +32,17 @@ func ComposeComponents(pkg *types.ZarfPackage, flavor string) (composedPkg *type // build the import chain chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, flavor) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } message.Debugf("%s", chain) // migrate any deprecated component configurations now - migrationWarnings := chain.Migrate(pkg.Build) - warnings = append(warnings, migrationWarnings...) + warnings = chain.Migrate(pkg.Build) // get the composed component composed, err := chain.Compose() if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } components = append(components, *composed) @@ -57,7 +57,5 @@ func ComposeComponents(pkg *types.ZarfPackage, flavor string) (composedPkg *type pkg.Variables = pkgVars pkg.Constants = pkgConsts - composedPkg = pkg - - return composedPkg, warnings, nil + return pkg, warnings, nil } diff --git a/src/pkg/packager/creator/creator.go b/src/pkg/packager/creator/creator.go index 16cc1d5b77..c9428d771a 100644 --- a/src/pkg/packager/creator/creator.go +++ b/src/pkg/packager/creator/creator.go @@ -11,7 +11,7 @@ import ( // Creator is an interface for creating Zarf packages. type Creator interface { - LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) + LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error - Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error + Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e96498bae1..fcc5cb3d8f 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -58,64 +58,62 @@ func NewPackageCreator(createOpts types.ZarfCreateOptions, pkgOpts *types.ZarfPa } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. -func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - var pkg types.ZarfPackage - +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { - return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - loadedPkg, err = setPackageMetadata(pkg, pc.createOpts) + pkg, err = setPackageMetadata(pkg, pc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - loadedPkg, composeWarnings, err := ComposeComponents(loadedPkg, pc.createOpts.Flavor) + pkg, composeWarnings, err := ComposeComponents(pkg, pc.createOpts.Flavor) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } warnings = append(warnings, composeWarnings...) // After components are composed, template the active package. - loadedPkg, templateWarnings, err := FillActiveTemplate(loadedPkg, pc.createOpts.SetVariables) + pkg, templateWarnings, err := FillActiveTemplate(pkg, pc.createOpts.SetVariables) if err != nil { - return nil, nil, fmt.Errorf("unable to fill values in template: %w", err) + return types.ZarfPackage{}, nil, fmt.Errorf("unable to fill values in template: %w", err) } warnings = append(warnings, templateWarnings...) // After templates are filled process any create extensions - loadedPkg.Components, err = pc.processExtensions(loadedPkg.Components, dst, loadedPkg.Metadata.YOLO) + pkg.Components, err = pc.processExtensions(pkg.Components, dst, pkg.Metadata.YOLO) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } // If we are creating a differential package, remove duplicate images and repos. - if loadedPkg.Build.Differential { + if pkg.Build.Differential { diffData, err := pc.loadDifferentialData() if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } - versionsMatch := diffData.DifferentialPackageVersion == loadedPkg.Metadata.Version + versionsMatch := diffData.DifferentialPackageVersion == pkg.Metadata.Version if versionsMatch { - return nil, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) + return types.ZarfPackage{}, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) } - noVersionSet := diffData.DifferentialPackageVersion == "" || loadedPkg.Metadata.Version == "" + noVersionSet := diffData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" if noVersionSet { - return nil, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) + return types.ZarfPackage{}, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) } - loadedPkg.Components, err = removeCopiesFromComponents(loadedPkg.Components, diffData) + pkg.Components, err = removeCopiesFromComponents(pkg.Components, diffData) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } } - return loadedPkg, warnings, nil + return pkg, warnings, nil } // Assemble assembles all of the package assets into Zarf's tmp directory layout. @@ -228,10 +226,10 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range loadedPkg.Components { + for _, component := range pkg.Components { // Make the component a tar archive if err := dst.Components.Archive(component, true); err != nil { return fmt.Errorf("unable to archive component: %s", err.Error()) @@ -243,16 +241,16 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } - loadedPkg.Metadata.AggregateChecksum = checksumChecksum + pkg.Metadata.AggregateChecksum = checksumChecksum // Record the migrations that will be ran on the package. - loadedPkg.Build.Migrations = []string{ + pkg.Build.Migrations = []string{ deprecated.ScriptsToActionsMigrated, deprecated.PluralizeSetVariable, } // Save the transformed config. - if err := utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } @@ -266,7 +264,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf // Create a remote ref + client for the package (if output is OCI) // then publish the package to the remote. if helpers.IsOCIURL(pc.createOpts.Output) { - ref, err := oci.ReferenceFromMetadata(pc.createOpts.Output, &loadedPkg.Metadata, &loadedPkg.Build) + ref, err := oci.ReferenceFromMetadata(pc.createOpts.Output, &pkg.Metadata, &pkg.Build) if err != nil { return err } @@ -275,7 +273,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf return err } - err = remote.PublishPackage(loadedPkg, dst, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(&pkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -290,7 +288,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(*loadedPkg, pc.createOpts.DifferentialData)) + packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(pkg, pc.createOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) @@ -311,7 +309,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zarf sbomDir = dst.SBOMs.Path if outputSBOM != "" { - out, err := dst.SBOMs.OutputSBOMFiles(outputSBOM, loadedPkg.Metadata.Name) + out, err := dst.SBOMs.OutputSBOMFiles(outputSBOM, pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index c9f733bf4a..d42d6bcaf3 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -39,35 +39,33 @@ func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.Za } // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. -func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (loadedPkg *types.ZarfPackage, warnings []string, err error) { - var pkg types.ZarfPackage - +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { - return nil, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - loadedPkg, err = setPackageMetadata(pkg, sc.createOpts) + pkg, err = setPackageMetadata(pkg, sc.createOpts) if err != nil { message.Warn(err.Error()) } // Compose components into a single zarf.yaml file - loadedPkg, composeWarnings, err := ComposeComponents(loadedPkg, sc.createOpts.Flavor) + pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } warnings = append(warnings, composeWarnings...) - loadedPkg.Components, err = sc.processExtensions(loadedPkg.Components, dst) + pkg.Components, err = sc.processExtensions(pkg.Components, dst) if err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } for _, warning := range warnings { message.Warn(warning) } - return loadedPkg, warnings, nil + return pkg, warnings, nil } // Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. @@ -94,8 +92,8 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, components []types // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.ZarfPackage) error { - for _, component := range loadedPkg.Components { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error { + for _, component := range pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } @@ -105,9 +103,9 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, loadedPkg *types.Zar if err != nil { return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) } - loadedPkg.Metadata.AggregateChecksum = checksumChecksum + pkg.Metadata.AggregateChecksum = checksumChecksum - if err := utils.WriteYaml(dst.ZarfYAML, loadedPkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return err } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index f840280dfc..fade169312 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -15,11 +15,12 @@ import ( ) // FillActiveTemplate merges user-specified variables into the configuration templates of a zarf.yaml. -func FillActiveTemplate(pkg *types.ZarfPackage, setVariables map[string]string) (templatedPkg *types.ZarfPackage, warnings []string, err error) { +func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) (types.ZarfPackage, []string, error) { templateMap := map[string]string{} + warnings := []string{} promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(pkg, templatePrefix, "###") + yamlTemplates, err := utils.FindYamlTemplates(&pkg, templatePrefix, "###") if err != nil { return err } @@ -52,28 +53,26 @@ func FillActiveTemplate(pkg *types.ZarfPackage, setVariables map[string]string) } // update the component templates on the package - if err := ReloadComponentTemplatesInPackage(pkg); err != nil { - return nil, nil, err + if err := ReloadComponentTemplatesInPackage(&pkg); err != nil { + return types.ZarfPackage{}, nil, err } if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return nil, nil, err + return types.ZarfPackage{}, nil, err } // Add special variable for the current package architecture templateMap[types.ZarfPackageArch] = pkg.Build.Architecture - if err := utils.ReloadYamlTemplate(pkg, templateMap); err != nil { - return nil, nil, err + if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { + return types.ZarfPackage{}, nil, err } - templatedPkg = pkg - - return templatedPkg, warnings, nil + return pkg, warnings, nil } // ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index bf4a5bd50a..38416ceb13 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -15,58 +15,56 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (configuredPkg *types.ZarfPackage, err error) { - configuredPkg = &pkg - +func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (types.ZarfPackage, error) { now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. // Record the name of the user creating the package. if runtime.GOOS == "windows" { - configuredPkg.Build.User = os.Getenv("USERNAME") + pkg.Build.User = os.Getenv("USERNAME") } else { - configuredPkg.Build.User = os.Getenv("USER") + pkg.Build.User = os.Getenv("USER") } hostname, err := os.Hostname() if err != nil { - return nil, err + return types.ZarfPackage{}, err } if pkg.IsInitConfig() { - configuredPkg.Metadata.Version = config.CLIVersion + pkg.Metadata.Version = config.CLIVersion } // Set package architecture if createOpts.IsSkeleton { - configuredPkg.Metadata.Architecture = "skeleton" + pkg.Metadata.Architecture = "skeleton" } - if configuredPkg.Metadata.Architecture == "" { - configuredPkg.Metadata.Architecture = config.GetArch() + if pkg.Metadata.Architecture == "" { + pkg.Metadata.Architecture = config.GetArch() } - configuredPkg.Build.Architecture = configuredPkg.Metadata.Architecture + pkg.Build.Architecture = pkg.Metadata.Architecture // Record the time of package creation. - configuredPkg.Build.Timestamp = now.Format(time.RFC1123Z) + pkg.Build.Timestamp = now.Format(time.RFC1123Z) // Record the Zarf Version the CLI was built with. - configuredPkg.Build.Version = config.CLIVersion + pkg.Build.Version = config.CLIVersion // Record the hostname of the package creation terminal. - configuredPkg.Build.Terminal = hostname + pkg.Build.Terminal = hostname // If the --differential flag was used, record that this is a differential package. if createOpts.DifferentialData.DifferentialPackagePath != "" { - configuredPkg.Build.Differential = true + pkg.Build.Differential = true } // Record the flavor of Zarf used to build this package (if any). - configuredPkg.Build.Flavor = createOpts.Flavor + pkg.Build.Flavor = createOpts.Flavor - configuredPkg.Build.RegistryOverrides = createOpts.RegistryOverrides + pkg.Build.RegistryOverrides = createOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. - configuredPkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return configuredPkg, nil + return pkg, nil } diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 2e32403807..56f3e8035e 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -32,15 +32,14 @@ func (p *Packager) DevDeploy() error { pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) - loadedPkg, warnings, err := pc.LoadPackageDefinition(p.layout) + var warnings []string + p.cfg.Pkg, warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { return err } p.warnings = append(p.warnings, warnings...) - p.cfg.Pkg = *loadedPkg - // Filter out components that are not compatible with this system p.filterComponents() @@ -51,7 +50,7 @@ func (p *Packager) DevDeploy() error { // all components and their dependencies, regardless of whether they are required or not p.cfg.Pkg.Components = p.getSelectedComponents() - if err := validate.Run(*loadedPkg); err != nil { + if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 5b68e0f3e0..99dfbd4a2e 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -59,7 +59,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(&p.cfg.Pkg, p.cfg.CreateOpts.Flavor) + composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) if err != nil { return nil, err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index d40173873f..3c2c2c527f 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -91,22 +91,21 @@ func (p *Packager) Publish() (err error) { sc := creator.NewSkeletonCreator(p.cfg.CreateOpts, p.cfg.PublishOpts) - loadedPkg, warnings, err := sc.LoadPackageDefinition(p.layout) + var warnings []string + p.cfg.Pkg, warnings, err = sc.LoadPackageDefinition(p.layout) if err != nil { return err } p.warnings = append(p.warnings, warnings...) - if err := sc.Assemble(p.layout, loadedPkg.Components, ""); err != nil { + if err := sc.Assemble(p.layout, p.cfg.Pkg.Components, ""); err != nil { return err } - if err := sc.Output(p.layout, loadedPkg); err != nil { + if err := sc.Output(p.layout, p.cfg.Pkg); err != nil { return err } - - p.cfg.Pkg = *loadedPkg } else { if err := p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) From e2b744462aeffa0f6eb7ead7f8878e284013fbb9 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:00:40 -0600 Subject: [PATCH 104/172] Directly assign LoadPackageDefinition warnings to p.warnings --- src/pkg/packager/create.go | 4 +--- src/pkg/packager/dev.go | 5 +---- src/pkg/packager/publish.go | 5 +---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 013db17ed4..462f467adb 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -29,12 +29,10 @@ func (p *Packager) Create() (err error) { pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) - var warnings []string - p.cfg.Pkg, warnings, err = pc.LoadPackageDefinition(p.layout) + p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { return err } - p.warnings = append(p.warnings, warnings...) // Perform early package validation. if err := validate.Run(p.cfg.Pkg); err != nil { diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 56f3e8035e..82aa4daf76 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -32,14 +32,11 @@ func (p *Packager) DevDeploy() error { pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) - var warnings []string - p.cfg.Pkg, warnings, err = pc.LoadPackageDefinition(p.layout) + p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { return err } - p.warnings = append(p.warnings, warnings...) - // Filter out components that are not compatible with this system p.filterComponents() diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 3c2c2c527f..574cc0fa4b 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -91,14 +91,11 @@ func (p *Packager) Publish() (err error) { sc := creator.NewSkeletonCreator(p.cfg.CreateOpts, p.cfg.PublishOpts) - var warnings []string - p.cfg.Pkg, warnings, err = sc.LoadPackageDefinition(p.layout) + p.cfg.Pkg, p.warnings, err = sc.LoadPackageDefinition(p.layout) if err != nil { return err } - p.warnings = append(p.warnings, warnings...) - if err := sc.Assemble(p.layout, p.cfg.Pkg.Components, ""); err != nil { return err } From dd952a9197af9395e454ffcda197ad98e4b1334e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:08:32 -0600 Subject: [PATCH 105/172] Use a fresh ZarfPackageOptions struct rathen than embedding in PackageCreator --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/differential.go | 6 ++++-- src/pkg/packager/creator/normal.go | 5 ++--- src/pkg/packager/dev.go | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 462f467adb..af33464450 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,7 +27,7 @@ func (p *Packager) Create() (err error) { message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 8e50a2984e..254921a922 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -21,9 +21,11 @@ import ( // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialData, err error) { + pkgOpts := types.ZarfPackageOptions{} + diffPkgPath := pc.createOpts.DifferentialData.DifferentialPackagePath if diffPkgPath != "" { - pc.pkgOpts.PackageSource = diffPkgPath + pkgOpts.PackageSource = diffPkgPath } tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) @@ -34,7 +36,7 @@ func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialDa diffLayout := layout.New(tmpdir) defer os.RemoveAll(diffLayout.Base) - src, err := sources.New(pc.pkgOpts) + src, err := sources.New(&pkgOpts) if err != nil { return nil, err } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index fcc5cb3d8f..e2194731d9 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -41,20 +41,19 @@ var ( // PackageCreator provides methods for creating normal (not skeleton) Zarf packages. type PackageCreator struct { createOpts types.ZarfCreateOptions - pkgOpts *types.ZarfPackageOptions // TODO: (@lucasrod16) remove PackagerConfig once actions do not depend on it: https://github.com/defenseunicorns/zarf/pull/2276 cfg *types.PackagerConfig } // NewPackageCreator returns a new PackageCreator. -func NewPackageCreator(createOpts types.ZarfCreateOptions, pkgOpts *types.ZarfPackageOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { +func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { // differentials are relative to the current working directory if createOpts.DifferentialData.DifferentialPackagePath != "" { createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) } - return &PackageCreator{createOpts: createOpts, pkgOpts: pkgOpts, cfg: cfg} + return &PackageCreator{createOpts, cfg} } // LoadPackageDefinition loads and configures a zarf.yaml file during package create. diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 82aa4daf76..bee1f3208a 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -30,7 +30,7 @@ func (p *Packager) DevDeploy() error { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - pc := creator.NewPackageCreator(p.cfg.CreateOpts, &p.cfg.PkgOpts, p.cfg, cwd) + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) if err != nil { From 26e2c1fa8cc634439603161c51135616ec2f1dd1 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:09:11 -0600 Subject: [PATCH 106/172] Update src/pkg/packager/creator/skeleton.go Co-authored-by: razzle --- src/pkg/packager/creator/skeleton.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index d42d6bcaf3..cc485c16f6 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -35,7 +35,7 @@ type SkeletonCreator struct { // NewSkeletonCreator returns a new SkeletonCreator. func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.ZarfPublishOptions) *SkeletonCreator { - return &SkeletonCreator{createOpts: createOpts, publishOpts: publishOpts} + return &SkeletonCreator{createOpts, publishOpts} } // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. From 5ab9da4d647272914e3e075308155a05f63ad335 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:09:45 -0600 Subject: [PATCH 107/172] Apply changes from PR feedback --- src/pkg/packager/creator/skeleton.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index d42d6bcaf3..cc485c16f6 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -35,7 +35,7 @@ type SkeletonCreator struct { // NewSkeletonCreator returns a new SkeletonCreator. func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.ZarfPublishOptions) *SkeletonCreator { - return &SkeletonCreator{createOpts: createOpts, publishOpts: publishOpts} + return &SkeletonCreator{createOpts, publishOpts} } // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. From 2758b2feb827ecdb77aec8da361fefa647e7edcf Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:22:32 -0600 Subject: [PATCH 108/172] Move SetBaseDirectory() to a reciever on ZarfCreateOptions --- src/cmd/common/utils.go | 18 ------------------ src/cmd/dev.go | 6 +++--- src/cmd/package.go | 2 +- src/types/runtime.go | 11 +++++++++++ 4 files changed, 15 insertions(+), 22 deletions(-) delete mode 100644 src/cmd/common/utils.go diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go deleted file mode 100644 index 6b8eba9e0b..0000000000 --- a/src/cmd/common/utils.go +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package common handles command configuration across all commands -package common - -import ( - "github.com/defenseunicorns/zarf/src/types" -) - -// SetBaseDirectory sets base directory on package config when given in args -func SetBaseDirectory(args []string, createOpts *types.ZarfCreateOptions) { - if len(args) > 0 { - createOpts.BaseDir = args[0] - } else { - createOpts.BaseDir = "." - } -} diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 8e483ba2b5..d65c34327f 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -40,7 +40,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig.CreateOpts) + pkgConfig.CreateOpts.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -186,7 +186,7 @@ var devFindImagesCmd = &cobra.Command{ Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { // If a directory was provided, use that as the base directory - common.SetBaseDirectory(args, &pkgConfig.CreateOpts) + pkgConfig.CreateOpts.SetBaseDirectory(args) // Ensure uppercase keys from viper v := common.GetViper() @@ -232,7 +232,7 @@ var devLintCmd = &cobra.Command{ Short: lang.CmdDevLintShort, Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig.CreateOpts) + pkgConfig.CreateOpts.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) diff --git a/src/cmd/package.go b/src/cmd/package.go index de1aefe3f3..031df8cd9a 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -40,7 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig.CreateOpts) + pkgConfig.CreateOpts.SetBaseDirectory(args) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/types/runtime.go b/src/types/runtime.go index 3878bb0aef..a7a9d4ee06 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -115,6 +115,17 @@ type ZarfCreateOptions struct { NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } +// SetBaseDirectory sets the base directory. +// +// This is a directory that contains a zarf.yaml. +func (createOpts *ZarfCreateOptions) SetBaseDirectory(args []string) { + if len(args) > 0 { + createOpts.BaseDir = args[0] + } else { + createOpts.BaseDir = "." + } +} + // ZarfSplitPackageData contains info about a split package. type ZarfSplitPackageData struct { Sha256Sum string `json:"sha256Sum" jsonschema:"description=The sha256sum of the package"` From c0a6388219297a6c3b3767c04e47560ec45a9290 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:25:23 -0600 Subject: [PATCH 109/172] lint --- src/cmd/dev.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index d65c34327f..fb452ebdfa 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -39,7 +39,7 @@ var devDeployCmd = &cobra.Command{ Args: cobra.MaximumNArgs(1), Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { pkgConfig.CreateOpts.SetBaseDirectory(args) v := common.GetViper() @@ -65,7 +65,7 @@ var devTransformGitLinksCmd = &cobra.Command{ Aliases: []string{"p"}, Short: lang.CmdDevPatchGitShort, Args: cobra.ExactArgs(2), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { host, fileName := args[0], args[1] // Read the contents of the given file From 8595e1fe24ae910504a7bd0657b8ea79ce63e338 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:36:16 -0600 Subject: [PATCH 110/172] Directly assign StageSBOMViewFiles warnings to p.warnings --- src/pkg/packager/mirror.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 972b033ed3..30038b3dda 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -27,13 +27,11 @@ func (p *Packager) Mirror() (err error) { return err } - sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + p.warnings, err = p.layout.SBOMs.StageSBOMViewFiles() if err != nil { return err } - p.warnings = append(p.warnings, sbomWarnings...) - // Confirm the overall package mirror if !p.confirmAction(config.ZarfMirrorStage) { return fmt.Errorf("mirror cancelled") From fdeb7a47c06dd6a69dcd2336d5e9f7dfde3941f9 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:38:25 -0600 Subject: [PATCH 111/172] Directly assign p.cfg.Pkg to output of ComposeComponents --- src/pkg/packager/prepare.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 99dfbd4a2e..f56fc329ab 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -59,15 +59,13 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) + p.cfg.Pkg, p.warnings, err = creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) if err != nil { return nil, err } - p.warnings = append(p.warnings, composeWarnings...) - // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts.SetVariables) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(p.cfg.Pkg, p.cfg.CreateOpts.SetVariables) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } From 0ac00cf17f0a36ed9432e076b33f2811c732b4f5 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:47:35 -0600 Subject: [PATCH 112/172] Fix prepare logic --- src/pkg/packager/prepare.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index f56fc329ab..d854f6ef29 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -59,17 +59,17 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) - p.cfg.Pkg, p.warnings, err = creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) + composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) if err != nil { return nil, err } + p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(p.cfg.Pkg, p.cfg.CreateOpts.SetVariables) + templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts.SetVariables) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } - p.warnings = append(p.warnings, templateWarnings...) for _, warning := range p.warnings { From da83c693fc5839b04b9031e1a06f6f4854627009 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 8 Feb 2024 23:59:31 -0600 Subject: [PATCH 113/172] Fix migration warnings in ComposeComponents --- src/pkg/packager/creator/compose.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index a0ce1a4a63..406e5e56f5 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -37,7 +37,8 @@ func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, message.Debugf("%s", chain) // migrate any deprecated component configurations now - warnings = chain.Migrate(pkg.Build) + warning := chain.Migrate(pkg.Build) + warnings = append(warnings, warning...) // get the composed component composed, err := chain.Compose() From 7f2616505723c27f63d812709406b6173cb73792 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 9 Feb 2024 01:15:08 -0600 Subject: [PATCH 114/172] Set pkg arch in LoadPackageDefinition Set pkg.Build.Differential in LoadPackageDefinition Call setPackageMetadata() in Output() --- src/pkg/packager/creator/normal.go | 14 ++++++++++---- src/pkg/packager/creator/skeleton.go | 10 ++++++---- src/pkg/packager/creator/template.go | 2 +- src/pkg/packager/creator/utils.go | 12 ------------ 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index e2194731d9..f2aeb6c96e 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -62,9 +62,8 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - pkg, err = setPackageMetadata(pkg, pc.createOpts) - if err != nil { - message.Warn(err.Error()) + if pkg.Metadata.Architecture == "" { + pkg.Metadata.Architecture = config.GetArch() } // Compose components into a single zarf.yaml file @@ -90,7 +89,9 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t } // If we are creating a differential package, remove duplicate images and repos. - if pkg.Build.Differential { + if pc.createOpts.DifferentialData.DifferentialPackagePath != "" { + pkg.Build.Differential = true + diffData, err := pc.loadDifferentialData() if err != nil { return types.ZarfPackage{}, nil, err @@ -248,6 +249,11 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage deprecated.PluralizeSetVariable, } + pkg, err = setPackageMetadata(pkg, pc.createOpts) + if err != nil { + return err + } + // Save the transformed config. if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index cc485c16f6..28acec4c10 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -44,10 +44,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - pkg, err = setPackageMetadata(pkg, sc.createOpts) - if err != nil { - message.Warn(err.Error()) - } + pkg.Metadata.Architecture = "skeleton" // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) @@ -105,6 +102,11 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackag } pkg.Metadata.AggregateChecksum = checksumChecksum + pkg, err = setPackageMetadata(pkg, sc.createOpts) + if err != nil { + return err + } + if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return err } diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index fade169312..192a9bc7f8 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -66,7 +66,7 @@ func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) ( } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = pkg.Build.Architecture + templateMap[types.ZarfPackageArch] = pkg.Metadata.Architecture if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { return types.ZarfPackage{}, nil, err diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 38416ceb13..a6f6ef4fdd 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -35,13 +35,6 @@ func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOption pkg.Metadata.Version = config.CLIVersion } - // Set package architecture - if createOpts.IsSkeleton { - pkg.Metadata.Architecture = "skeleton" - } - if pkg.Metadata.Architecture == "" { - pkg.Metadata.Architecture = config.GetArch() - } pkg.Build.Architecture = pkg.Metadata.Architecture // Record the time of package creation. @@ -53,11 +46,6 @@ func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOption // Record the hostname of the package creation terminal. pkg.Build.Terminal = hostname - // If the --differential flag was used, record that this is a differential package. - if createOpts.DifferentialData.DifferentialPackagePath != "" { - pkg.Build.Differential = true - } - // Record the flavor of Zarf used to build this package (if any). pkg.Build.Flavor = createOpts.Flavor From 4b77a9061aeb8a710f3dcb1452473af8e7e0c524 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 9 Feb 2024 01:19:01 -0600 Subject: [PATCH 115/172] lint --- src/cmd/initialize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index cf2780839d..3f0f5a6e67 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -33,7 +33,7 @@ var initCmd = &cobra.Command{ Short: lang.CmdInitShort, Long: lang.CmdInitLong, Example: lang.CmdInitExample, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { zarfLogo := message.GetLogo() _, _ = fmt.Fprintln(os.Stderr, zarfLogo) From b5a05ca7c1c511a73b30aaf1b23df773a0a3f1fa Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 9 Feb 2024 02:25:21 -0600 Subject: [PATCH 116/172] Update Output() to receive a pointer ZarfPackage --- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/creator.go | 2 +- src/pkg/packager/creator/normal.go | 9 ++++----- src/pkg/packager/creator/skeleton.go | 5 ++--- src/pkg/packager/creator/utils.go | 6 +++--- src/pkg/packager/publish.go | 2 +- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index af33464450..9b01c6fefe 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -52,5 +52,5 @@ func (p *Packager) Create() (err error) { return err } - return pc.Output(p.layout, p.cfg.Pkg) + return pc.Output(p.layout, &p.cfg.Pkg) } diff --git a/src/pkg/packager/creator/creator.go b/src/pkg/packager/creator/creator.go index c9428d771a..5b0f3c27b1 100644 --- a/src/pkg/packager/creator/creator.go +++ b/src/pkg/packager/creator/creator.go @@ -13,5 +13,5 @@ import ( type Creator interface { LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error - Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error + Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index f2aeb6c96e..7813395615 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -226,7 +226,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range pkg.Components { @@ -249,8 +249,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage deprecated.PluralizeSetVariable, } - pkg, err = setPackageMetadata(pkg, pc.createOpts) - if err != nil { + if err := setPackageMetadata(pkg, pc.createOpts); err != nil { return err } @@ -278,7 +277,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage return err } - err = remote.PublishPackage(&pkg, dst, config.CommonOptions.OCIConcurrency) + err = remote.PublishPackage(pkg, dst, config.CommonOptions.OCIConcurrency) if err != nil { return fmt.Errorf("unable to publish package: %w", err) } @@ -293,7 +292,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(pkg, pc.createOpts.DifferentialData)) + packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(*pkg, pc.createOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 28acec4c10..85eaff2a93 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -89,7 +89,7 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, components []types // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackage) error { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error { for _, component := range pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err @@ -102,8 +102,7 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg types.ZarfPackag } pkg.Metadata.AggregateChecksum = checksumChecksum - pkg, err = setPackageMetadata(pkg, sc.createOpts) - if err != nil { + if err := setPackageMetadata(pkg, sc.createOpts); err != nil { return err } diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index a6f6ef4fdd..114812fd58 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -15,7 +15,7 @@ import ( ) // setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOptions) (types.ZarfPackage, error) { +func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) error { now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. @@ -28,7 +28,7 @@ func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOption hostname, err := os.Hostname() if err != nil { - return types.ZarfPackage{}, err + return err } if pkg.IsInitConfig() { @@ -54,5 +54,5 @@ func setPackageMetadata(pkg types.ZarfPackage, createOpts types.ZarfCreateOption // Record the latest version of Zarf without breaking changes to the package structure. pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - return pkg, nil + return nil } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 574cc0fa4b..6b89d45e08 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -100,7 +100,7 @@ func (p *Packager) Publish() (err error) { return err } - if err := sc.Output(p.layout, p.cfg.Pkg); err != nil { + if err := sc.Output(p.layout, &p.cfg.Pkg); err != nil { return err } } else { From 34384a9f4b4f1844c6504a4f48316ad5fefd4c02 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 10:05:46 -0600 Subject: [PATCH 117/172] Veryify to verify --- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/creator/skeleton.go | 2 +- src/pkg/packager/sources/oci.go | 2 +- src/pkg/packager/sources/split.go | 2 +- src/pkg/packager/sources/tarball.go | 2 +- src/pkg/packager/sources/url.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 7813395615..f319705bcc 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -34,7 +34,7 @@ import ( ) var ( - // veryify that PackageCreator implements Creator + // verify that PackageCreator implements Creator _ Creator = (*PackageCreator)(nil) ) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 85eaff2a93..e9be36943d 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -23,7 +23,7 @@ import ( ) var ( - // veryify that SkeletonCreator implements Creator + // verify that SkeletonCreator implements Creator _ Creator = (*SkeletonCreator)(nil) ) diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 371c53a7b5..92dcac1216 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -23,7 +23,7 @@ import ( ) var ( - // veryify that OCISource implements PackageSource + // verify that OCISource implements PackageSource _ PackageSource = (*OCISource)(nil) ) diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 1aade2eee5..fdd6f511ff 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -20,7 +20,7 @@ import ( ) var ( - // veryify that SplitTarballSource implements PackageSource + // verify that SplitTarballSource implements PackageSource _ PackageSource = (*SplitTarballSource)(nil) ) diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e65d0b962a..87b01407b3 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -21,7 +21,7 @@ import ( ) var ( - // veryify that TarballSource implements PackageSource + // verify that TarballSource implements PackageSource _ PackageSource = (*TarballSource)(nil) ) diff --git a/src/pkg/packager/sources/url.go b/src/pkg/packager/sources/url.go index 8d65cf6a9c..11f8664302 100644 --- a/src/pkg/packager/sources/url.go +++ b/src/pkg/packager/sources/url.go @@ -18,7 +18,7 @@ import ( ) var ( - // veryify that URLSource implements PackageSource + // verify that URLSource implements PackageSource _ PackageSource = (*URLSource)(nil) ) From c2581c332e9fd88273ebdda2e25a4c43a25aa2e2 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 10:59:50 -0600 Subject: [PATCH 118/172] No need to double-check if diffPkgPath is not empty --- src/pkg/packager/creator/differential.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 254921a922..35f40d8d4d 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -22,11 +22,7 @@ import ( // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialData, err error) { pkgOpts := types.ZarfPackageOptions{} - - diffPkgPath := pc.createOpts.DifferentialData.DifferentialPackagePath - if diffPkgPath != "" { - pkgOpts.PackageSource = diffPkgPath - } + pkgOpts.PackageSource = pc.createOpts.DifferentialData.DifferentialPackagePath tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { From a25d584e93c22b4b3059f1f8e21e73026ac1a404 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 12:31:30 -0600 Subject: [PATCH 119/172] Refactor SetBaseDirectory() to match the razzle version --- src/cmd/common/utils.go | 14 ++++++++++++++ src/cmd/dev.go | 7 +++---- src/cmd/package.go | 2 +- src/types/runtime.go | 11 ----------- 4 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 src/cmd/common/utils.go diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go new file mode 100644 index 0000000000..4982c578c2 --- /dev/null +++ b/src/cmd/common/utils.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package common handles command configuration across all commands +package common + +// SetBaseDirectory sets the base directory. This is a directory with a zarf.yaml. +func SetBaseDirectory(args []string) string { + if len(args) > 0 { + return args[0] + } else { + return "." + } +} diff --git a/src/cmd/dev.go b/src/cmd/dev.go index fb452ebdfa..0642b18c10 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -40,7 +40,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(_ *cobra.Command, args []string) { - pkgConfig.CreateOpts.SetBaseDirectory(args) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -185,8 +185,7 @@ var devFindImagesCmd = &cobra.Command{ Short: lang.CmdDevFindImagesShort, Long: lang.CmdDevFindImagesLong, Run: func(cmd *cobra.Command, args []string) { - // If a directory was provided, use that as the base directory - pkgConfig.CreateOpts.SetBaseDirectory(args) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) // Ensure uppercase keys from viper v := common.GetViper() @@ -232,7 +231,7 @@ var devLintCmd = &cobra.Command{ Short: lang.CmdDevLintShort, Long: lang.CmdDevLintLong, Run: func(cmd *cobra.Command, args []string) { - pkgConfig.CreateOpts.SetBaseDirectory(args) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) diff --git a/src/cmd/package.go b/src/cmd/package.go index 031df8cd9a..b9d83453af 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -40,7 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - pkgConfig.CreateOpts.SetBaseDirectory(args) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { diff --git a/src/types/runtime.go b/src/types/runtime.go index a7a9d4ee06..3878bb0aef 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -115,17 +115,6 @@ type ZarfCreateOptions struct { NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } -// SetBaseDirectory sets the base directory. -// -// This is a directory that contains a zarf.yaml. -func (createOpts *ZarfCreateOptions) SetBaseDirectory(args []string) { - if len(args) > 0 { - createOpts.BaseDir = args[0] - } else { - createOpts.BaseDir = "." - } -} - // ZarfSplitPackageData contains info about a split package. type ZarfSplitPackageData struct { Sha256Sum string `json:"sha256Sum" jsonschema:"description=The sha256sum of the package"` From 41bb66b7230e0a9673fc1952670a64695583f5f1 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 12:36:12 -0600 Subject: [PATCH 120/172] Move get pkg name functions to sources --- src/cmd/initialize.go | 2 +- src/pkg/packager/common.go | 4 +-- src/pkg/packager/creator/normal.go | 3 ++- src/pkg/packager/sources/utils.go | 31 +++++++++++++++++++++ src/pkg/utils/zarf_package.go | 43 ------------------------------ 5 files changed, 36 insertions(+), 47 deletions(-) delete mode 100644 src/pkg/utils/zarf_package.go diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 3f0f5a6e67..3aa5326be8 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -42,7 +42,7 @@ var initCmd = &cobra.Command{ } // Continue running package deploy for all components like any other package - initPackageName := utils.GetInitPackageName("") + initPackageName := sources.GetInitPackageName("") pkgConfig.PkgOpts.PackageSource = initPackageName // Try to use an init-package in the executable directory if none exist in current working directory diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 3a53528ccc..fe36dc85be 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -46,10 +46,10 @@ type Packager struct { // Zarf Packager Variables. var ( // Find zarf-packages on the local system (https://regex101.com/r/TUUftK/1) - ZarfPackagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) + // ZarfPackagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) // Find zarf-init packages on the local system - ZarfInitPattern = regexp.MustCompile(utils.GetInitPackageName("") + "$") + ZarfInitPattern = regexp.MustCompile(sources.GetInitPackageName("") + "$") ) // Modifier is a function that modifies the packager. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index f319705bcc..70b1b1be2d 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -26,6 +26,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/actions" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -292,7 +293,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackag message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.createOpts.Output, utils.GetPackageName(*pkg, pc.createOpts.DifferentialData)) + packageName := filepath.Join(pc.createOpts.Output, sources.GetPackageName(*pkg, pc.createOpts.DifferentialData)) // Try to remove the package if it already exists. _ = os.Remove(packageName) diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 383cc262cd..1c1b72db3b 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -132,3 +132,34 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { return name } + +// GetInitPackageName returns the formatted name of the init package. +func GetInitPackageName(arch string) string { + if arch == "" { + // No package has been loaded yet so lookup GetArch() with no package info + arch = config.GetArch() + } + return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) +} + +// GetPackageName returns the formatted name of the package. +func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { + if pkg.IsInitConfig() { + return GetInitPackageName(pkg.Metadata.Architecture) + } + + packageName := pkg.Metadata.Name + suffix := "tar.zst" + if pkg.Metadata.Uncompressed { + suffix = "tar" + } + + packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) + if pkg.Build.Differential { + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) + } else if pkg.Metadata.Version != "" { + packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) + } + + return fmt.Sprintf("%s.%s", packageFileName, suffix) +} diff --git a/src/pkg/utils/zarf_package.go b/src/pkg/utils/zarf_package.go deleted file mode 100644 index 84c4f614d7..0000000000 --- a/src/pkg/utils/zarf_package.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/types" -) - -// GetInitPackageName returns the formatted name of the init package. -func GetInitPackageName(arch string) string { - if arch == "" { - // No package has been loaded yet so lookup GetArch() with no package info - arch = config.GetArch() - } - return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) -} - -// GetPackageName returns the formatted name of the package. -func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { - if pkg.IsInitConfig() { - return GetInitPackageName(pkg.Metadata.Architecture) - } - - packageName := pkg.Metadata.Name - suffix := "tar.zst" - if pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) - if pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) - } else if pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) - } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) -} From 0a498201ce21b32be673e64ef9e5bfdeb4178635 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 12:59:14 -0600 Subject: [PATCH 121/172] Bring back IsTarball --- src/pkg/layout/package.go | 2 +- src/pkg/layout/sbom.go | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 4f2347cac3..0c55f5f0ac 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -323,7 +323,7 @@ func (pp *PackagePaths) Files() map[string]string { add(tarball) } - if filepath.Ext(pp.SBOMs.Path) == ".tar" { + if pp.SBOMs.IsTarball() { add(pp.SBOMs.Path) } return pathMap diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 47a94dac47..fe7dcefd02 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -70,8 +70,7 @@ func (s *SBOMs) Archive() (err error) { // StageSBOMViewFiles copies SBOM viewer HTML files to the Zarf SBOM directory. func (s *SBOMs) StageSBOMViewFiles() (warnings []string, err error) { - isTarball := !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" - if isTarball { + if s.IsTarball() { return nil, fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", s.Path) } @@ -105,3 +104,8 @@ func (s *SBOMs) OutputSBOMFiles(outputDir, packageName string) (string, error) { return packagePath, utils.CreatePathAndCopy(s.Path, packagePath) } + +// IsTarball returns true if the SBOMs are a tarball. +func (s SBOMs) IsTarball() bool { + return !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" +} From d4e9e853d803a4e83e8e6110e5f65cb7fc3afdd7 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:11:04 -0600 Subject: [PATCH 122/172] Update package create lifecycle diagram --- .../5-package-create-lifecycle.md | 73 ++++++++++--------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md index 58da969df8..0698789af0 100644 --- a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md +++ b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md @@ -6,50 +6,51 @@ The following diagram shows the order of operations for the `zarf package create ```mermaid graph TD - A1(set working directory)-->A2 - A2(parse zarf.yaml)-->A3 - A3(filter components by architecture)-->A4 - A4(detect init package)-->A5 - A5(handle deprecations)-->A6 + A1(cd to directory with zarf.yaml)-->A2 + A2(load zarf.yaml into memory)-->A3 + A3(set pkg architecture if not provided)-->A4 + A4(filter components by arch and flavor)-->A5 + A5(migrate deprecated component configs)-->A6 A6(parse component imports)-->A7 A7(process create-time variables)-->A8 - A8(write build data and zarf.yaml)-->A9 - - A9(run validations)-->A10 - A10(confirm package create):::prompt-->A11 - A11{Init package?} - A11 -->|Yes| A12(add seed image)-->A13 - A11 -->|No| A13 - - subgraph - A13(add each component)-->A13 - A13 --> A14(run each '.actions.onCreate.before'):::action-->A14 - A14 --> A15(load '.charts')-->A16 - A16(load '.files')-->A17 - A17(load '.dataInjections')-->A18 - A18(load '.manifests')-->A19 - A19(load '.repos')-->A20 - A20(run each '.actions.onCreate.after'):::action-->A20 - A20-->A21{Success?} - A21-->|Yes|A22(run each\n'.actions.onCreate.success'):::action-->A22 - A21-->|No|A23(run each\n'.actions.onCreate.failure'):::action-->A23-->A999 + A8(process extensions)-->A9 + A9(remove duplicate images/repos if --differential flag used)-->A10 + A10(run validations)-->A11 + A11(confirm package create):::prompt-->A12 + + subgraph Add Each Component + A12(run each '.actions.onCreate.before'):::action-->A13(load '.charts') + A13-->A14(load '.files') + A14-->A15(load '.dataInjections') + A15-->A16(load '.manifests') + A16-->A17(load '.repos') + A17-->A18(run each '.actions.onCreate.after'):::action + A18-->A19{Success?} + A19-->|Yes|A20(run each\n'.actions.onCreate.success'):::action + A19-->|No|A999 end - A22-->A24(load all '.images') - A24-->A25{Skip SBOM?} - A25-->|Yes|A27 - A25-->|No|A26 - A26(generate SBOM)-->A27 - A27(reset working directory)-->A28 - A28(create package archive)-->A29 - A29{Is multipart?} - A29-->|Yes|A30(split package archive)-->A31 - A29-->|No|A31 - A31(handle sbom view/out flags) + A20-->A21(load '.images') + A21-->A22(generate SBOMs unless --skip-sbom flag was used) + A22-->A23(cd back to original working directory) + A23-->A24(archive components into tarballs) + A24-->A25(generate checksums for all pkg files) + A25-->A26(record pkg build metadata) + A26-->A27(write the zarf.yaml to disk) + A27-->A28(sign the package if a key was provided) + A28-->A29{Output to OCI?} + A29-->|Yes|A30(publish pkg to OCI registry) + A29-->|No|A31(archive pkg into a tarball and write to disk) + A30-->A32 + A31-->A32 + A32(write SBOM files to disk if --sbom or --sbom-out flags used)-->A33 + A33(view SBOMs if --sbom flag used)-->A34 + A34[Zarf Package Create Successful]:::success A999[Abort]:::fail classDef prompt fill:#4adede,color:#000000 classDef action fill:#bd93f9,color:#000000 classDef fail fill:#aa0000 + classDef success fill:#008000,color:#fff; ``` From 9a4a8bea55c3c0deee6c76b66934adf688e43cce Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:12:15 -0600 Subject: [PATCH 123/172] Update src/pkg/layout/package.go Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/pkg/layout/package.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 0c55f5f0ac..6946b7df03 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -145,9 +145,7 @@ func (pp *PackagePaths) IsLegacyLayout() bool { // SignPackage signs the zarf.yaml in a Zarf package. func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { - if signingKeyPath != "" { - pp.Signature = filepath.Join(pp.Base, Signature) - } + pp.Signature = filepath.Join(pp.Base, Signature) passwordFunc := func(_ bool) ([]byte, error) { if signingKeyPassword != "" { From b1746d184f906cd67637c5f610269a58686171d5 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:13:24 -0600 Subject: [PATCH 124/172] Update src/pkg/layout/package_test.go Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/pkg/layout/package_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index 1a3d4c5aa0..8e7036f066 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -90,6 +90,15 @@ func TestPackageFiles(t *testing.T) { "checksums.txt": normalizePath("test/checksums.txt"), } require.Equal(t, expected, files) + + pp.SBOMs.Path = normalizePath("test/sboms.tar") + files = pp.Files() + expected = map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "sboms.tar": normalizePath("test/sboms.tar"), + } + require.Equal(t, expected, files) }) t.Run("Verify Files() with paths mapped to package paths", func(t *testing.T) { From a4a90dd0bee0bac085066300b4e98240fd5cae53 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:16:11 -0600 Subject: [PATCH 125/172] Remove unused pkg pattern vars --- src/pkg/packager/common.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index fe36dc85be..a81f178c3e 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "os" - "regexp" "strings" "time" @@ -43,15 +42,6 @@ type Packager struct { generation int } -// Zarf Packager Variables. -var ( - // Find zarf-packages on the local system (https://regex101.com/r/TUUftK/1) - // ZarfPackagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) - - // Find zarf-init packages on the local system - ZarfInitPattern = regexp.MustCompile(sources.GetInitPackageName("") + "$") -) - // Modifier is a function that modifies the packager. type Modifier func(*Packager) From 6903909024b7a8009e9fe6c8bb56286a97956e79 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:17:34 -0600 Subject: [PATCH 126/172] Apply PR feedback --- src/pkg/packager/creator/compose.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index 406e5e56f5..deb15938ca 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -10,7 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// ComposeComponents composes components and their dependencies into a single zarf.yaml using an import chain. +// ComposeComponents composes components and their dependencies into a single Zarf package using an import chain. func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, []string, error) { components := []types.ZarfComponent{} warnings := []string{} From 12bb5dd951eb6e69a78521e5f928a0cd666331ff Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 12 Feb 2024 16:23:52 -0600 Subject: [PATCH 127/172] lint --- src/cmd/dev.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 0642b18c10..b20cbbba75 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -108,7 +108,7 @@ var devSha256SumCmd = &cobra.Command{ Aliases: []string{"s"}, Short: lang.CmdDevSha256sumShort, Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, args []string) { fileName := args[0] var tmp string From f013bbf8f2d8a33bdcb7d1a1d101d1aba00fa83b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 13 Feb 2024 13:47:48 -0600 Subject: [PATCH 128/172] Apply feedback to package create lifecycle diagram --- .../5-package-create-lifecycle.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md index 0698789af0..b4de448c31 100644 --- a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md +++ b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md @@ -8,8 +8,8 @@ The following diagram shows the order of operations for the `zarf package create graph TD A1(cd to directory with zarf.yaml)-->A2 A2(load zarf.yaml into memory)-->A3 - A3(set pkg architecture if not provided)-->A4 - A4(filter components by arch and flavor)-->A5 + A3(set package architecture if not provided)-->A4 + A4(filter components by architecture and flavor)-->A5 A5(migrate deprecated component configs)-->A6 A6(parse component imports)-->A7 A7(process create-time variables)-->A8 @@ -30,17 +30,17 @@ graph TD A19-->|No|A999 end - A20-->A21(load '.images') + A20-->A21(load all '.images') A21-->A22(generate SBOMs unless --skip-sbom flag was used) A22-->A23(cd back to original working directory) A23-->A24(archive components into tarballs) - A24-->A25(generate checksums for all pkg files) - A25-->A26(record pkg build metadata) + A24-->A25(generate checksums for all package files) + A25-->A26(record package build metadata) A26-->A27(write the zarf.yaml to disk) A27-->A28(sign the package if a key was provided) A28-->A29{Output to OCI?} - A29-->|Yes|A30(publish pkg to OCI registry) - A29-->|No|A31(archive pkg into a tarball and write to disk) + A29-->|Yes|A30(publish package to OCI registry) + A29-->|No|A31(archive package into a tarball and write to disk) A30-->A32 A31-->A32 A32(write SBOM files to disk if --sbom or --sbom-out flags used)-->A33 From f804d07528ead4bfbaef192ffd24acfec17c3005 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 13 Feb 2024 13:54:30 -0600 Subject: [PATCH 129/172] Apply feedback to package create lifecycle diagram --- docs/3-create-a-zarf-package/5-package-create-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md index b4de448c31..fa1a0a8d6d 100644 --- a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md +++ b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md @@ -18,7 +18,7 @@ graph TD A10(run validations)-->A11 A11(confirm package create):::prompt-->A12 - subgraph Add Each Component + subgraph A12(run each '.actions.onCreate.before'):::action-->A13(load '.charts') A13-->A14(load '.files') A14-->A15(load '.dataInjections') From a4e160fa947660fc1cdfbd3480d9f2d09f44cb8b Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 13 Feb 2024 16:09:18 -0600 Subject: [PATCH 130/172] Update comment to include filter by flavor --- src/pkg/packager/creator/compose.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index deb15938ca..343e8f9bc8 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -20,7 +20,7 @@ func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, for i, component := range pkg.Components { arch := pkg.Metadata.Architecture - // filter by architecture + // filter by architecture and flavor if !composer.CompatibleComponent(component, arch, flavor) { continue } From dc6d5e3efcf3e8763d766ff478a3468bd43a922d Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 13 Feb 2024 18:38:56 -0600 Subject: [PATCH 131/172] Build kustomizations during skeleton create --- src/pkg/packager/creator/skeleton.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index e9be36943d..058ffea8a2 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/extensions/bigbang" "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -301,6 +302,19 @@ func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layo updatedComponent.Manifests[manifestIdx].Files[fileIdx] = rel } + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return nil, fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + } + // remove kustomizations updatedComponent.Manifests[manifestIdx].Kustomizations = nil } From bdcd265ee90092351c14a15bd53385c3bc8b6cae Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 14 Feb 2024 12:07:36 -0600 Subject: [PATCH 132/172] Add logic to skeleton test suite to verify kustomize-generated manifests --- src/test/e2e/51_oci_compose_test.go | 19 +++++++++++++++++++ .../packages/09-composable-packages/zarf.yaml | 1 + 2 files changed, 20 insertions(+) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 4ad8769d8e..c7725c97f5 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -71,6 +71,11 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.NoError(err) suite.Contains(stdErr, "Published "+ref) + composable := filepath.Join("src", "test", "packages", "09-composable-packages") + _, stdErr, err = e2e.Zarf("package", "publish", composable, "oci://"+ref, "--insecure") + suite.NoError(err) + suite.Contains(stdErr, "Published "+ref) + _, stdErr, err = e2e.Zarf("package", "publish", importEverything, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -86,6 +91,9 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0", "-o", "build", "--insecure", "-a", "skeleton") suite.NoError(err) + + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/test-compose-package:0.0.1", "-o", "build", "--insecure", "-a", "skeleton") + suite.NoError(err) } func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { @@ -122,6 +130,7 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst"), filepath.Join("build", "zarf-package-big-bang-min-skeleton-2.10.0.tar.zst"), + filepath.Join("build", "zarf-package-test-compose-package-skeleton-0.0.1.tar.zst"), } for _, pkgTar := range pkgTars { @@ -134,6 +143,16 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { suite.NoError(err) suite.DirExists(unpacked) + // Verify skeleton contains kustomize-generated manifests. + if strings.HasSuffix(pkgTar, "zarf-package-test-compose-package-skeleton-0.0.1.tar.zst") { + kustomizeGeneratedManifests := []string{"kustomization-connect-service-0.yaml", "kustomization-connect-service-1.yaml", "kustomization-connect-service-two-0.yaml"} + manifestPath := filepath.Join(unpacked, "components", "test-compose-package", "manifests") + for _, manifest := range kustomizeGeneratedManifests { + fullPath := filepath.Join(manifestPath, manifest) + suite.FileExists(fullPath, "expected to find kustomize-generated manifest: %q", fullPath) + } + } + err = utils.ReadYaml(filepath.Join(unpacked, layout.ZarfYAML), &pkg) suite.NoError(err) suite.NotNil(pkg) diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 21e40e5df2..cf121120e6 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -2,6 +2,7 @@ kind: ZarfPackageConfig metadata: name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing + version: 0.0.1 components: - name: test-compose-package From 5139b883741757d4f9afb692ec44ba1e9b55ab69 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 14 Feb 2024 13:34:42 -0600 Subject: [PATCH 133/172] Verify kustomize-generated manifests in skeleton are of kind ConfigMap --- src/test/e2e/51_oci_compose_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index c7725c97f5..6ac1e8a5e5 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -19,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" "oras.land/oras-go/v2/registry" ) @@ -150,6 +151,10 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { for _, manifest := range kustomizeGeneratedManifests { fullPath := filepath.Join(manifestPath, manifest) suite.FileExists(fullPath, "expected to find kustomize-generated manifest: %q", fullPath) + var configMap corev1.ConfigMap + err := utils.ReadYaml(fullPath, &configMap) + suite.NoError(err) + suite.Equal("ConfigMap", configMap.Kind, "expected manifest %q to be of kind ConfigMap", fullPath) } } From bbb848dcadd0ffcce58d06a597b81310867a15be Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 15 Feb 2024 14:38:06 -0600 Subject: [PATCH 134/172] Add line-wrapping and rename things to be more readable --- src/test/e2e/51_oci_compose_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 6ac1e8a5e5..4412e13226 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -146,15 +146,19 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { // Verify skeleton contains kustomize-generated manifests. if strings.HasSuffix(pkgTar, "zarf-package-test-compose-package-skeleton-0.0.1.tar.zst") { - kustomizeGeneratedManifests := []string{"kustomization-connect-service-0.yaml", "kustomization-connect-service-1.yaml", "kustomization-connect-service-two-0.yaml"} - manifestPath := filepath.Join(unpacked, "components", "test-compose-package", "manifests") + kustomizeGeneratedManifests := []string{ + "kustomization-connect-service-0.yaml", + "kustomization-connect-service-1.yaml", + "kustomization-connect-service-two-0.yaml", + } + manifestDir := filepath.Join(unpacked, "components", "test-compose-package", "manifests") for _, manifest := range kustomizeGeneratedManifests { - fullPath := filepath.Join(manifestPath, manifest) - suite.FileExists(fullPath, "expected to find kustomize-generated manifest: %q", fullPath) + manifestPath := filepath.Join(manifestDir, manifest) + suite.FileExists(manifestPath, "expected to find kustomize-generated manifest: %q", manifestPath) var configMap corev1.ConfigMap - err := utils.ReadYaml(fullPath, &configMap) + err := utils.ReadYaml(manifestPath, &configMap) suite.NoError(err) - suite.Equal("ConfigMap", configMap.Kind, "expected manifest %q to be of kind ConfigMap", fullPath) + suite.Equal("ConfigMap", configMap.Kind, "expected manifest %q to be of kind ConfigMap", manifestPath) } } From 9ae12d5a8fb4b63ca6a8882e7413b8e7e5295241 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 15 Feb 2024 14:50:35 -0600 Subject: [PATCH 135/172] Fix lint warning --- src/cmd/common/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go index 4982c578c2..1da7e456ee 100644 --- a/src/cmd/common/utils.go +++ b/src/cmd/common/utils.go @@ -8,7 +8,6 @@ package common func SetBaseDirectory(args []string) string { if len(args) > 0 { return args[0] - } else { - return "." } + return "." } From 9f080fc87bda3a2f381e7af5e6c8b24c64b43918 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 15 Feb 2024 15:15:35 -0600 Subject: [PATCH 136/172] Use a single pkg var for assignment in FindImages() --- src/pkg/packager/prepare.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index d854f6ef29..042dcefc30 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -38,6 +38,7 @@ type imageMap map[string]bool func (p *Packager) FindImages() (imgMap map[string][]string, err error) { repoHelmChartPath := p.cfg.FindImagesOpts.RepoHelmChartPath kubeVersionOverride := p.cfg.FindImagesOpts.KubeVersionOverride + pkg := p.cfg.Pkg imagesMap := make(map[string][]string) erroredCharts := []string{} @@ -53,20 +54,20 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err := utils.ReadYaml(layout.ZarfYAML, &p.cfg.Pkg); err != nil { + if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { return nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) } - p.cfg.Pkg.Metadata.Architecture = config.GetArch(p.cfg.Pkg.Metadata.Architecture) + pkg.Metadata.Architecture = config.GetArch(pkg.Metadata.Architecture) - composedPkg, composeWarnings, err := creator.ComposeComponents(p.cfg.Pkg, p.cfg.CreateOpts.Flavor) + pkg, composeWarnings, err := creator.ComposeComponents(pkg, p.cfg.CreateOpts.Flavor) if err != nil { return nil, err } p.warnings = append(p.warnings, composeWarnings...) // After components are composed, template the active package - templatedPkg, templateWarnings, err := creator.FillActiveTemplate(composedPkg, p.cfg.CreateOpts.SetVariables) + pkg, templateWarnings, err := creator.FillActiveTemplate(pkg, p.cfg.CreateOpts.SetVariables) if err != nil { return nil, fmt.Errorf("unable to fill values in template: %w", err) } @@ -76,7 +77,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { message.Warn(warning) } - for _, component := range templatedPkg.Components { + for _, component := range pkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + "if any repos contain helm charts you want to template and " + @@ -87,7 +88,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { componentDefinition := "\ncomponents:\n" - for _, component := range templatedPkg.Components { + for _, component := range pkg.Components { if len(component.Charts)+len(component.Manifests)+len(component.Repos) < 1 { // Skip if it doesn't have what we need @@ -237,7 +238,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } if len(validImages) > 0 { - componentDefinition += fmt.Sprintf(" # Possible images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) + componentDefinition += fmt.Sprintf(" # Possible images - %s - %s\n", pkg.Metadata.Name, component.Name) for _, image := range validImages { imagesMap[component.Name] = append(imagesMap[component.Name], image) componentDefinition += fmt.Sprintf(" - %s\n", image) @@ -267,7 +268,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { if len(cosignArtifactList) > 0 { imagesMap[component.Name] = append(imagesMap[component.Name], cosignArtifactList...) - componentDefinition += fmt.Sprintf(" # Cosign artifacts for images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) + componentDefinition += fmt.Sprintf(" # Cosign artifacts for images - %s - %s\n", pkg.Metadata.Name, component.Name) for _, cosignArtifact := range cosignArtifactList { componentDefinition += fmt.Sprintf(" - %s\n", cosignArtifact) } From 09ef713ae00c44e57ca97658238713acc413feae Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 09:07:00 -0600 Subject: [PATCH 137/172] Move pkgOpts to a single call in sources.New --- src/pkg/packager/creator/differential.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 35f40d8d4d..7cc148b0d8 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -21,9 +21,6 @@ import ( // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialData, err error) { - pkgOpts := types.ZarfPackageOptions{} - pkgOpts.PackageSource = pc.createOpts.DifferentialData.DifferentialPackagePath - tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return nil, err @@ -32,7 +29,9 @@ func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialDa diffLayout := layout.New(tmpdir) defer os.RemoveAll(diffLayout.Base) - src, err := sources.New(&pkgOpts) + src, err := sources.New(&types.ZarfPackageOptions{ + PackageSource: pc.createOpts.DifferentialData.DifferentialPackagePath, + }) if err != nil { return nil, err } From 6a34119f8a637e9cee35ff350333294a2944f6f2 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 09:34:09 -0600 Subject: [PATCH 138/172] Move diffPkgPath field to createOpts and move diffData out of createOpts --- src/cmd/package.go | 2 +- src/pkg/packager/creator/differential.go | 14 +++++------ src/pkg/packager/creator/normal.go | 10 +++++--- src/pkg/packager/sources/utils.go | 4 +-- src/types/package.go | 23 +++++++++--------- src/types/runtime.go | 31 ++++++++++++------------ 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/cmd/package.go b/src/cmd/package.go index 86d18a4fbb..d019bf7395 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -349,7 +349,7 @@ func bindCreateFlags(v *viper.Viper) { createFlags.StringVar(&pkgConfig.CreateOpts.Output, "output-directory", v.GetString("package.create.output_directory"), lang.CmdPackageCreateFlagOutput) createFlags.StringVarP(&pkgConfig.CreateOpts.Output, "output", "o", v.GetString(common.VPkgCreateOutput), lang.CmdPackageCreateFlagOutput) - createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialData.DifferentialPackagePath, "differential", v.GetString(common.VPkgCreateDifferential), lang.CmdPackageCreateFlagDifferential) + createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialPackagePath, "differential", v.GetString(common.VPkgCreateDifferential), lang.CmdPackageCreateFlagDifferential) createFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) createFlags.BoolVarP(&pkgConfig.CreateOpts.ViewSBOM, "sbom", "s", v.GetBool(common.VPkgCreateSbom), lang.CmdPackageCreateFlagSbom) createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(common.VPkgCreateSbomOutput), lang.CmdPackageCreateFlagSbomOut) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 7cc148b0d8..78d83c75d6 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -30,7 +30,7 @@ func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialDa defer os.RemoveAll(diffLayout.Base) src, err := sources.New(&types.ZarfPackageOptions{ - PackageSource: pc.createOpts.DifferentialData.DifferentialPackagePath, + PackageSource: pc.createOpts.DifferentialPackagePath, }) if err != nil { return nil, err @@ -57,13 +57,11 @@ func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialDa } } - pc.createOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - pc.createOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - pc.createOpts.DifferentialData.DifferentialPackageVersion = diffPkg.Metadata.Version - - diffData = &pc.createOpts.DifferentialData - - return diffData, nil + return &types.DifferentialData{ + DifferentialImages: allIncludedImagesMap, + DifferentialRepos: allIncludedReposMap, + DifferentialPackageVersion: diffPkg.Metadata.Version, + }, nil } // removeCopiesFromComponents removes any images and repos already present in the reference package components. diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 70b1b1be2d..8ab03db0cd 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -50,8 +50,8 @@ type PackageCreator struct { // NewPackageCreator returns a new PackageCreator. func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { // differentials are relative to the current working directory - if createOpts.DifferentialData.DifferentialPackagePath != "" { - createOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialData.DifferentialPackagePath) + if createOpts.DifferentialPackagePath != "" { + createOpts.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialPackagePath) } return &PackageCreator{createOpts, cfg} @@ -90,7 +90,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t } // If we are creating a differential package, remove duplicate images and repos. - if pc.createOpts.DifferentialData.DifferentialPackagePath != "" { + if pc.createOpts.DifferentialPackagePath != "" { pkg.Build.Differential = true diffData, err := pc.loadDifferentialData() @@ -98,6 +98,8 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t return types.ZarfPackage{}, nil, err } + pkg.Build.DifferentialPackageVersion = diffData.DifferentialPackageVersion + versionsMatch := diffData.DifferentialPackageVersion == pkg.Metadata.Version if versionsMatch { return types.ZarfPackage{}, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) @@ -293,7 +295,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackag message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.createOpts.Output, sources.GetPackageName(*pkg, pc.createOpts.DifferentialData)) + packageName := filepath.Join(pc.createOpts.Output, sources.GetPackageName(*pkg)) // Try to remove the package if it already exists. _ = os.Remove(packageName) diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 1c1b72db3b..96974ca0c6 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -143,7 +143,7 @@ func GetInitPackageName(arch string) string { } // GetPackageName returns the formatted name of the package. -func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) string { +func GetPackageName(pkg types.ZarfPackage) string { if pkg.IsInitConfig() { return GetInitPackageName(pkg.Metadata.Architecture) } @@ -156,7 +156,7 @@ func GetPackageName(pkg types.ZarfPackage, diffData types.DifferentialData) stri packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) if pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, diffData.DifferentialPackageVersion, pkg.Metadata.Version) + packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) } else if pkg.Metadata.Version != "" { packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) } diff --git a/src/types/package.go b/src/types/package.go index cee57b3ef8..7a4e91a503 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -58,17 +58,18 @@ type ZarfMetadata struct { // ZarfBuildData is written during the packager.Create() operation to track details of the created package. type ZarfBuildData struct { - Terminal string `json:"terminal" jsonschema:"description=The machine name that created this package"` - User string `json:"user" jsonschema:"description=The username who created this package"` - Architecture string `json:"architecture" jsonschema:"description=The architecture this package was created on"` - Timestamp string `json:"timestamp" jsonschema:"description=The timestamp when this package was created"` - Version string `json:"version" jsonschema:"description=The version of Zarf used to build this package"` - Migrations []string `json:"migrations,omitempty" jsonschema:"description=Any migrations that have been run on this package"` - Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` - RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` - DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` - LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` - Flavor string `json:"flavor,omitempty" jsonschema:"description=The flavor of Zarf used to build this package"` + Terminal string `json:"terminal" jsonschema:"description=The machine name that created this package"` + User string `json:"user" jsonschema:"description=The username who created this package"` + Architecture string `json:"architecture" jsonschema:"description=The architecture this package was created on"` + Timestamp string `json:"timestamp" jsonschema:"description=The timestamp when this package was created"` + Version string `json:"version" jsonschema:"description=The version of Zarf used to build this package"` + Migrations []string `json:"migrations,omitempty" jsonschema:"description=Any migrations that have been run on this package"` + RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` + Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` + DifferentialPackageVersion string `json:"differentialPackageVersion,omitempty" jsonschema:"description=Version of a previously built package used as the basis for creating this differential package"` + DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` + LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` + Flavor string `json:"flavor,omitempty" jsonschema:"description=The flavor of Zarf used to build this package"` } // ZarfPackageVariable are variables that can be used to dynamically template K8s resources. diff --git a/src/types/runtime.go b/src/types/runtime.go index 3878bb0aef..c32af55bb2 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -99,20 +99,20 @@ type ZarfInitOptions struct { // ZarfCreateOptions tracks the user-defined options used to create the package. type ZarfCreateOptions struct { - SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` - BaseDir string `json:"baseDir" jsonschema:"description=Location where the Zarf package will be created from"` - Output string `json:"output" jsonschema:"description=Location where the finalized Zarf package will be placed"` - ViewSBOM bool `json:"sbom" jsonschema:"description=Whether to pause to allow for viewing the SBOM post-creation"` - SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` - SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` - MaxPackageSizeMB int `json:"maxPackageSizeMB" jsonschema:"description=Size of chunks to use when splitting a zarf package into multiple files in megabytes"` - SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` - SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sigh the created package"` - DifferentialData DifferentialData `json:"differential" jsonschema:"description=A package's differential images and git repositories from a referenced previously built package"` - RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` - Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` - IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` - NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` + SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` + BaseDir string `json:"baseDir" jsonschema:"description=Location where the Zarf package will be created from"` + Output string `json:"output" jsonschema:"description=Location where the finalized Zarf package will be placed"` + ViewSBOM bool `json:"sbom" jsonschema:"description=Whether to pause to allow for viewing the SBOM post-creation"` + SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` + SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` + MaxPackageSizeMB int `json:"maxPackageSizeMB" jsonschema:"description=Size of chunks to use when splitting a zarf package into multiple files in megabytes"` + SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` + SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sigh the created package"` + DifferentialPackagePath string `json:"differentialPackagePath" jsonschema:"description=Path to a previously built package used as the basis for creating a differential package"` + RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` + Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` + IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` + NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } // ZarfSplitPackageData contains info about a split package. @@ -142,8 +142,7 @@ type ConnectStrings map[string]ConnectString // DifferentialData contains image and repository information about the package a Differential Package is Based on. type DifferentialData struct { - DifferentialPackagePath string - DifferentialPackageVersion string DifferentialImages map[string]bool DifferentialRepos map[string]bool + DifferentialPackageVersion string } From 77c2535907476ae629551f6fe42ce773e6a61fd9 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 09:36:44 -0600 Subject: [PATCH 139/172] Change composeWarnings to warnings --- src/pkg/packager/creator/skeleton.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 058ffea8a2..73efe366d4 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -48,11 +48,10 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg pkg.Metadata.Architecture = "skeleton" // Compose components into a single zarf.yaml file - pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) + pkg, warnings, err = ComposeComponents(pkg, sc.createOpts.Flavor) if err != nil { return types.ZarfPackage{}, nil, err } - warnings = append(warnings, composeWarnings...) pkg.Components, err = sc.processExtensions(pkg.Components, dst) if err != nil { From c533353a133b087c79e4a17fbc04908d747e5c13 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 09:43:19 -0600 Subject: [PATCH 140/172] Use LoadPackageDefinition in FindImages --- src/pkg/packager/prepare.go | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 042dcefc30..2c816cf7ad 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -16,7 +16,6 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -50,28 +49,16 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return nil, fmt.Errorf("unable to access directory '%s': %w", p.cfg.CreateOpts.BaseDir, err) + return nil, fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) - } - - pkg.Metadata.Architecture = config.GetArch(pkg.Metadata.Architecture) + c := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) - pkg, composeWarnings, err := creator.ComposeComponents(pkg, p.cfg.CreateOpts.Flavor) + pkg, p.warnings, err = c.LoadPackageDefinition(p.layout) if err != nil { return nil, err } - p.warnings = append(p.warnings, composeWarnings...) - - // After components are composed, template the active package - pkg, templateWarnings, err := creator.FillActiveTemplate(pkg, p.cfg.CreateOpts.SetVariables) - if err != nil { - return nil, fmt.Errorf("unable to fill values in template: %w", err) - } - p.warnings = append(p.warnings, templateWarnings...) for _, warning := range p.warnings { message.Warn(warning) From cabb9d222217aa53453c5444e61b6c59dd5aa63c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 15:56:05 -0600 Subject: [PATCH 141/172] Update src/types/packager.go Co-authored-by: razzle --- src/types/packager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/packager.go b/src/types/packager.go index 7c943ca313..d831aae166 100644 --- a/src/types/packager.go +++ b/src/types/packager.go @@ -60,7 +60,7 @@ func (cfg *PackagerConfig) SetVariable(name, value string, sensitive bool, autoI } // CheckVariablePattern checks to see if a variable is set to a value that matches its pattern. -func (cfg PackagerConfig) CheckVariablePattern(name, pattern string) error { +func (cfg *PackagerConfig) CheckVariablePattern(name, pattern string) error { if regexp.MustCompile(pattern).MatchString(cfg.SetVariableMap[name].Value) { return nil } From a128a2ed1dc0926e7b55a98b18cc14c3037844de Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 16:02:27 -0600 Subject: [PATCH 142/172] Move loadDifferentialData from a PackageCreator receiver to a standalone func --- src/pkg/packager/creator/differential.go | 4 ++-- src/pkg/packager/creator/normal.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 78d83c75d6..909bff8a31 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -20,7 +20,7 @@ import ( ) // loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. -func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialData, err error) { +func loadDifferentialData(diffPkgPath string) (diffData *types.DifferentialData, err error) { tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { return nil, err @@ -30,7 +30,7 @@ func (pc *PackageCreator) loadDifferentialData() (diffData *types.DifferentialDa defer os.RemoveAll(diffLayout.Base) src, err := sources.New(&types.ZarfPackageOptions{ - PackageSource: pc.createOpts.DifferentialPackagePath, + PackageSource: diffPkgPath, }) if err != nil { return nil, err diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 8ab03db0cd..3019c2975f 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -93,7 +93,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t if pc.createOpts.DifferentialPackagePath != "" { pkg.Build.Differential = true - diffData, err := pc.loadDifferentialData() + diffData, err := loadDifferentialData(pc.createOpts.DifferentialPackagePath) if err != nil { return types.ZarfPackage{}, nil, err } From a8c4d11c07315d10f943b9546ff0185a9d118665 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 16:05:28 -0600 Subject: [PATCH 143/172] Remove debug statements from removeCopiesFromComponents --- src/pkg/packager/creator/differential.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go index 909bff8a31..4533531e03 100644 --- a/src/pkg/packager/creator/differential.go +++ b/src/pkg/packager/creator/differential.go @@ -11,7 +11,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -80,8 +79,6 @@ func removeCopiesFromComponents(components []types.ZarfComponent, loadedDiffData includeImage := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" if includeImage || !loadedDiffData.DifferentialImages[img] { newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) } } @@ -99,8 +96,6 @@ func removeCopiesFromComponents(components []types.ZarfComponent, loadedDiffData includeRepo := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) if includeRepo || !loadedDiffData.DifferentialRepos[repoURL] { newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) } } From b40aa62e08eed37edc0deb895d3647bdbd6a8e5e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 16:06:57 -0600 Subject: [PATCH 144/172] make docs-and-schema --- docs/3-create-a-zarf-package/4-zarf-schema.md | 34 ++++++++++++++----- zarf.schema.json | 12 ++++--- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 80a989a03b..050539669f 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -409,6 +409,25 @@ Must be one of: +
+ + registryOverrides + +  +
+ + ## build > registryOverrides + +**Description:** Any registry domains that were overridden on package create when pulling images + +| | | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Type** | `object` | +| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | + +
+
+
differential @@ -425,21 +444,18 @@ Must be one of:
-
+
- registryOverrides + differentialPackageVersion  
- ## build > registryOverrides - -**Description:** Any registry domains that were overridden on package create when pulling images +**Description:** Version of a previously built package used as the basis for creating this differential package -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | +| | | +| -------- | -------- | +| **Type** | `string` |
diff --git a/zarf.schema.json b/zarf.schema.json index f8e53e98e2..539081048a 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -118,10 +118,6 @@ "type": "array", "description": "Any migrations that have been run on this package" }, - "differential": { - "type": "boolean", - "description": "Whether this package was created with differential components" - }, "registryOverrides": { "patternProperties": { ".*": { @@ -131,6 +127,14 @@ "type": "object", "description": "Any registry domains that were overridden on package create when pulling images" }, + "differential": { + "type": "boolean", + "description": "Whether this package was created with differential components" + }, + "differentialPackageVersion": { + "type": "string", + "description": "Version of a previously built package used as the basis for creating this differential package" + }, "differentialMissing": { "items": { "type": "string" From 60d5b1a80b6758963ffdc43ed670715b380d2f24 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Tue, 27 Feb 2024 16:38:17 -0600 Subject: [PATCH 145/172] Combine GetPackageName w/ NameFromMetadata --- src/cmd/initialize.go | 2 +- src/pkg/packager/creator/normal.go | 7 +++--- src/pkg/packager/sources/oci.go | 9 +------- src/pkg/packager/sources/utils.go | 35 +++++++++--------------------- 4 files changed, 16 insertions(+), 37 deletions(-) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 3aa5326be8..7ea378d494 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -42,7 +42,7 @@ var initCmd = &cobra.Command{ } // Continue running package deploy for all components like any other package - initPackageName := sources.GetInitPackageName("") + initPackageName := sources.GetInitPackageName() pkgConfig.PkgOpts.PackageSource = initPackageName // Try to use an init-package in the executable directory if none exist in current working directory diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 3019c2975f..3b44201281 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -295,13 +295,14 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackag message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) } else { // Use the output path if the user specified it. - packageName := filepath.Join(pc.createOpts.Output, sources.GetPackageName(*pkg)) + packageName := fmt.Sprintf("%s%s", sources.NameFromMetadata(pkg, pc.createOpts.IsSkeleton), sources.PkgSuffix(pkg.Metadata.Uncompressed)) + tarballPath := filepath.Join(pc.createOpts.Output, packageName) // Try to remove the package if it already exists. - _ = os.Remove(packageName) + _ = os.Remove(tarballPath) // Create the package tarball. - if err := dst.ArchivePackage(packageName, pc.createOpts.MaxPackageSizeMB); err != nil { + if err := dst.ArchivePackage(tarballPath, pc.createOpts.MaxPackageSizeMB); err != nil { return fmt.Errorf("unable to archive package: %w", err) } } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 92dcac1216..a9bcbee86d 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -199,17 +199,10 @@ func (s *OCISource) Collect(dir string) (string, error) { // TODO (@Noxsios) remove the suffix check at v1.0.0 isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, oci.SkeletonArch) - name := NameFromMetadata(&pkg, isSkeleton) + name := fmt.Sprintf("%s%s", NameFromMetadata(&pkg, isSkeleton), PkgSuffix(pkg.Metadata.Uncompressed)) dstTarball := filepath.Join(dir, name) - // honor uncompressed flag - if pkg.Metadata.Uncompressed { - dstTarball = dstTarball + ".tar" - } else { - dstTarball = dstTarball + ".tar.zst" - } - allTheLayers, err := filepath.Glob(filepath.Join(tmp, "*")) if err != nil { return "", err diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 96974ca0c6..65c3506b10 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -125,7 +125,7 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { } if pkg.Build.Differential { - name = fmt.Sprintf("%s-differential-%s", name, pkg.Metadata.Version) + name = fmt.Sprintf("%s-%s-differential-%s", name, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) } else if pkg.Metadata.Version != "" { name = fmt.Sprintf("%s-%s", name, pkg.Metadata.Version) } @@ -134,32 +134,17 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { } // GetInitPackageName returns the formatted name of the init package. -func GetInitPackageName(arch string) string { - if arch == "" { - // No package has been loaded yet so lookup GetArch() with no package info - arch = config.GetArch() - } +func GetInitPackageName() string { + // No package has been loaded yet so lookup GetArch() with no package info + arch := config.GetArch() return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) } -// GetPackageName returns the formatted name of the package. -func GetPackageName(pkg types.ZarfPackage) string { - if pkg.IsInitConfig() { - return GetInitPackageName(pkg.Metadata.Architecture) - } - - packageName := pkg.Metadata.Name - suffix := "tar.zst" - if pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, pkg.Metadata.Architecture) - if pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) - } else if pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, pkg.Metadata.Version) +// PkgSuffix returns a package suffix based on whether it is uncompressed or not. +func PkgSuffix(uncompressed bool) (suffix string) { + suffix = ".tar.zst" + if uncompressed { + suffix = ".tar" } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) + return suffix } From db52da107d7c6d39d1d458a74a716882dad7b516 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:05:06 -0600 Subject: [PATCH 146/172] Bring back ReadZarfYAML as a PackagePaths receiver --- src/pkg/layout/package.go | 24 ++++++++++++++++++++++++ src/pkg/packager/creator/normal.go | 5 +++-- src/pkg/packager/creator/skeleton.go | 9 ++++++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 6946b7df03..e9c9bde5dd 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -14,6 +14,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" @@ -55,6 +56,29 @@ func New(baseDir string) *PackagePaths { } } +// ReadZarfYAML reads a zarf.yaml file into memory, +// checks if it's using the legacy layout, and migrates deprecated component configs. +func (pp *PackagePaths) ReadZarfYAML(path string) (pkg types.ZarfPackage, warnings []string, err error) { + if err := utils.ReadYaml(path, &pkg); err != nil { + return types.ZarfPackage{}, nil, fmt.Errorf("unable to read zarf.yaml file at %q", path) + } + + if pp.IsLegacyLayout() { + warnings = append(warnings, "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0") + } + + if len(pkg.Build.Migrations) > 0 { + var componentWarnings []string + for idx, component := range pkg.Components { + // Handle component configuration deprecations + pkg.Components[idx], componentWarnings = deprecated.MigrateComponent(pkg.Build, component) + warnings = append(warnings, componentWarnings...) + } + } + + return pkg, warnings, nil +} + // MigrateLegacy migrates a legacy package layout to the new layout. func (pp *PackagePaths) MigrateLegacy() (err error) { var pkg types.ZarfPackage diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 3b44201281..01aaf05565 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -59,8 +59,9 @@ func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerCo // LoadPackageDefinition loads and configures a zarf.yaml file during package create. func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { - if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { - return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + pkg, warnings, err = dst.ReadZarfYAML(layout.ZarfYAML) + if err != nil { + return types.ZarfPackage{}, nil, err } if pkg.Metadata.Architecture == "" { diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 73efe366d4..97996e6db0 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -41,18 +41,21 @@ func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.Za // LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { - if err := utils.ReadYaml(layout.ZarfYAML, &pkg); err != nil { - return types.ZarfPackage{}, nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + pkg, warnings, err = dst.ReadZarfYAML(layout.ZarfYAML) + if err != nil { + return types.ZarfPackage{}, nil, err } pkg.Metadata.Architecture = "skeleton" // Compose components into a single zarf.yaml file - pkg, warnings, err = ComposeComponents(pkg, sc.createOpts.Flavor) + pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) if err != nil { return types.ZarfPackage{}, nil, err } + warnings = append(warnings, composeWarnings...) + pkg.Components, err = sc.processExtensions(pkg.Components, dst) if err != nil { return types.ZarfPackage{}, nil, err From 17f34be2bd9e9d434404b4cac26b21401bc8ac80 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:13:48 -0600 Subject: [PATCH 147/172] Use ReadZarfYAML on Deploy() and delete runMigrations() --- src/pkg/packager/deploy.go | 6 +++--- src/pkg/packager/migrate.go | 24 ------------------------ 2 files changed, 3 insertions(+), 27 deletions(-) delete mode 100644 src/pkg/packager/migrate.go diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index a3376bf8ed..252dc12a6b 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -46,12 +46,12 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - p.runMigrations() + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) if err := p.validateLastNonBreakingVersion(); err != nil { return err diff --git a/src/pkg/packager/migrate.go b/src/pkg/packager/migrate.go deleted file mode 100644 index f0e652a0a8..0000000000 --- a/src/pkg/packager/migrate.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - -func (p *Packager) runMigrations() { - var warnings []string - - if p.layout.IsLegacyLayout() { - warning := "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0" - p.warnings = append(p.warnings, warning) - } - - if len(p.cfg.Pkg.Build.Migrations) > 0 { - for idx, component := range p.cfg.Pkg.Components { - // Handle component configuration deprecations - p.cfg.Pkg.Components[idx], warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - p.warnings = append(p.warnings, warnings...) - } - } -} From a7a93a00c742982ca86f44cb356e16e85878b00a Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:18:07 -0600 Subject: [PATCH 148/172] Use ReadZarfYAML in Inspect() --- src/pkg/packager/inspect.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index b73f71c765..bfc9c1aac8 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -17,7 +17,8 @@ func (p *Packager) Inspect() (err error) { return err } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } From 4bd3aca540cd6238d631be521ed6fcfb13d4b2c7 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:20:10 -0600 Subject: [PATCH 149/172] Use ReadZarfYAML in Mirror() --- src/pkg/packager/mirror.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 30038b3dda..dab9d0c66c 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -10,7 +10,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -23,7 +22,8 @@ func (p *Packager) Mirror() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } From 1a322ffc3c7eacf52e75c329d3d4acc8efe51631 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:26:38 -0600 Subject: [PATCH 150/172] Use ReadZarfYAML in Publish() --- src/pkg/packager/publish.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6b89d45e08..3310243913 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -107,7 +107,9 @@ func (p *Packager) Publish() (err error) { if err := p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } From 4a459ff36b51c9e6f0278599e7f5465f9ce5edd9 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:41:46 -0600 Subject: [PATCH 151/172] Use ReadZarfYAML in Remove() --- src/pkg/packager/remove.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index da2d2dcaab..25e1b27095 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -17,7 +17,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/actions" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "helm.sh/helm/v3/pkg/storage/driver" @@ -40,9 +39,12 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err := utils.ReadYaml(p.layout.ZarfYAML, &p.cfg.Pkg); err != nil { + + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } + p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) p.filterComponents() From eccb085a9fc9be3d7e97127e39532fda076af62e Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 29 Feb 2024 17:47:10 -0600 Subject: [PATCH 152/172] Fix ReadZarfYAML error --- src/pkg/layout/package.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index e9c9bde5dd..cca7e9afb6 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -60,7 +60,7 @@ func New(baseDir string) *PackagePaths { // checks if it's using the legacy layout, and migrates deprecated component configs. func (pp *PackagePaths) ReadZarfYAML(path string) (pkg types.ZarfPackage, warnings []string, err error) { if err := utils.ReadYaml(path, &pkg); err != nil { - return types.ZarfPackage{}, nil, fmt.Errorf("unable to read zarf.yaml file at %q", path) + return types.ZarfPackage{}, nil, fmt.Errorf("unable to read zarf.yaml file") } if pp.IsLegacyLayout() { From 09d9960ebbefe24979d473a214f12eff180c9eed Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 1 Mar 2024 12:37:24 -0600 Subject: [PATCH 153/172] Remove p.arch to reduce passing around arch across codebase --- src/pkg/packager/common.go | 7 +++---- src/pkg/packager/common_test.go | 2 +- src/pkg/packager/components.go | 4 ++-- src/pkg/packager/deploy.go | 2 -- src/pkg/packager/mirror.go | 4 +++- src/pkg/packager/remove.go | 2 -- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index a81f178c3e..11101ecec2 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -32,7 +32,6 @@ type Packager struct { cfg *types.PackagerConfig cluster *cluster.Cluster layout *layout.PackagePaths - arch string warnings []string valueTemplate *template.Values hpaModified bool @@ -219,7 +218,7 @@ func (p *Packager) attemptClusterChecks() (err error) { // validatePackageArchitecture validates that the package architecture matches the target cluster architecture. func (p *Packager) validatePackageArchitecture() error { // Ignore this check if the architecture is explicitly "multi", we don't have a cluster connection, or the package contains no images - if p.arch == "multi" || !p.isConnectedToCluster() || !p.hasImages() { + if p.cfg.Pkg.Metadata.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { return nil } @@ -229,8 +228,8 @@ func (p *Packager) validatePackageArchitecture() error { } // Check if the package architecture and the cluster architecture are the same. - if !slices.Contains(clusterArchitectures, p.arch) { - return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.arch, strings.Join(clusterArchitectures, ", ")) + if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Metadata.Architecture) { + return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Metadata.Architecture, strings.Join(clusterArchitectures, ", ")) } return nil diff --git a/src/pkg/packager/common_test.go b/src/pkg/packager/common_test.go index 105daf0242..1f497415be 100644 --- a/src/pkg/packager/common_test.go +++ b/src/pkg/packager/common_test.go @@ -85,7 +85,6 @@ func TestValidatePackageArchitecture(t *testing.T) { // Create a Packager instance with package architecture set and a mock Kubernetes client. p := &Packager{ - arch: testCase.pkgArch, cluster: &cluster.Cluster{ K8s: &k8s.K8s{ Clientset: mockClient, @@ -94,6 +93,7 @@ func TestValidatePackageArchitecture(t *testing.T) { }, cfg: &types.PackagerConfig{ Pkg: types.ZarfPackage{ + Metadata: types.ZarfMetadata{Architecture: testCase.pkgArch}, Components: []types.ZarfComponent{ { Images: testCase.images, diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index c9faccf869..84e0b59d04 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -34,10 +34,10 @@ func (p *Packager) filterComponents() { var validArch, validOS bool // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.arch { + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Metadata.Architecture { validArch = true } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.arch) + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Metadata.Architecture) } // Test for a valid OS diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 252dc12a6b..f21d6e1351 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -51,8 +51,6 @@ func (p *Packager) Deploy() (err error) { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - if err := p.validateLastNonBreakingVersion(); err != nil { return err } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index dab9d0c66c..6dabec8a7f 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -27,11 +27,13 @@ func (p *Packager) Mirror() (err error) { return err } - p.warnings, err = p.layout.SBOMs.StageSBOMViewFiles() + sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() if err != nil { return err } + p.warnings = append(p.warnings, sbomWarnings...) + // Confirm the overall package mirror if !p.confirmAction(config.ZarfMirrorStage) { return fmt.Errorf("mirror cancelled") diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 25e1b27095..2f7bf05e01 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -45,8 +45,6 @@ func (p *Packager) Remove() (err error) { return err } - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name From 1ddf9a2d3908c26dcebd45337a91272b8987575a Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Fri, 1 Mar 2024 12:48:31 -0600 Subject: [PATCH 154/172] pc to sc for SkeletonCreator receiver --- src/pkg/packager/creator/skeleton.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 97996e6db0..9f7271b7e8 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -123,7 +123,7 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPacka return nil } -func (pc *SkeletonCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths) (processedComponents []types.ZarfComponent, err error) { +func (sc *SkeletonCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths) (processedComponents []types.ZarfComponent, err error) { // Create component paths and process extensions for each component. for _, c := range components { componentPaths, err := layout.Components.Create(c) From 4c175d30fc55c01a05b0d2f74d7be7fe875e6e06 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Mon, 4 Mar 2024 09:51:18 -0600 Subject: [PATCH 155/172] Record build migrations in recordPackageMetadata() --- src/pkg/packager/creator/normal.go | 15 +++------------ src/pkg/packager/creator/skeleton.go | 12 ++++++------ src/pkg/packager/creator/utils.go | 14 ++++++++++---- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 01aaf05565..07892bb888 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -25,7 +25,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/actions" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -230,7 +229,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. // // - writes the Zarf package as a tarball to a local directory, // or an OCI registry based on the --output flag -func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error { +func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) (err error) { // Process the component directories into compressed tarballs // NOTE: This is purposefully being done after the SBOM cataloging for _, component := range pkg.Components { @@ -241,23 +240,15 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackag } // Calculate all the checksums - checksumChecksum, err := dst.GenerateChecksums() + pkg.Metadata.AggregateChecksum, err = dst.GenerateChecksums() if err != nil { return fmt.Errorf("unable to generate checksums for the package: %w", err) } - pkg.Metadata.AggregateChecksum = checksumChecksum - // Record the migrations that will be ran on the package. - pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - if err := setPackageMetadata(pkg, pc.createOpts); err != nil { + if err := recordPackageMetadata(pkg, pc.createOpts); err != nil { return err } - // Save the transformed config. if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 9f7271b7e8..45041ba0a8 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -92,25 +92,25 @@ func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, components []types // - writes the loaded zarf.yaml to disk // // - signs the package -func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error { +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) (err error) { for _, component := range pkg.Components { if err := dst.Components.Archive(component, false); err != nil { return err } } - checksumChecksum, err := dst.GenerateChecksums() + // Calculate all the checksums + pkg.Metadata.AggregateChecksum, err = dst.GenerateChecksums() if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + return fmt.Errorf("unable to generate checksums for the package: %w", err) } - pkg.Metadata.AggregateChecksum = checksumChecksum - if err := setPackageMetadata(pkg, sc.createOpts); err != nil { + if err := recordPackageMetadata(pkg, sc.createOpts); err != nil { return err } if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { - return err + return fmt.Errorf("unable to write zarf.yaml: %w", err) } // Sign the package if a key has been provided diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 114812fd58..a1e4f3e64e 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -14,8 +14,8 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// setPackageMetadata sets various package metadata. -func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) error { +// recordPackageMetadata records various package metadata during package create. +func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) error { now := time.Now() // Just use $USER env variable to avoid CGO issue. // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. @@ -31,6 +31,9 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptio return err } + // Record the hostname of the package creation terminal. + pkg.Build.Terminal = hostname + if pkg.IsInitConfig() { pkg.Metadata.Version = config.CLIVersion } @@ -43,8 +46,11 @@ func setPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptio // Record the Zarf Version the CLI was built with. pkg.Build.Version = config.CLIVersion - // Record the hostname of the package creation terminal. - pkg.Build.Terminal = hostname + // Record the migrations that will be ran on the package. + pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } // Record the flavor of Zarf used to build this package (if any). pkg.Build.Flavor = createOpts.Flavor From ec503e9e7e40d0543ab360af8836d0f1e2f51832 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 6 Mar 2024 09:33:59 -0600 Subject: [PATCH 156/172] Record pkg build version right after recording pkg metadata version for init pkg --- src/pkg/packager/creator/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index a1e4f3e64e..ad90f14d20 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -38,14 +38,14 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp pkg.Metadata.Version = config.CLIVersion } + // Record the Zarf Version the CLI was built with. + pkg.Build.Version = config.CLIVersion + pkg.Build.Architecture = pkg.Metadata.Architecture // Record the time of package creation. pkg.Build.Timestamp = now.Format(time.RFC1123Z) - // Record the Zarf Version the CLI was built with. - pkg.Build.Version = config.CLIVersion - // Record the migrations that will be ran on the package. pkg.Build.Migrations = []string{ deprecated.ScriptsToActionsMigrated, From 69472cf317e1d161b45a9ffd4b76e91e755e8a5c Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 7 Mar 2024 13:02:46 -0600 Subject: [PATCH 157/172] Add SetVariableMapInConfig back to FindImages --- src/pkg/packager/prepare.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 56531b7317..0fff732aa1 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -21,6 +21,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -80,6 +81,10 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { componentDefinition := "\ncomponents:\n" + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { + return nil, err + } + for _, component := range p.cfg.Pkg.Components { if len(component.Charts)+len(component.Manifests)+len(component.Repos) < 1 { From b2b944618d79601d2b7e1fe864f63aa0cec61675 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 7 Mar 2024 13:29:19 -0600 Subject: [PATCH 158/172] Fix pkg var in FindImages --- src/pkg/packager/prepare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 0fff732aa1..d25653f48f 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -85,7 +85,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { return nil, err } - for _, component := range p.cfg.Pkg.Components { + for _, component := range pkg.Components { if len(component.Charts)+len(component.Manifests)+len(component.Repos) < 1 { // Skip if it doesn't have what we need From 8f134677ca3dce63c2ed3b9a8b5e7a4016bdb904 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Thu, 7 Mar 2024 14:44:26 -0600 Subject: [PATCH 159/172] Remove shadown pkg var in FindImages and directly modify p.cfg.Pkg --- src/pkg/packager/creator/template.go | 1 - src/pkg/packager/prepare.go | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 192a9bc7f8..53b22ba206 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -40,7 +40,6 @@ func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) ( } setVariables[key] = setVal } else if !present { - // erroring out here return fmt.Errorf("template %q must be '--set' when using the '--confirm' flag", key) } } diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index d25653f48f..3437ea7794 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -41,7 +41,6 @@ type imageMap map[string]bool func (p *Packager) FindImages() (imgMap map[string][]string, err error) { repoHelmChartPath := p.cfg.FindImagesOpts.RepoHelmChartPath kubeVersionOverride := p.cfg.FindImagesOpts.KubeVersionOverride - pkg := p.cfg.Pkg whyImage := p.cfg.FindImagesOpts.Why imagesMap := make(map[string][]string) @@ -61,7 +60,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { c := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) - pkg, p.warnings, err = c.LoadPackageDefinition(p.layout) + p.cfg.Pkg, p.warnings, err = c.LoadPackageDefinition(p.layout) if err != nil { return nil, err } @@ -70,7 +69,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { message.Warn(warning) } - for _, component := range pkg.Components { + for _, component := range p.cfg.Pkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + "if any repos contain helm charts you want to template and " + @@ -85,7 +84,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { return nil, err } - for _, component := range pkg.Components { + for _, component := range p.cfg.Pkg.Components { if len(component.Charts)+len(component.Manifests)+len(component.Repos) < 1 { // Skip if it doesn't have what we need @@ -291,7 +290,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } if len(validImages) > 0 { - componentDefinition += fmt.Sprintf(" # Possible images - %s - %s\n", pkg.Metadata.Name, component.Name) + componentDefinition += fmt.Sprintf(" # Possible images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) for _, image := range validImages { imagesMap[component.Name] = append(imagesMap[component.Name], image) componentDefinition += fmt.Sprintf(" - %s\n", image) @@ -321,7 +320,7 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { if len(cosignArtifactList) > 0 { imagesMap[component.Name] = append(imagesMap[component.Name], cosignArtifactList...) - componentDefinition += fmt.Sprintf(" # Cosign artifacts for images - %s - %s\n", pkg.Metadata.Name, component.Name) + componentDefinition += fmt.Sprintf(" # Cosign artifacts for images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) for _, cosignArtifact := range cosignArtifactList { componentDefinition += fmt.Sprintf(" - %s\n", cosignArtifact) } From c41f5e917cb2c0d03aab847168883b65e97bb139 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 12:32:07 -0500 Subject: [PATCH 160/172] update to keep in sync Signed-off-by: razzle --- src/pkg/packager/create_stages.go | 782 ------------------------------ src/pkg/packager/publish.go | 2 +- 2 files changed, 1 insertion(+), 783 deletions(-) delete mode 100644 src/pkg/packager/create_stages.go diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go deleted file mode 100644 index a4c0aa7e58..0000000000 --- a/src/pkg/packager/create_stages.go +++ /dev/null @@ -1,782 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/pkg/zoci" - "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" -) - -func (p *Packager) cdToBaseDir(base string, cwd string) error { - if err := os.Chdir(base); err != nil { - return fmt.Errorf("unable to access directory %q: %w", base, err) - } - message.Note(fmt.Sprintf("Using build directory %s", base)) - - // differentials are relative to the current working directory - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} - -func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { - return err - } - - if p.cfg.CreateOpts.IsSkeleton { - return nil - } - - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { - return err - } - - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - - return nil -} - -func (p *Packager) assemble() error { - componentSBOMs := map[string]*layout.ComponentSBOM{} - var imageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - if err := p.addComponent(idx, component); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if !p.cfg.CreateOpts.SkipSBOM { - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - return fmt.Errorf("unable to create component SBOM: %w", err) - } - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - imageList = append(imageList, refInfo) - } - } - - imageList = helpers.Unique(imageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - var err error - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, p.cfg.PkgOpts.Retries, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after %d attempts: %w", p.cfg.PkgOpts.Retries, err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - return nil -} - -func (p *Packager) assembleSkeleton() error { - if err := p.skeletonizeExtensions(); err != nil { - return err - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range p.cfg.Pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} - -// output assumes it is running from cwd, not the build directory -func (p *Packager) output() error { - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range p.cfg.Pkg.Components { - // Make the component a tar archive - if err := p.layout.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Save the transformed config. - if err := p.writeYaml(); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // Sign the config file if a key was provided - if p.cfg.CreateOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - // Create a remote ref + client for the package (if output is OCI) - // then publish the package to the remote. - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := zoci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) - if err != nil { - return err - } - remote, err := zoci.NewRemote(ref, oci.PlatformForArch(config.GetArch())) - if err != nil { - return err - } - - ctx := context.TODO() - err = remote.PublishPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := p.archivePackage(packageName); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := p.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := p.layout.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = p.layout.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if p.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - return nil -} - -func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return nil, err - } - // Create an struct to hold the SBOM information for this component. - componentSBOM := &layout.ComponentSBOM{ - Files: []string{}, - Component: componentPaths, - } - - appendSBOMFiles := func(path string) error { - if utils.IsDir(path) { - files, err := utils.RecursiveFileList(path, nil, false) - if err != nil { - return err - } - componentSBOM.Files = append(componentSBOM.Files, files...) - } else { - info, err := os.Lstat(path) - if err != nil { - return err - } - if info.Mode().IsRegular() { - componentSBOM.Files = append(componentSBOM.Files, path) - } - } - - return nil - } - - for filesIdx, file := range component.Files { - path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - err := appendSBOMFiles(path) - if err != nil { - return nil, err - } - } - - for dataIdx, data := range component.DataInjections { - path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - - err := appendSBOMFiles(path) - if err != nil { - return nil, err - } - } - - return componentSBOM, nil -} - -func (p *Packager) addComponent(index int, component types.ZarfComponent) error { - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - - isSkeleton := p.cfg.CreateOpts.IsSkeleton - - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return err - } - - if isSkeleton && component.DeprecatedCosignKeyPath != "" { - dst := filepath.Join(componentPaths.Base, "cosign.pub") - err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) - if err != nil { - return err - } - p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" - } - - // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - if isSkeleton { - component.Actions.OnCreate.Defaults.Dir = "" - resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for idx := range actions { - actions[idx].Dir = nil - } - return actions - } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) - } - - onCreate := component.Actions.OnCreate - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { - return fmt.Errorf("unable to run component before action: %w", err) - } - } - - // If any helm charts are defined, process them. - for chartIdx, chart := range component.Charts { - - helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) - - if isSkeleton { - if chart.LocalPath != "" { - rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - err := utils.CreatePathAndCopy(chart.LocalPath, dst) - if err != nil { - return err - } - - p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel - } - - for valuesIdx, path := range chart.ValuesFiles { - if helpers.IsURL(path) { - continue - } - - rel := helm.StandardValuesName(layout.ValuesDir, chart, valuesIdx) - p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel - - if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) - } - } - } else { - err := helmCfg.PackageChart(componentPaths.Charts) - if err != nil { - return err - } - } - } - - for filesIdx, file := range component.Files { - message.Debugf("Loading %#v", file) - - rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - dst := filepath.Join(componentPaths.Base, rel) - destinationDir := filepath.Dir(dst) - - if helpers.IsURL(file.Source) { - if isSkeleton { - continue - } - - if file.ExtractPath != "" { - - // get the compressedFileName from the source - compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) - if err != nil { - return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) - } - - compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) - - // If the file is an archive, download it to the componentPath.Temp - if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - - err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) - } - - } else { - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - } - - } else { - if file.ExtractPath != "" { - if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) - } - } - - } - - if file.ExtractPath != "" { - // Make sure dst reflects the actual file or directory. - updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) - if updatedExtractedFileOrDir != dst { - if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) - } - } - } - - if isSkeleton { - // Change the source to the new relative source directory (any remote files will have been skipped above) - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel - // Remove the extractPath from a skeleton since it will already extract it - p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" - } - - // Abort packaging on invalid shasum (if one is specified). - if file.Shasum != "" { - if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err - } - } - - if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) - } else { - _ = os.Chmod(dst, helpers.ReadWriteUser) - } - } - - if len(component.DataInjections) > 0 { - spinner := message.NewProgressSpinner("Loading data injections") - defer spinner.Stop() - - for dataIdx, data := range component.DataInjections { - spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) - - rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - dst := filepath.Join(componentPaths.Base, rel) - - if helpers.IsURL(data.Source) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) - } - if isSkeleton { - p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel - } - } - } - spinner.Success() - } - - if len(component.Manifests) > 0 { - // Get the proper count of total manifests to add. - manifestCount := 0 - - for _, manifest := range component.Manifests { - manifestCount += len(manifest.Files) - manifestCount += len(manifest.Kustomizations) - } - - spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) - defer spinner.Stop() - - // Iterate over all manifests. - for manifestIdx, manifest := range component.Manifests { - for fileIdx, path := range manifest.Files { - rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - // Copy manifests without any processing. - spinner.Updatef("Copying manifest %s", path) - if helpers.IsURL(path) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, path, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel - } - } - } - - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) - } - } - if isSkeleton { - // remove kustomizations - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil - } - } - spinner.Success() - } - - // Load all specified git repos. - if len(component.Repos) > 0 && !isSkeleton { - spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) - defer spinner.Stop() - - for _, url := range component.Repos { - // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { - return fmt.Errorf("unable to pull git repo %s: %w", url, err) - } - } - spinner.Success() - } - - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { - return fmt.Errorf("unable to run component after action: %w", err) - } - } - - return nil -} - -// generateChecksum walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksum returns a SHA256 checksum of the checksums.txt file. -func (p *Packager) generatePackageChecksums() (string, error) { - // Loop over the "loaded" files - var checksumsData = []string{} - for rel, abs := range p.layout.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) - } - slices.Sort(checksumsData) - - // Create the checksums file - checksumsFilePath := p.layout.Checksums - if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), helpers.ReadWriteUser); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := zoci.NewRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML(context.TODO()) - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, helpers.ReadWriteUser) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil -} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 669aefd743..c438b83456 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -92,7 +92,7 @@ func (p *Packager) Publish() (err error) { if p.cfg.CreateOpts.IsSkeleton { platform = zoci.PlatformForSkeleton() } else { - platform = oci.PlatformForArch(p.arch) + platform = oci.PlatformForArch(p.cfg.Pkg.Build.Architecture) } remote, err := zoci.NewRemote(ref, platform) if err != nil { From 1ad5e2608cb40c105db599299f093e525c9e742f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 14:24:47 -0500 Subject: [PATCH 161/172] use build instead of metadata Signed-off-by: razzle --- src/config/lang/english.go | 4 ++-- src/pkg/packager/common.go | 6 +++--- src/pkg/packager/common_test.go | 2 +- src/pkg/packager/components.go | 4 ++-- src/pkg/packager/composer/list.go | 3 +++ src/pkg/packager/create.go | 7 ++++++- src/pkg/packager/creator/compose.go | 9 ++++++++- src/pkg/packager/creator/normal.go | 4 +--- src/pkg/packager/creator/skeleton.go | 3 ++- src/pkg/packager/creator/template.go | 2 +- src/pkg/packager/creator/utils.go | 2 -- src/pkg/packager/dev.go | 5 +++-- 12 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 8c2b88b1aa..9b40aeff9c 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -630,8 +630,8 @@ const ( // Package create const ( - PkgCreateErrDifferentialSameVersion = "unable to create differential package. Please ensure the differential package version and reference package version are not the same. The package version must be incremented." - PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure both package versions are set." + PkgCreateErrDifferentialSameVersion = "unable to create differential package. Please ensure the differential package version and reference package version are not the same. The package version must be incremented" + PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure both package versions are set" ) // Package deploy diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 11101ecec2..a4347ff5cb 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -218,7 +218,7 @@ func (p *Packager) attemptClusterChecks() (err error) { // validatePackageArchitecture validates that the package architecture matches the target cluster architecture. func (p *Packager) validatePackageArchitecture() error { // Ignore this check if the architecture is explicitly "multi", we don't have a cluster connection, or the package contains no images - if p.cfg.Pkg.Metadata.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { + if p.cfg.Pkg.Build.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { return nil } @@ -228,8 +228,8 @@ func (p *Packager) validatePackageArchitecture() error { } // Check if the package architecture and the cluster architecture are the same. - if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Metadata.Architecture) { - return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Metadata.Architecture, strings.Join(clusterArchitectures, ", ")) + if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Build.Architecture) { + return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Build.Architecture, strings.Join(clusterArchitectures, ", ")) } return nil diff --git a/src/pkg/packager/common_test.go b/src/pkg/packager/common_test.go index 1f497415be..bf560c8673 100644 --- a/src/pkg/packager/common_test.go +++ b/src/pkg/packager/common_test.go @@ -93,7 +93,7 @@ func TestValidatePackageArchitecture(t *testing.T) { }, cfg: &types.PackagerConfig{ Pkg: types.ZarfPackage{ - Metadata: types.ZarfMetadata{Architecture: testCase.pkgArch}, + Build: types.ZarfBuildData{Architecture: testCase.pkgArch}, Components: []types.ZarfComponent{ { Images: testCase.images, diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 84e0b59d04..782528232c 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -34,10 +34,10 @@ func (p *Packager) filterComponents() { var validArch, validOS bool // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Metadata.Architecture { + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Build.Architecture { validArch = true } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Metadata.Architecture) + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Build.Architecture) } // Test for a valid OS diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index a037812101..ea23533d96 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -347,6 +347,9 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer // CompatibleComponent determines if this component is compatible with the given create options func CompatibleComponent(c types.ZarfComponent, arch, flavor string) bool { + if arch == zoci.PlatformForSkeleton().Architecture { + return true + } satisfiesArch := c.Only.Cluster.Architecture == "" || c.Only.Cluster.Architecture == arch satisfiesFlavor := c.Only.Flavor == "" || c.Only.Flavor == flavor return satisfiesArch && satisfiesFlavor diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 9b01c6fefe..d98b1840a3 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -24,6 +24,11 @@ func (p *Packager) Create() (err error) { if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } + defer func() { + if err != nil { + os.Chdir(cwd) + } + }() message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) @@ -43,7 +48,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Build.Architecture); err != nil { return err } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index 343e8f9bc8..a1827c74d2 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -5,6 +5,8 @@ package creator import ( + "fmt" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" @@ -18,8 +20,13 @@ func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, pkgVars := pkg.Variables pkgConsts := pkg.Constants + arch := pkg.Build.Architecture + + if arch == "" { + return types.ZarfPackage{}, nil, fmt.Errorf("build.architecture is required to compose components") + } + for i, component := range pkg.Components { - arch := pkg.Metadata.Architecture // filter by architecture and flavor if !composer.CompatibleComponent(component, arch, flavor) { continue diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 9df0a92835..0a6b2976b2 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -65,9 +65,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t return types.ZarfPackage{}, nil, err } - if pkg.Metadata.Architecture == "" { - pkg.Metadata.Architecture = config.GetArch() - } + pkg.Build.Architecture = config.GetArch(pkg.Metadata.Architecture) // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, pc.createOpts.Flavor) diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 45041ba0a8..6fca92fdc7 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -19,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ) @@ -46,7 +47,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg return types.ZarfPackage{}, nil, err } - pkg.Metadata.Architecture = "skeleton" + pkg.Build.Architecture = zoci.PlatformForSkeleton().Architecture // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 53b22ba206..5313fbb2b7 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -65,7 +65,7 @@ func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) ( } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = pkg.Metadata.Architecture + templateMap[types.ZarfPackageArch] = pkg.Build.Architecture if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { return types.ZarfPackage{}, nil, err diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index ad90f14d20..7e143d89ff 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -41,8 +41,6 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp // Record the Zarf Version the CLI was built with. pkg.Build.Version = config.CLIVersion - pkg.Build.Architecture = pkg.Metadata.Architecture - // Record the time of package creation. pkg.Build.Timestamp = now.Format(time.RFC1123Z) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 2b978ec479..22bba01fe0 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -29,6 +29,7 @@ func (p *Packager) DevDeploy() error { if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } + defer os.Chdir(cwd) pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) @@ -59,7 +60,7 @@ func (p *Packager) DevDeploy() error { } } - if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Build.Architecture); err != nil { return err } @@ -98,5 +99,5 @@ func (p *Packager) DevDeploy() error { message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) // cd back - return os.Chdir(cwd) + return nil } From de2e1d7a4086e9a82716f7d403a7695255696262 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 14:31:25 -0500 Subject: [PATCH 162/172] dry Signed-off-by: razzle --- src/pkg/packager/sources/oci.go | 2 +- src/pkg/packager/sources/utils.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 79968d5aeb..2e8a21377c 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -201,7 +201,7 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() // TODO (@Noxsios) remove the suffix check at v1.0.0 - isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) + isSkeleton := pkg.Build.Architecture == zoci.PlatformForSkeleton().Architecture || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) name := fmt.Sprintf("%s%s", NameFromMetadata(&pkg, isSkeleton), PkgSuffix(pkg.Metadata.Uncompressed)) dstTarball := filepath.Join(dir, name) diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 65c3506b10..b8f5b30712 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/mholt/archiver/v3" @@ -112,7 +113,7 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { arch := config.GetArch(pkg.Metadata.Architecture, pkg.Build.Architecture) if isSkeleton { - arch = "skeleton" + arch = zoci.PlatformForSkeleton().Architecture } switch pkg.Kind { From ba17472348be1d7afd7050a6e6c5d3d7a6efe86d Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 14:33:08 -0500 Subject: [PATCH 163/172] dry Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/creator/skeleton.go | 2 +- src/pkg/packager/sources/oci.go | 2 +- src/pkg/packager/sources/utils.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ea23533d96..9ab104b8b1 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -347,7 +347,7 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer // CompatibleComponent determines if this component is compatible with the given create options func CompatibleComponent(c types.ZarfComponent, arch, flavor string) bool { - if arch == zoci.PlatformForSkeleton().Architecture { + if arch == zoci.SkeletonArch { return true } satisfiesArch := c.Only.Cluster.Architecture == "" || c.Only.Cluster.Architecture == arch diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index 6fca92fdc7..c9cacc52d5 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -47,7 +47,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg return types.ZarfPackage{}, nil, err } - pkg.Build.Architecture = zoci.PlatformForSkeleton().Architecture + pkg.Build.Architecture = zoci.SkeletonArch // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 2e8a21377c..b9a0857c9d 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -201,7 +201,7 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() // TODO (@Noxsios) remove the suffix check at v1.0.0 - isSkeleton := pkg.Build.Architecture == zoci.PlatformForSkeleton().Architecture || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) + isSkeleton := pkg.Build.Architecture == zoci.SkeletonArch || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) name := fmt.Sprintf("%s%s", NameFromMetadata(&pkg, isSkeleton), PkgSuffix(pkg.Metadata.Uncompressed)) dstTarball := filepath.Join(dir, name) diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index b8f5b30712..1347ee4127 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -113,7 +113,7 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { arch := config.GetArch(pkg.Metadata.Architecture, pkg.Build.Architecture) if isSkeleton { - arch = zoci.PlatformForSkeleton().Architecture + arch = zoci.SkeletonArch } switch pkg.Kind { From 653b32876766d4486657604d1ae42991e0df14ec Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:02:42 -0500 Subject: [PATCH 164/172] going back on my word Signed-off-by: razzle --- src/pkg/packager/components.go | 4 ++-- src/pkg/packager/create.go | 2 +- src/pkg/packager/creator/compose.go | 8 +------- src/pkg/packager/creator/normal.go | 2 +- src/pkg/packager/creator/template.go | 2 +- src/pkg/packager/creator/utils.go | 9 ++++----- src/pkg/packager/dev.go | 2 +- 7 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 782528232c..84e0b59d04 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -34,10 +34,10 @@ func (p *Packager) filterComponents() { var validArch, validOS bool // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Build.Architecture { + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Metadata.Architecture { validArch = true } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Build.Architecture) + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Metadata.Architecture) } // Test for a valid OS diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index d98b1840a3..d5f3ea6742 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -48,7 +48,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Build.Architecture); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } diff --git a/src/pkg/packager/creator/compose.go b/src/pkg/packager/creator/compose.go index a1827c74d2..dd2e429731 100644 --- a/src/pkg/packager/creator/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -5,8 +5,6 @@ package creator import ( - "fmt" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" @@ -20,11 +18,7 @@ func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, pkgVars := pkg.Variables pkgConsts := pkg.Constants - arch := pkg.Build.Architecture - - if arch == "" { - return types.ZarfPackage{}, nil, fmt.Errorf("build.architecture is required to compose components") - } + arch := pkg.Metadata.Architecture for i, component := range pkg.Components { // filter by architecture and flavor diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 0a6b2976b2..159edd82b7 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -65,7 +65,7 @@ func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg t return types.ZarfPackage{}, nil, err } - pkg.Build.Architecture = config.GetArch(pkg.Metadata.Architecture) + pkg.Metadata.Architecture = config.GetArch(pkg.Metadata.Architecture) // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, pc.createOpts.Flavor) diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go index 5313fbb2b7..53b22ba206 100644 --- a/src/pkg/packager/creator/template.go +++ b/src/pkg/packager/creator/template.go @@ -65,7 +65,7 @@ func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) ( } // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = pkg.Build.Architecture + templateMap[types.ZarfPackageArch] = pkg.Metadata.Architecture if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { return types.ZarfPackage{}, nil, err diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go index 7e143d89ff..43ab469565 100644 --- a/src/pkg/packager/creator/utils.go +++ b/src/pkg/packager/creator/utils.go @@ -26,18 +26,17 @@ func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOp pkg.Build.User = os.Getenv("USER") } - hostname, err := os.Hostname() - if err != nil { - return err - } - // Record the hostname of the package creation terminal. + // The error here is ignored because the hostname is not critical to the package creation. + hostname, _ := os.Hostname() pkg.Build.Terminal = hostname if pkg.IsInitConfig() { pkg.Metadata.Version = config.CLIVersion } + pkg.Build.Architecture = pkg.Metadata.Architecture + // Record the Zarf Version the CLI was built with. pkg.Build.Version = config.CLIVersion diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 22bba01fe0..95686e3476 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -60,7 +60,7 @@ func (p *Packager) DevDeploy() error { } } - if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Build.Architecture); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } From 38b315009bbf38491409d3aed9dcbe4a3deac44c Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:09:30 -0500 Subject: [PATCH 165/172] going back on my word Signed-off-by: razzle --- src/pkg/packager/common.go | 6 +++--- src/pkg/packager/creator/skeleton.go | 2 +- src/pkg/packager/deploy.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index a4347ff5cb..11101ecec2 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -218,7 +218,7 @@ func (p *Packager) attemptClusterChecks() (err error) { // validatePackageArchitecture validates that the package architecture matches the target cluster architecture. func (p *Packager) validatePackageArchitecture() error { // Ignore this check if the architecture is explicitly "multi", we don't have a cluster connection, or the package contains no images - if p.cfg.Pkg.Build.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { + if p.cfg.Pkg.Metadata.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { return nil } @@ -228,8 +228,8 @@ func (p *Packager) validatePackageArchitecture() error { } // Check if the package architecture and the cluster architecture are the same. - if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Build.Architecture) { - return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Build.Architecture, strings.Join(clusterArchitectures, ", ")) + if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Metadata.Architecture) { + return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Metadata.Architecture, strings.Join(clusterArchitectures, ", ")) } return nil diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index c9cacc52d5..d40f36860e 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -47,7 +47,7 @@ func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg return types.ZarfPackage{}, nil, err } - pkg.Build.Architecture = zoci.SkeletonArch + pkg.Metadata.Architecture = zoci.SkeletonArch // Compose components into a single zarf.yaml file pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 98abf20f9b..a0f94ecb5f 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -471,7 +471,7 @@ func (p *Packager) pushImagesToRegistry(componentImages []string, noImgChecksum NoChecksum: noImgChecksum, RegInfo: p.cfg.State.RegistryInfo, Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + Architectures: []string{p.cfg.Pkg.Metadata.Architecture}, } return helpers.Retry(func() error { From 42f3d963223944b68c524bdda6cc9d86e1043cda Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:26:07 -0500 Subject: [PATCH 166/172] going back on my word Signed-off-by: razzle --- src/pkg/packager/create.go | 5 ----- src/pkg/packager/dev.go | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index d5f3ea6742..9b01c6fefe 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -24,11 +24,6 @@ func (p *Packager) Create() (err error) { if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - defer func() { - if err != nil { - os.Chdir(cwd) - } - }() message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 95686e3476..2b978ec479 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -29,7 +29,6 @@ func (p *Packager) DevDeploy() error { if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - defer os.Chdir(cwd) pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) @@ -99,5 +98,5 @@ func (p *Packager) DevDeploy() error { message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) // cd back - return nil + return os.Chdir(cwd) } From 9bd4c04601e8bcffa17bdb7e17fc960266c7a1b6 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:36:22 -0500 Subject: [PATCH 167/172] file perms Signed-off-by: razzle --- src/cmd/dev.go | 2 +- src/pkg/packager/creator/normal.go | 6 +++--- src/pkg/packager/creator/skeleton.go | 6 +++--- src/pkg/utils/helpers/io.go | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 8eeb312eac..808db053b5 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -95,7 +95,7 @@ var devTransformGitLinksCmd = &cobra.Command{ if confirm { // Overwrite the file - err = os.WriteFile(fileName, []byte(processedText), 0640) + err = os.WriteFile(fileName, []byte(processedText), helpers.ReadAllWriteUser) if err != nil { message.Fatal(err, lang.CmdDevPatchGitFileWriteErr) } diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 159edd82b7..36a86940ae 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -249,7 +249,7 @@ func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackag return err } - if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pkg, helpers.ReadUser); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } @@ -427,9 +427,9 @@ func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layou } if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) + _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) } else { - _ = os.Chmod(dst, 0600) + _ = os.Chmod(dst, helpers.ReadWriteUser) } } diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go index d40f36860e..d23081f1a7 100644 --- a/src/pkg/packager/creator/skeleton.go +++ b/src/pkg/packager/creator/skeleton.go @@ -110,7 +110,7 @@ func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPacka return err } - if err := utils.WriteYaml(dst.ZarfYAML, pkg, 0400); err != nil { + if err := utils.WriteYaml(dst.ZarfYAML, pkg, helpers.ReadUser); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) } @@ -251,9 +251,9 @@ func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layo } if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) + _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) } else { - _ = os.Chmod(dst, 0600) + _ = os.Chmod(dst, helpers.ReadWriteUser) } } diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go index e0b1879f86..996d643e52 100644 --- a/src/pkg/utils/helpers/io.go +++ b/src/pkg/utils/helpers/io.go @@ -5,11 +5,12 @@ package helpers const ( + // ReadUser is used for any internal file to be read only + ReadUser = 0400 // ReadWriteUser is used for any internal file not normally used by the end user or containing sensitive data ReadWriteUser = 0600 // ReadAllWriteUser is used for any non sensitive file intended to be consumed by the end user ReadAllWriteUser = 0644 - // ReadWriteExecuteUser is used for any directory or executable not normally used by the end user or containing sensitive data ReadWriteExecuteUser = 0700 // ReadExecuteAllWriteUser is used for any non sensitive directory or executable intended to be consumed by the end user From 8b257311b94e2efbe9d75e2bc4125b4b7b7ab7ee Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:47:50 -0500 Subject: [PATCH 168/172] put cleanup back in Signed-off-by: razzle --- src/test/e2e/03_deprecations_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/e2e/03_deprecations_test.go b/src/test/e2e/03_deprecations_test.go index f8db32e6a2..9d1c96b0c9 100644 --- a/src/test/e2e/03_deprecations_test.go +++ b/src/test/e2e/03_deprecations_test.go @@ -24,6 +24,7 @@ func TestDeprecatedComponentScripts(t *testing.T) { "test-deprecated-deploy-after-hook.txt", } allArtifacts := append(deployArtifacts, prepareArtifact) + e2e.CleanFiles(allArtifacts...) defer e2e.CleanFiles(allArtifacts...) // 1. Try creating the package to test the create scripts From 8748b0c85a80244b1599bd0332ab284fca59e172 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 12 Mar 2024 16:53:54 -0500 Subject: [PATCH 169/172] fix what i broke Signed-off-by: razzle --- src/pkg/packager/common_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/common_test.go b/src/pkg/packager/common_test.go index bf560c8673..1f497415be 100644 --- a/src/pkg/packager/common_test.go +++ b/src/pkg/packager/common_test.go @@ -93,7 +93,7 @@ func TestValidatePackageArchitecture(t *testing.T) { }, cfg: &types.PackagerConfig{ Pkg: types.ZarfPackage{ - Build: types.ZarfBuildData{Architecture: testCase.pkgArch}, + Metadata: types.ZarfMetadata{Architecture: testCase.pkgArch}, Components: []types.ZarfComponent{ { Images: testCase.images, From 6fa3dd75a37000e12869496f9c1db8f45ef634eb Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 13 Mar 2024 09:37:03 -0500 Subject: [PATCH 170/172] remove p.arch Signed-off-by: razzle --- src/pkg/packager/generate.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index a82fcdeca3..286807d186 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -62,7 +62,6 @@ func (p *Packager) Generate() (err error) { generatedComponent, }, } - p.arch = config.GetArch() images, err := p.findImages() if err != nil { From bbadddf5c31fed9295bfc76a3bd44593d87982df Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 13 Mar 2024 09:39:45 -0500 Subject: [PATCH 171/172] keep warnings where they belong Signed-off-by: razzle --- src/pkg/packager/prepare.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 3a5a726d45..c4cfc95d3c 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -61,6 +61,10 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { return nil, err } + for _, warning := range p.warnings { + message.Warn(warning) + } + return p.findImages() } @@ -74,10 +78,6 @@ func (p *Packager) findImages() (imgMap map[string][]string, err error) { erroredCosignLookups := []string{} whyResources := []string{} - for _, warning := range p.warnings { - message.Warn(warning) - } - for _, component := range p.cfg.Pkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + From a5d356632d8cfe700a173d4ae1a0210ba6f60e83 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 13 Mar 2024 10:33:54 -0500 Subject: [PATCH 172/172] Update src/pkg/packager/deploy.go Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/pkg/packager/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index a0f94ecb5f..7a1cece304 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -471,7 +471,7 @@ func (p *Packager) pushImagesToRegistry(componentImages []string, noImgChecksum NoChecksum: noImgChecksum, RegInfo: p.cfg.State.RegistryInfo, Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture}, + Architectures: []string{p.cfg.Pkg.Build.Architecture}, } return helpers.Retry(func() error {