From e7853efddca897640da8983ff5fcd9898ee9dd41 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Wed, 5 Jul 2023 18:09:36 -0400 Subject: [PATCH 1/3] [TASK] refactor for smaller code libs for zarf agent packages Signed-off-by: Case Wylie --- src/cmd/initialize.go | 5 +-- src/cmd/internal.go | 3 +- src/cmd/package.go | 11 +++--- src/cmd/prepare.go | 7 ++-- src/cmd/tools/common.go | 4 +-- src/extensions/bigbang/bigbang.go | 3 +- src/extensions/bigbang/flux.go | 3 +- src/internal/cluster/tunnel.go | 3 +- src/internal/packager/helm/images.go | 4 +-- src/pkg/oci/manifest.go | 4 +-- src/pkg/oci/pull.go | 9 ++--- src/pkg/packager/actions.go | 3 +- src/pkg/packager/checksum.go | 5 +-- src/pkg/packager/common.go | 2 +- src/pkg/packager/create.go | 7 ++-- src/pkg/packager/deploy.go | 7 ++-- src/pkg/packager/deprecated/common.go | 6 ++-- src/pkg/packager/remove.go | 15 +++++---- src/pkg/packager/variables.go | 5 +-- src/pkg/transform/artifact.go | 4 +-- src/pkg/transform/git.go | 12 +++---- src/pkg/transform/image.go | 4 +-- src/pkg/utils/auth.go | 2 +- src/pkg/utils/auth_test.go | 2 +- src/pkg/utils/bytes.go | 2 +- src/pkg/utils/credentials.go | 2 +- src/pkg/utils/hash.go | 48 --------------------------- src/pkg/utils/helpers/hash.go | 28 ++++++++++++++++ src/pkg/utils/{ => helpers}/misc.go | 4 +-- src/pkg/utils/htpasswd.go | 2 +- src/pkg/utils/image.go | 2 +- src/pkg/utils/io.go | 22 ++++++++++++ src/pkg/utils/json.go | 2 +- src/pkg/utils/network.go | 2 +- src/pkg/utils/network_test.go | 2 +- src/pkg/utils/random.go | 2 +- src/pkg/utils/sget.go | 2 +- src/pkg/utils/sort.go | 2 +- src/pkg/utils/sort_test.go | 2 +- src/pkg/utils/transport.go | 2 +- src/pkg/utils/yaml.go | 2 +- src/test/common.go | 4 +-- 42 files changed, 139 insertions(+), 123 deletions(-) delete mode 100644 src/pkg/utils/hash.go create mode 100644 src/pkg/utils/helpers/hash.go rename src/pkg/utils/{ => helpers}/misc.go (97%) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index f20dac3079..bf6bbf055d 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" ) @@ -50,8 +51,8 @@ var initCmd = &cobra.Command{ pkgConfig.PkgSourcePath = pkgConfig.DeployOpts.PackagePath // Ensure uppercase keys from viper - viperConfig := utils.TransformMapKeys(v.GetStringMapString(V_PKG_DEPLOY_SET), strings.ToUpper) - pkgConfig.DeployOpts.SetVariables = utils.MergeMap(viperConfig, pkgConfig.DeployOpts.SetVariables) + viperConfig := helpers.TransformMapKeys(v.GetStringMapString(V_PKG_DEPLOY_SET), strings.ToUpper) + pkgConfig.DeployOpts.SetVariables = helpers.MergeMap(viperConfig, pkgConfig.DeployOpts.SetVariables) // Configure the packager pkgClient := packager.NewOrDie(&pkgConfig) diff --git a/src/cmd/internal.go b/src/cmd/internal.go index f333967a77..2ad2605dfe 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" @@ -185,7 +186,7 @@ var computeCrc32 = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { text := args[0] - hash := utils.GetCRCHash(text) + hash := helpers.GetCRCHash(text) fmt.Printf("%d\n", hash) }, } diff --git a/src/cmd/package.go b/src/cmd/package.go index bd4ec2ed28..9976fef180 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/pterm/pterm" "oras.land/oras-go/v2/registry" @@ -55,8 +56,8 @@ var packageCreateCmd = &cobra.Command{ } // Ensure uppercase keys from viper - viperConfig := utils.TransformMapKeys(v.GetStringMapString(V_PKG_CREATE_SET), strings.ToUpper) - pkgConfig.CreateOpts.SetVariables = utils.MergeMap(viperConfig, pkgConfig.CreateOpts.SetVariables) + viperConfig := helpers.TransformMapKeys(v.GetStringMapString(V_PKG_CREATE_SET), strings.ToUpper) + pkgConfig.CreateOpts.SetVariables = helpers.MergeMap(viperConfig, pkgConfig.CreateOpts.SetVariables) // Configure the packager pkgClient := packager.NewOrDie(&pkgConfig) @@ -79,11 +80,11 @@ var packageDeployCmd = &cobra.Command{ pkgConfig.DeployOpts.PackagePath = choosePackage(args) // Ensure uppercase keys from viper and CLI --set - viperConfigSetVariables := utils.TransformMapKeys(v.GetStringMapString(V_PKG_DEPLOY_SET), strings.ToUpper) - pkgConfig.DeployOpts.SetVariables = utils.TransformMapKeys(pkgConfig.DeployOpts.SetVariables, strings.ToUpper) + viperConfigSetVariables := helpers.TransformMapKeys(v.GetStringMapString(V_PKG_DEPLOY_SET), strings.ToUpper) + pkgConfig.DeployOpts.SetVariables = helpers.TransformMapKeys(pkgConfig.DeployOpts.SetVariables, strings.ToUpper) // Merge the viper config file variables and provided CLI flag variables (CLI takes precedence)) - pkgConfig.DeployOpts.SetVariables = utils.MergeMap(viperConfigSetVariables, pkgConfig.DeployOpts.SetVariables) + pkgConfig.DeployOpts.SetVariables = helpers.MergeMap(viperConfigSetVariables, pkgConfig.DeployOpts.SetVariables) pkgConfig.PkgSourcePath = pkgConfig.DeployOpts.PackagePath diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 95cf322b34..6bcbce35b5 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" ) @@ -77,7 +78,7 @@ var prepareComputeFileSha256sum = &cobra.Command{ Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { fileName := args[0] - hash, err := utils.GetCryptoHash(fileName, crypto.SHA256) + hash, err := utils.GetCryptoHashFromFile(fileName, crypto.SHA256) if err != nil { message.Fatal(err, lang.CmdPrepareSha256sumHashErr) } else { @@ -101,8 +102,8 @@ var prepareFindImages = &cobra.Command{ } // Ensure uppercase keys from viper - viperConfig := utils.TransformMapKeys(v.GetStringMapString(V_PKG_CREATE_SET), strings.ToUpper) - pkgConfig.CreateOpts.SetVariables = utils.MergeMap(viperConfig, pkgConfig.CreateOpts.SetVariables) + viperConfig := helpers.TransformMapKeys(v.GetStringMapString(V_PKG_CREATE_SET), strings.ToUpper) + pkgConfig.CreateOpts.SetVariables = helpers.MergeMap(viperConfig, pkgConfig.CreateOpts.SetVariables) // Configure the packager pkgClient := packager.NewOrDie(&pkgConfig) diff --git a/src/cmd/tools/common.go b/src/cmd/tools/common.go index 5af735090a..9ce2bf54d8 100644 --- a/src/cmd/tools/common.go +++ b/src/cmd/tools/common.go @@ -10,7 +10,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/spf13/cobra" ) @@ -61,7 +61,7 @@ func CheckVendorOnlyFromPath(cmd *cobra.Command) bool { func isVendorCmd(args []string, vendoredCmds []string) bool { if len(args) > 2 { if args[1] == "tools" || args[1] == "t" { - if utils.SliceContains(vendoredCmds, args[2]) { + if helpers.SliceContains(vendoredCmds, args[2]) { return true } } diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index bcd0f5e839..a0715fbb8c 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/defenseunicorns/zarf/src/types/extensions" fluxHelmCtrl "github.com/fluxcd/helm-controller/api/v2beta1" @@ -231,7 +232,7 @@ func Run(YOLO bool, tmpPaths types.ComponentPaths, c types.ZarfComponent) (types } // Make sure the list of images is unique. - c.Images = utils.Unique(c.Images) + c.Images = helpers.Unique(c.Images) } // Create the flux wrapper around Big Bang for deployment. diff --git a/src/extensions/bigbang/flux.go b/src/extensions/bigbang/flux.go index e2b8b1f5cc..b527a67c97 100644 --- a/src/extensions/bigbang/flux.go +++ b/src/extensions/bigbang/flux.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/defenseunicorns/zarf/src/types/extensions" fluxHelmCtrl "github.com/fluxcd/helm-controller/api/v2beta1" @@ -149,7 +150,7 @@ func composeValues(hr HelmReleaseDependency, secrets map[string]corev1.Secret, c return nil, fmt.Errorf("unable to read values from key '%s' in %s '%s': %w", v.GetValuesKey(), v.Kind, hr.Name(), err) } - valuesMap = utils.MergeMapRecursive(valuesMap, values) + valuesMap = helpers.MergeMapRecursive(valuesMap, values) } return valuesMap, nil diff --git a/src/internal/cluster/tunnel.go b/src/internal/cluster/tunnel.go index 081c772f73..90d01c9a71 100644 --- a/src/internal/cluster/tunnel.go +++ b/src/internal/cluster/tunnel.go @@ -28,6 +28,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/transport/spdy" @@ -167,7 +168,7 @@ func ServiceInfoFromServiceURL(serviceURL string) (*ServiceInfo, error) { // Match hostname against local cluster service format. pattern := regexp.MustCompile(serviceURLPattern) - get, err := utils.MatchRegex(pattern, parsedURL.Hostname()) + get, err := helpers.MatchRegex(pattern, parsedURL.Hostname()) // If incomplete match, return an error. if err != nil { diff --git a/src/internal/packager/helm/images.go b/src/internal/packager/helm/images.go index db60b17cfa..9991582c94 100644 --- a/src/internal/packager/helm/images.go +++ b/src/internal/packager/helm/images.go @@ -2,7 +2,7 @@ package helm import ( "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/goccy/go-yaml" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" @@ -27,7 +27,7 @@ func FindAnnotatedImagesForChart(chartPath string, values chartutil.Values) (ima if err != nil { return images, err } - values = utils.MergeMapRecursive(chart.Values, values) + values = helpers.MergeMapRecursive(chart.Values, values) imageAnnotation := chart.Metadata.Annotations["helm.sh/images"] diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go index 803bf15d34..707221e8a3 100644 --- a/src/pkg/oci/manifest.go +++ b/src/pkg/oci/manifest.go @@ -7,7 +7,7 @@ package oci import ( "path/filepath" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -34,7 +34,7 @@ func NewZarfOCIManifest(manifest *ocispec.Manifest) *ZarfOCIManifest { // Locate returns the descriptor for the layer with the given path. func (m *ZarfOCIManifest) Locate(path string) ocispec.Descriptor { - return utils.Find(m.Layers, func(layer ocispec.Descriptor) bool { + return helpers.Find(m.Layers, func(layer ocispec.Descriptor) bool { return layer.Annotations[ocispec.AnnotationTitle] == path }) } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 5f4e2c1f80..42dc71b568 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pterm/pterm" @@ -60,7 +61,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) images := map[string]bool{} tarballFormat := "%s.tar" for _, name := range requestedComponents { - component := utils.Find(pkg.Components, func(component types.ZarfComponent) bool { + component := helpers.Find(pkg.Components, func(component types.ZarfComponent) bool { return component.Name == name }) if component.Name == "" { @@ -69,7 +70,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) } for _, component := range pkg.Components { // If we requested this component, or it is required, we need to pull its images and tarball - if utils.SliceContains(requestedComponents, component.Name) || component.Required { + if helpers.SliceContains(requestedComponents, component.Name) || component.Required { for _, image := range component.Images { images[image] = true } @@ -91,7 +92,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) return nil, err } for image := range images { - manifestDescriptor := utils.Find(index.Manifests, func(layer ocispec.Descriptor) bool { + manifestDescriptor := helpers.Find(index.Manifests, func(layer ocispec.Descriptor) bool { return layer.Annotations[ocispec.AnnotationBaseImageName] == image }) manifest, err := o.FetchManifest(manifestDescriptor) @@ -167,7 +168,7 @@ func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersT } var ret []ocispec.Descriptor for _, node := range nodes { - if utils.SliceContains(paths, node.Annotations[ocispec.AnnotationTitle]) { + if helpers.SliceContains(paths, node.Annotations[ocispec.AnnotationTitle]) { ret = append(ret, node) } } diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions.go index 00a469986b..e3566bbdcc 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -212,7 +213,7 @@ func actionCmdMutation(cmd string, shellPref types.ZarfComponentActionShell) (st // Convert any ${ZARF_VAR_*} or $ZARF_VAR_* to ${env:ZARF_VAR_*} or $env:ZARF_VAR_* respectively (also TF_VAR_*). // https://regex101.com/r/xk1rkw/1 envVarRegex := regexp.MustCompile(`(?P\${?(?P(ZARF|TF)_VAR_([a-zA-Z0-9_-])+)}?)`) - get, err := utils.MatchRegex(envVarRegex, cmd) + get, err := helpers.MatchRegex(envVarRegex, cmd) if err == nil { newCmd := strings.ReplaceAll(cmd, get("envIndicator"), fmt.Sprintf("$Env:%s", get("varName"))) message.Debugf("Converted command \"%s\" to \"%s\" t", cmd, newCmd) diff --git a/src/pkg/packager/checksum.go b/src/pkg/packager/checksum.go index 01d75e9e94..e47c51f74f 100644 --- a/src/pkg/packager/checksum.go +++ b/src/pkg/packager/checksum.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) // validatePackageChecksums validates the checksums of a Zarf package. @@ -37,7 +38,7 @@ func (p *Packager) validatePackageChecksums(baseDir string, aggregateChecksum st isPartial := false if len(pathsToCheck) > 0 { - pathsToCheck = utils.Unique(pathsToCheck) + pathsToCheck = helpers.Unique(pathsToCheck) isPartial = true message.Debugf("Validating checksums for a subset of files in the package - %v", pathsToCheck) for idx, path := range pathsToCheck { @@ -78,7 +79,7 @@ func (p *Packager) validatePackageChecksums(baseDir string, aggregateChecksum st if utils.InvalidPath(path) { if !isPartial && !checkedMap[path] { return fmt.Errorf("unable to validate checksums - missing file: %s", rel) - } else if utils.SliceContains(pathsToCheck, path) { + } else if helpers.SliceContains(pathsToCheck, path) { return fmt.Errorf("unable to validate partial checksums - missing file: %s", rel) } // it's okay if we're doing a partial check and the file isn't there as long as the path isn't in the list of paths to check diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 281bd6f80f..78f005f5cf 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -417,7 +417,7 @@ func (p *Packager) handleIfPartialPkg() error { } var shasum string - if shasum, err = utils.GetCryptoHash(destination, crypto.SHA256); err != nil { + if shasum, err = utils.GetCryptoHashFromFile(destination, crypto.SHA256); err != nil { return fmt.Errorf("unable to get sha256sum of package: %w", err) } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 94654fc01a..fb31a11590 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,6 +27,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/go-git/go-git/v5/plumbing" "github.com/mholt/archiver/v3" @@ -157,7 +158,7 @@ func (p *Packager) Create(baseDir string) error { } } - imgList := utils.Unique(combinedImageList) + imgList := helpers.Unique(combinedImageList) // Images are handled separately from other component assets. if len(imgList) > 0 { @@ -175,7 +176,7 @@ func (p *Packager) Create(baseDir string) error { return imgConfig.PullAll() } - if err := utils.Retry(doPull, 3, 5*time.Second); err != nil { + if err := helpers.Retry(doPull, 3, 5*time.Second); err != nil { return fmt.Errorf("unable to pull images after 3 attempts: %w", err) } } @@ -403,7 +404,7 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkel // Abort packaging on invalid shasum (if one is specified). if file.Shasum != "" { - if actualShasum, _ := utils.GetCryptoHash(dst, crypto.SHA256); actualShasum != file.Shasum { + if actualShasum, _ := utils.GetCryptoHashFromFile(dst, crypto.SHA256); actualShasum != file.Shasum { return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", file.Source, file.Shasum, actualShasum) } } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index a663e7e590..798ffffdbc 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -24,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/pterm/pterm" corev1 "k8s.io/api/core/v1" @@ -315,7 +316,7 @@ func (p *Packager) processComponentFiles(component types.ZarfComponent, pkgLocat // If a shasum is specified check it again on deployment as well if file.Shasum != "" { spinner.Updatef("Validating SHASUM for %s", file.Target) - if shasum, _ := utils.GetCryptoHash(fileLocation, crypto.SHA256); shasum != file.Shasum { + if shasum, _ := utils.GetCryptoHashFromFile(fileLocation, crypto.SHA256); shasum != file.Shasum { return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", file.Source, file.Shasum, shasum) } } @@ -448,7 +449,7 @@ func (p *Packager) pushImagesToRegistry(componentImages []string, noImgChecksum Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, } - return utils.Retry(func() error { + return helpers.Retry(func() error { return imgConfig.PushToZarfRegistry() }, 3, 5*time.Second) } @@ -480,7 +481,7 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error } // Try repo push up to 3 times - if err := utils.Retry(tryPush, 3, 5*time.Second); err != nil { + if err := helpers.Retry(tryPush, 3, 5*time.Second); err != nil { return fmt.Errorf("unable to push repo %s to the Git Server: %w", repoURL, err) } } diff --git a/src/pkg/packager/deprecated/common.go b/src/pkg/packager/deprecated/common.go index bfe9cdbcde..f2a79822e1 100644 --- a/src/pkg/packager/deprecated/common.go +++ b/src/pkg/packager/deprecated/common.go @@ -10,7 +10,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/pterm/pterm" ) @@ -43,7 +43,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) migratedComponent = component // If the component has already been migrated, clear the deprecated scripts. - if utils.SliceContains(build.Migrations, ScriptsToActionsMigrated) { + if helpers.SliceContains(build.Migrations, ScriptsToActionsMigrated) { migratedComponent.DeprecatedScripts = types.DeprecatedZarfComponentScripts{} } else { // Otherwise, run the migration. @@ -54,7 +54,7 @@ func MigrateComponent(build types.ZarfBuildData, component types.ZarfComponent) } // If the component has already been migrated, clear the setVariable definitions. - if utils.SliceContains(build.Migrations, PluralizeSetVariable) { + if helpers.SliceContains(build.Migrations, PluralizeSetVariable) { migratedComponent = clearSetVariables(migratedComponent) } else { // Otherwise, run the migration. diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 4876b60641..a45fc3fb71 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" "helm.sh/helm/v3/pkg/storage/driver" @@ -73,7 +74,7 @@ func (p *Packager) Remove(packageName string) (err error) { // Flip requested based on if this is a partial removal requested := !partialRemove - if utils.SliceContains(requestedComponents, component.Name) { + if helpers.SliceContains(requestedComponents, component.Name) { requested = true } @@ -118,9 +119,9 @@ func (p *Packager) Remove(packageName string) (err error) { } } - for _, c := range utils.Reverse(deployedPackage.DeployedComponents) { + for _, c := range helpers.Reverse(deployedPackage.DeployedComponents) { // Only remove the component if it was requested or if we are removing the whole package - if partialRemove && !utils.SliceContains(requestedComponents, c.Name) { + if partialRemove && !helpers.SliceContains(requestedComponents, c.Name) { continue } @@ -156,7 +157,7 @@ func (p *Packager) updatePackageSecret(deployedPackage types.DeployedPackage, pa func (p *Packager) removeComponent(deployedPackage types.DeployedPackage, deployedComponent types.DeployedComponent, spinner *message.Spinner) (types.DeployedPackage, error) { components := deployedPackage.Data.Components - c := utils.Find(components, func(t types.ZarfComponent) bool { + c := helpers.Find(components, func(t types.ZarfComponent) bool { return t.Name == deployedComponent.Name }) @@ -172,7 +173,7 @@ func (p *Packager) removeComponent(deployedPackage types.DeployedPackage, deploy return deployedPackage, fmt.Errorf("unable to run the before action for component (%s): %w", c.Name, err) } - for _, chart := range utils.Reverse(deployedComponent.InstalledCharts) { + for _, chart := range helpers.Reverse(deployedComponent.InstalledCharts) { spinner.Updatef("Uninstalling chart '%s' from the '%s' component", chart.ChartName, deployedComponent.Name) helmCfg := helm.Helm{} @@ -190,7 +191,7 @@ func (p *Packager) removeComponent(deployedPackage types.DeployedPackage, deploy // NOTE: We are saving the secret as we remove charts in case a failure happens later on in the process of removing the component. // If we don't save the secrets as we remove charts, we will run into issues if we try to remove the component again as we will // be trying to remove charts that have already been removed. - deployedComponent.InstalledCharts = utils.RemoveMatches(deployedComponent.InstalledCharts, func(t types.InstalledChart) bool { + deployedComponent.InstalledCharts = helpers.RemoveMatches(deployedComponent.InstalledCharts, func(t types.InstalledChart) bool { return t.ChartName == chart.ChartName }) p.updatePackageSecret(deployedPackage, deployedPackage.Name) @@ -207,7 +208,7 @@ func (p *Packager) removeComponent(deployedPackage types.DeployedPackage, deploy } // Remove the component we just removed from the array - deployedPackage.DeployedComponents = utils.RemoveMatches(deployedPackage.DeployedComponents, func(t types.DeployedComponent) bool { + deployedPackage.DeployedComponents = helpers.RemoveMatches(deployedPackage.DeployedComponents, func(t types.DeployedComponent) bool { return t.Name == c.Name }) diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index c58fdf1f7c..5652059147 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -21,7 +22,7 @@ func (p *Packager) fillActiveTemplate() error { promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { // Ensure uppercase keys - setFromCLIConfig := utils.TransformMapKeys(p.cfg.CreateOpts.SetVariables, strings.ToUpper) + setFromCLIConfig := helpers.TransformMapKeys(p.cfg.CreateOpts.SetVariables, strings.ToUpper) yamlTemplates, err := utils.FindYamlTemplates(&p.cfg.Pkg, templatePrefix, "###") if err != nil { @@ -73,7 +74,7 @@ func (p *Packager) fillActiveTemplate() error { // setVariableMapInConfig handles setting the active variables used to template component files. func (p *Packager) setVariableMapInConfig() error { // Ensure uppercase keys - setVariableValues := utils.TransformMapKeys(p.cfg.DeployOpts.SetVariables, strings.ToUpper) + setVariableValues := helpers.TransformMapKeys(p.cfg.DeployOpts.SetVariables, strings.ToUpper) for name, value := range setVariableValues { p.setVariableInConfig(name, value, false, false) } diff --git a/src/pkg/transform/artifact.go b/src/pkg/transform/artifact.go index 105a6f9f50..0b122c5420 100644 --- a/src/pkg/transform/artifact.go +++ b/src/pkg/transform/artifact.go @@ -10,7 +10,7 @@ import ( "regexp" "strings" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) const ( @@ -77,7 +77,7 @@ func GenTransformURL(targetBaseURL string, sourceURL string) (*url.URL, error) { packageName = fileName } // Add crc32 hash of the url to the end of the package name - packageNameGlobal := fmt.Sprintf("%s-%d", packageName, utils.GetCRCHash(sanitizedURL)) + packageNameGlobal := fmt.Sprintf("%s-%d", packageName, helpers.GetCRCHash(sanitizedURL)) version := strings.ReplaceAll(matches[idx("version")], "/", "") if version == "" { diff --git a/src/pkg/transform/git.go b/src/pkg/transform/git.go index 84bb23386d..f3b847fd6a 100644 --- a/src/pkg/transform/git.go +++ b/src/pkg/transform/git.go @@ -10,7 +10,7 @@ import ( "regexp" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) // For further explanation: https://regex101.com/r/YxpfhC/3 @@ -32,7 +32,7 @@ func MutateGitURLsInText(targetBaseURL string, text string, pushUser string) str // GitTransformURLSplitRef takes a git url and returns a separated source url and zarf reference. func GitTransformURLSplitRef(sourceURL string) (string, string, error) { - get, err := utils.MatchRegex(gitURLRegex, sourceURL) + get, err := helpers.MatchRegex(gitURLRegex, sourceURL) if err != nil { return "", "", fmt.Errorf("unable to get extract the source url and ref from the url %s", sourceURL) @@ -46,7 +46,7 @@ func GitTransformURLSplitRef(sourceURL string) (string, string, error) { // GitTransformURLtoFolderName takes a git url and returns the folder name for the repo in the Zarf package. func GitTransformURLtoFolderName(sourceURL string) (string, error) { - get, err := utils.MatchRegex(gitURLRegex, sourceURL) + get, err := helpers.MatchRegex(gitURLRegex, sourceURL) if err != nil { // Unable to find a substring match for the regex @@ -58,7 +58,7 @@ func GitTransformURLtoFolderName(sourceURL string) (string, error) { // Add crc32 hash of the repoName to the end of the repo gitURL := fmt.Sprintf("%s%s/%s%s%s", get("proto"), get("hostPath"), get("repo"), get("git"), get("atRef")) - checksum := utils.GetCRCHash(gitURL) + checksum := helpers.GetCRCHash(gitURL) newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) @@ -67,7 +67,7 @@ func GitTransformURLtoFolderName(sourceURL string) (string, error) { // GitTransformURLtoRepoName takes a git url and returns the name of the repo in the remote airgap repository. func GitTransformURLtoRepoName(sourceURL string) (string, error) { - get, err := utils.MatchRegex(gitURLRegex, sourceURL) + get, err := helpers.MatchRegex(gitURLRegex, sourceURL) if err != nil { // Unable to find a substring match for the regex @@ -80,7 +80,7 @@ func GitTransformURLtoRepoName(sourceURL string) (string, error) { sanitizedURL := fmt.Sprintf("%s/%s", get("hostPath"), repoName) // Add crc32 hash of the repoName to the end of the repo - checksum := utils.GetCRCHash(sanitizedURL) + checksum := helpers.GetCRCHash(sanitizedURL) newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) diff --git a/src/pkg/transform/image.go b/src/pkg/transform/image.go index 53dcc5167c..af2c54a4af 100644 --- a/src/pkg/transform/image.go +++ b/src/pkg/transform/image.go @@ -7,7 +7,7 @@ package transform import ( "fmt" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/distribution/distribution/reference" ) @@ -30,7 +30,7 @@ func ImageTransformHost(targetHost, srcReference string) (string, error) { } // Generate a crc32 hash of the image host + name - checksum := utils.GetCRCHash(image.Name) + checksum := helpers.GetCRCHash(image.Name) // If this image is specified by digest then don't add a checksum it as it will already be a specific SHA if image.Digest != "" { diff --git a/src/pkg/utils/auth.go b/src/pkg/utils/auth.go index e52d2c19a2..69a313cc13 100644 --- a/src/pkg/utils/auth.go +++ b/src/pkg/utils/auth.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/auth_test.go b/src/pkg/utils/auth_test.go index bac0aa1d79..6260e7af0d 100644 --- a/src/pkg/utils/auth_test.go +++ b/src/pkg/utils/auth_test.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions.package utils package utils import ( diff --git a/src/pkg/utils/bytes.go b/src/pkg/utils/bytes.go index 094f96b78c..27321723db 100644 --- a/src/pkg/utils/bytes.go +++ b/src/pkg/utils/bytes.go @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors // forked from https://www.socketloop.com/tutorials/golang-byte-format-example -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/credentials.go b/src/pkg/utils/credentials.go index e9854e71e4..83d64ac981 100644 --- a/src/pkg/utils/credentials.go +++ b/src/pkg/utils/credentials.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/hash.go b/src/pkg/utils/hash.go deleted file mode 100644 index af5e4bd4c9..0000000000 --- a/src/pkg/utils/hash.go +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic helper functions -package utils - -import ( - "crypto" - "encoding/hex" - "hash/crc32" - "io" - "os" - - "github.com/defenseunicorns/zarf/src/pkg/message" -) - -// GetCryptoHash returns the computed SHA256 Sum of a given file -func GetCryptoHash(path string, hashName crypto.Hash) (string, error) { - var data io.ReadCloser - var err error - - if IsURL(path) { - // Handle download from URL - message.Warn("This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link.") - data = Fetch(path) - } else { - // Handle local file - data, err = os.Open(path) - if err != nil { - return "", err - } - } - - defer data.Close() - - hash := hashName.New() - if _, err = io.Copy(hash, data); err != nil { - return "", err - } - - return hex.EncodeToString(hash.Sum(nil)), nil -} - -// GetCRCHash returns the computed CRC32 Sum of a given string -func GetCRCHash(text string) uint32 { - table := crc32.MakeTable(crc32.IEEE) - return crc32.Checksum([]byte(text), table) -} diff --git a/src/pkg/utils/helpers/hash.go b/src/pkg/utils/helpers/hash.go new file mode 100644 index 0000000000..c02d16452d --- /dev/null +++ b/src/pkg/utils/helpers/hash.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import ( + "crypto" + "encoding/hex" + "hash/crc32" + "io" +) + +// GetCryptoHash returns the computed SHA256 Sum of a given file +func GetCryptoHash(data io.ReadCloser, hashName crypto.Hash) (string, error) { + hash := hashName.New() + if _, err := io.Copy(hash, data); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// GetCRCHash returns the computed CRC32 Sum of a given string +func GetCRCHash(text string) uint32 { + table := crc32.MakeTable(crc32.IEEE) + return crc32.Checksum([]byte(text), table) +} diff --git a/src/pkg/utils/misc.go b/src/pkg/utils/helpers/misc.go similarity index 97% rename from src/pkg/utils/misc.go rename to src/pkg/utils/helpers/misc.go index 70eab334a7..f3c328630a 100644 --- a/src/pkg/utils/misc.go +++ b/src/pkg/utils/helpers/misc.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. -package utils +// Package helpers provides generic helper functions with no external imports +package helpers import ( "fmt" diff --git a/src/pkg/utils/htpasswd.go b/src/pkg/utils/htpasswd.go index 69478b57a8..3a830002d9 100644 --- a/src/pkg/utils/htpasswd.go +++ b/src/pkg/utils/htpasswd.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/image.go b/src/pkg/utils/image.go index 0060d6a458..57c4717122 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/utils/image.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index e989b3a2f9..e00d7280cc 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -6,6 +6,7 @@ package utils import ( "bufio" + "crypto" "crypto/sha256" "encoding/hex" "fmt" @@ -19,6 +20,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/otiai10/copy" ) @@ -27,6 +29,26 @@ const ( tmpPathPrefix = "zarf-" ) +func GetCryptoHashFromFile(path string, hashName crypto.Hash) (string, error) { + var data io.ReadCloser + var err error + + if IsURL(path) { + // Handle download from URL + message.Warn("This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link.") + data = Fetch(path) + } else { + // Handle local file + data, err = os.Open(path) + if err != nil { + return "", err + } + } + + defer data.Close() + return helpers.GetCryptoHash(data, hashName) +} + // TextTemplate represents a value to be templated into a text file. type TextTemplate struct { Sensitive bool diff --git a/src/pkg/utils/json.go b/src/pkg/utils/json.go index 17987c2838..fe14f7289d 100644 --- a/src/pkg/utils/json.go +++ b/src/pkg/utils/json.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/network.go b/src/pkg/utils/network.go index 0b2d598f67..eb96967375 100644 --- a/src/pkg/utils/network.go +++ b/src/pkg/utils/network.go @@ -138,7 +138,7 @@ func DownloadToFile(src string, dst string, cosignKeyPath string) (err error) { // If the file has a checksum, validate it if len(checksum) > 0 { - received, err := GetCryptoHash(dst, crypto.SHA256) + received, err := GetCryptoHashFromFile(dst, crypto.SHA256) if err != nil { return err } diff --git a/src/pkg/utils/network_test.go b/src/pkg/utils/network_test.go index c44976b7ba..3cd5923590 100644 --- a/src/pkg/utils/network_test.go +++ b/src/pkg/utils/network_test.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go index f41e48c3e9..0c074580d2 100644 --- a/src/pkg/utils/random.go +++ b/src/pkg/utils/random.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/sget.go b/src/pkg/utils/sget.go index 08bf2a03a1..79c9fca473 100644 --- a/src/pkg/utils/sget.go +++ b/src/pkg/utils/sget.go @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/sort.go b/src/pkg/utils/sort.go index d66b576ec0..6fd625c96d 100644 --- a/src/pkg/utils/sort.go +++ b/src/pkg/utils/sort.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import "fmt" diff --git a/src/pkg/utils/sort_test.go b/src/pkg/utils/sort_test.go index 052fb48a7e..2cd2a29c42 100644 --- a/src/pkg/utils/sort_test.go +++ b/src/pkg/utils/sort_test.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/transport.go b/src/pkg/utils/transport.go index ee14aafa6f..437c1918cb 100644 --- a/src/pkg/utils/transport.go +++ b/src/pkg/utils/transport.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils import ( diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index a191ee0e80..e10b6c7e6a 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. +// Package utils provides generic utility functions. package utils // fork from https://github.com/goccy/go-yaml/blob/master/cmd/ycat/ycat.go diff --git a/src/test/common.go b/src/test/common.go index 25710193f5..e538fadf52 100644 --- a/src/test/common.go +++ b/src/test/common.go @@ -13,8 +13,8 @@ import ( "strings" "testing" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" dconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" "github.com/stretchr/testify/require" @@ -92,7 +92,7 @@ func (e2e *ZarfE2ETest) GetMismatchedArch() string { // GetLogFileContents gets the log file contents from a given run's std error. func (e2e *ZarfE2ETest) GetLogFileContents(t *testing.T, stdErr string) string { - get, err := utils.MatchRegex(logRegex, stdErr) + get, err := helpers.MatchRegex(logRegex, stdErr) require.NoError(t, err) logFile := get("logFile") logContents, err := os.ReadFile(logFile) From b10d7828c73c5daefc9b7dc465bb8e97f42e4486 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Wed, 5 Jul 2023 18:57:53 -0400 Subject: [PATCH 2/3] [TASK] using logger Signed-off-by: Case Wylie --- src/cmd/prepare.go | 2 +- src/internal/packager/git/push.go | 2 +- src/pkg/transform/git.go | 6 ++---- src/pkg/transform/git_test.go | 3 ++- src/pkg/transform/types.go | 4 ++++ 5 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 src/pkg/transform/types.go diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 6bcbce35b5..81117b8ad8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -46,7 +46,7 @@ var prepareTransformGitLinks = &cobra.Command{ // Perform git url transformation via regex text := string(content) - processedText := transform.MutateGitURLsInText(pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) + processedText := transform.MutateGitURLsInText(message.Warnf, pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) // Print the differences message.PrintDiff(text, processedText) diff --git a/src/internal/packager/git/push.go b/src/internal/packager/git/push.go index 54759ff922..12ffd4f36b 100644 --- a/src/internal/packager/git/push.go +++ b/src/internal/packager/git/push.go @@ -98,7 +98,7 @@ func (g *Git) prepRepoForPush() (*git.Repository, error) { if err != nil { return nil, fmt.Errorf("unable to transform the git url: %w", err) } - + message.Debugf("Rewrite git URL: %s -> %s", remoteURL, targetURL.String()) // Remove any preexisting offlineRemotes (happens when a retry is triggered) _ = repo.DeleteRemote(offlineRemoteName) diff --git a/src/pkg/transform/git.go b/src/pkg/transform/git.go index f3b847fd6a..95e08c6be5 100644 --- a/src/pkg/transform/git.go +++ b/src/pkg/transform/git.go @@ -9,7 +9,6 @@ import ( "net/url" "regexp" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" ) @@ -17,12 +16,12 @@ import ( var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/(?P[\w\-\.]+?)(?P\.git)?(?P@(?P\+)?(?P[\/\+\w\-\.]+))?(?P\/(?Pinfo\/.*|git-upload-pack|git-receive-pack))?$`) // MutateGitURLsInText changes the gitURL hostname to use the repository Zarf is configured to use. -func MutateGitURLsInText(targetBaseURL string, text string, pushUser string) string { +func MutateGitURLsInText(logger Log, targetBaseURL string, text string, pushUser string) string { extractPathRegex := regexp.MustCompile(`[a-z]+:\/\/[^\/]+\/(.*\.git)`) output := extractPathRegex.ReplaceAllStringFunc(text, func(match string) string { output, err := GitTransformURL(targetBaseURL, match, pushUser) if err != nil { - message.Warnf("Unable to transform the git url, using the original url we have: %s", match) + logger("Unable to transform the git url, using the original url we have: %s", match) return match } return output.String() @@ -104,6 +103,5 @@ func GitTransformURL(targetBaseURL string, sourceURL string, pushUser string) (* } output := fmt.Sprintf("%s/%s/%s%s%s", targetBaseURL, pushUser, repoName, matches[idx("git")], matches[idx("gitPath")]) - message.Debugf("Rewrite git URL: %s -> %s", sourceURL, output) return url.Parse(output) } diff --git a/src/pkg/transform/git_test.go b/src/pkg/transform/git_test.go index c0c5aff7b8..4b7b0fd52c 100644 --- a/src/pkg/transform/git_test.go +++ b/src/pkg/transform/git_test.go @@ -41,6 +41,7 @@ var badGitURLs = []string{ } func TestMutateGitURLsInText(t *testing.T) { + dummyLogger := func(content string, args ...any) {} originalText := ` # Here we handle invalid URLs (see below comment) # We transform https://*/*.git URLs @@ -65,7 +66,7 @@ func TestMutateGitURLsInText(t *testing.T) { https://www.defenseunicorns.com/ ` - resultingText := MutateGitURLsInText("https://gitlab.com", originalText, "repo-owner") + resultingText := MutateGitURLsInText(dummyLogger, "https://gitlab.com", originalText, "repo-owner") require.Equal(t, expectedText, resultingText) } diff --git a/src/pkg/transform/types.go b/src/pkg/transform/types.go new file mode 100644 index 0000000000..7b19eaaee2 --- /dev/null +++ b/src/pkg/transform/types.go @@ -0,0 +1,4 @@ +package transform + +// Log is a function that logs a message. +type Log func(string, ...any) From ed1ba9d02172cee377795dcb5221b56360793021 Mon Sep 17 00:00:00 2001 From: Case Wylie Date: Thu, 6 Jul 2023 14:11:21 -0400 Subject: [PATCH 3/3] [TASK] lint Signed-off-by: Case Wylie --- src/pkg/utils/io.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index e00d7280cc..6bee5fe0fd 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -29,6 +29,7 @@ const ( tmpPathPrefix = "zarf-" ) +// GetCryptoHashFromFile returns the computed SHA256 Sum of a given file func GetCryptoHashFromFile(path string, hashName crypto.Hash) (string, error) { var data io.ReadCloser var err error