From a5793553089ef16590f5a94032a29a675876bccf Mon Sep 17 00:00:00 2001 From: Saurabh Parekh Date: Wed, 31 Jul 2024 20:31:50 -0700 Subject: [PATCH] Remove promo flag and other dependent flags from the generatebundlefile CLI --- generatebundlefile/README.md | 64 +--- generatebundlefile/buildspecs/buildspec.yml | 11 - generatebundlefile/bundle.go | 24 -- generatebundlefile/bundle_test.go | 132 ------- generatebundlefile/bundle_types.go | 18 - .../data/promote/autoscaler/promote.yaml | 39 -- .../eks-anywhere-packages/promote.yaml | 31 -- .../data/promote/emissary/promote.yaml | 16 - .../data/promote/metallb/promote.yaml | 18 - .../data/promote/metrics-server/promote.yaml | 39 -- generatebundlefile/ecr.go | 133 +------ generatebundlefile/ecr_helper.go | 215 +---------- generatebundlefile/ecr_public.go | 97 +---- generatebundlefile/ecr_test.go | 88 +---- generatebundlefile/go.mod | 3 +- generatebundlefile/go.sum | 6 - generatebundlefile/hack/common.sh | 7 - generatebundlefile/hack/promote.sh | 51 --- generatebundlefile/helm.go | 11 +- generatebundlefile/input.go | 20 +- generatebundlefile/main.go | 358 +----------------- generatebundlefile/options.go | 22 +- generatebundlefile/promote.go | 232 +----------- generatebundlefile/region.go | 74 ---- generatebundlefile/sts.go | 2 +- 25 files changed, 44 insertions(+), 1667 deletions(-) delete mode 100644 generatebundlefile/buildspecs/buildspec.yml delete mode 100644 generatebundlefile/data/promote/autoscaler/promote.yaml delete mode 100644 generatebundlefile/data/promote/eks-anywhere-packages/promote.yaml delete mode 100644 generatebundlefile/data/promote/emissary/promote.yaml delete mode 100644 generatebundlefile/data/promote/metallb/promote.yaml delete mode 100644 generatebundlefile/data/promote/metrics-server/promote.yaml delete mode 100755 generatebundlefile/hack/promote.sh delete mode 100644 generatebundlefile/region.go diff --git a/generatebundlefile/README.md b/generatebundlefile/README.md index ff7674ae8..fa40fafea 100644 --- a/generatebundlefile/README.md +++ b/generatebundlefile/README.md @@ -2,7 +2,7 @@ ## Overview -This binary reads an input file describing the curated packages to be included in the bundle then generates a bundle custom resource file. In addition, it has the ability to promote images and Helm charts between container registries. +This binary reads an input file describing the curated packages to be included in the bundle then generates a bundle custom resource file. There are three types of helm version tags we can input: @@ -23,15 +23,16 @@ make build ### Bundle Generation To generate a signed bundle you need an AWS KMS key that is: + - Asymmetric - ECC_NIST_P256 -- Key Policy enabled for CLI user to have `kms:Sign` configured. +- Key Policy enabled for CLI user to have `kms:Sign` configured. ```sh generatebundlefile --input data/sample_input.yaml --key alias/signingPackagesKey ``` -This will output all the corresponding CRD's into `output/bundle.yaml` +This will output all the corresponding CRD's into `output/bundle.yaml` #### Sample Bundle Generation @@ -41,47 +42,11 @@ generatebundlefile --generate-sample This will output a sample bundle file to ./output. - -### Public Promotion to another Account - -```sh -generatebundlefile --public-profile "profile-name" --input data/sample_input.yaml -``` - -This command will move **Only** the helm chart from the listed input files to the target **public** ECR in another account. - -### Private Promotion to another Account - -```sh -generatebundlefile --private-profile "profile-name" --input data/sample_input.yaml -``` - -This command will move **Only** the images from the listed helm charts in the input files to the target **private** ECR in another account. - -### Package Promotion - -To promote a package from a private ECR to public you need the repository name. This repository must contain a Helm chart built by the process in the eks-anywhere-build-tooling git repository. -This will **both** for the helm chart, and the imgaes required to the public ECR of the calling user. - -To promote the latest helm chart from the private repository. - -```sh -generatebundlefile --promote hello-eks-anywhere -``` - -To promote multiple versions of a helm chart in one command - -```sh -generatebundlefile --promote hello-eks-anywhere --input data/promote.yaml -``` - ### Input File Supported Formats -Currently the following formats can be used as input files each of the following command flags. +Currently the following formats can be used as input files with each of the following command flags. --key alias/signingPackagesKey ---private-profile ---public--profile #### Private Registry @@ -176,22 +141,3 @@ packages: - name: 0.1.1-92904119e6e1bae35bf88663d0875259d42346f8 - name: latest ``` - - -#### Public Profile with Image Copy - -```yaml -name: "v1-22-1001" -kubernetesVersion: "1.22" -packages: - - org: aws-containers - projects: - - name: hello-eks-anywhere - copyimages: true - repository: hello-eks-anywhere - registry: public.ecr.aws/eks-anywhere - versions: - - name: 0.1.1-92904119e6e1bae35bf88663d0875259d42346f8 - - name: latest - -``` diff --git a/generatebundlefile/buildspecs/buildspec.yml b/generatebundlefile/buildspecs/buildspec.yml deleted file mode 100644 index b876761fb..000000000 --- a/generatebundlefile/buildspecs/buildspec.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 0.2 - -phases: - pre_build: - commands: - - ./generatebundlefile/hack/setup.sh - - build: - commands: - - cd generatebundlefile - - ./hack/promote.sh diff --git a/generatebundlefile/bundle.go b/generatebundlefile/bundle.go index 5e5253a5b..44484f2fb 100644 --- a/generatebundlefile/bundle.go +++ b/generatebundlefile/bundle.go @@ -120,30 +120,6 @@ func AddMetadata(s api.PackageBundleSpec, name string) *api.PackageBundle { } } -// IfSignature checks if a signature exsits on a Packagebundle -func IfSignature(bundle *api.PackageBundle) (bool, error) { - annotations := bundle.Annotations - if annotations != nil { - return true, nil - } - return false, nil -} - -// CheckSignature checks if current signature is equal to signature to added as an annotation, and skips if they are the same. -func CheckSignature(bundle *api.PackageBundle, signature string) (bool, error) { - if signature == "" || bundle == nil { - return false, fmt.Errorf("either signature or bundle is missing, but are required") - } - annotations := map[string]string{ - FullSignatureAnnotation: signature, - } - // If current signature on file isn't at the --signature input return false, otherwsie true - if annotations[FullSignatureAnnotation] != bundle.Annotations[FullSignatureAnnotation] { - return false, fmt.Errorf("A signature already exists on the input file signatue") - } - return true, nil -} - // GetBundleSignature calls KMS and retrieves a signature, then base64 decodes it and returns that back func GetBundleSignature(ctx context.Context, bundle *api.PackageBundle, key string) (string, error) { digest, _, err := sig.GetDigest(bundle, sig.EksaDomain) diff --git a/generatebundlefile/bundle_test.go b/generatebundlefile/bundle_test.go index 2686669a6..e152b679a 100644 --- a/generatebundlefile/bundle_test.go +++ b/generatebundlefile/bundle_test.go @@ -190,138 +190,6 @@ func TestNewPackageFromInput(t *testing.T) { } } -func TestIfSignature(t *testing.T) { - tests := []struct { - testname string - testbundle *api.PackageBundle - wantBool bool - }{ - { - testname: "Test no annotations", - testbundle: &api.PackageBundle{ - TypeMeta: metav1.TypeMeta{ - Kind: api.PackageBundleKind, - APIVersion: api.SchemeBuilder.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "1.20", - Namespace: "eksa-packages", - }, - }, - wantBool: false, - }, - { - testname: "Test with annotations", - testbundle: &api.PackageBundle{ - TypeMeta: metav1.TypeMeta{ - Kind: api.PackageBundleKind, - APIVersion: api.SchemeBuilder.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "1.20", - Namespace: "eksa-packages", - Annotations: map[string]string{ - "eksa.aws.com/signature": "123", - }, - }, - }, - wantBool: true, - }, - } - for _, tc := range tests { - t.Run(tc.testname, func(tt *testing.T) { - got, err := IfSignature(tc.testbundle) - if err != nil { - tt.Fatalf("IfSignature() error = %v", err) - } - if got != tc.wantBool { - tt.Fatalf("IfSignature() = %#v\n\n\n, want %#v", got, tc.wantBool) - } - }) - } -} - -func TestCheckSignature(t *testing.T) { - tests := []struct { - testname string - testbundle *api.PackageBundle - signature string - wantBool bool - wantErr bool - }{ - { - testname: "Test empty signature", - testbundle: &api.PackageBundle{ - TypeMeta: metav1.TypeMeta{ - Kind: api.PackageBundleKind, - APIVersion: api.SchemeBuilder.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "1.20", - Namespace: "eksa-packages", - }, - }, - signature: "", - wantErr: true, - }, - { - testname: "Test empty Bundle", - testbundle: nil, - signature: "signature-123", - wantErr: true, - }, - { - testname: "Test same signature", - testbundle: &api.PackageBundle{ - TypeMeta: metav1.TypeMeta{ - Kind: api.PackageBundleKind, - APIVersion: api.SchemeBuilder.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "1.20", - Namespace: "eksa-packages", - Annotations: map[string]string{ - "eksa.aws.com/signature": "signature-123", - }, - }, - }, - signature: "signature-123", - wantBool: true, - wantErr: false, - }, - { - testname: "Test different signature", - testbundle: &api.PackageBundle{ - TypeMeta: metav1.TypeMeta{ - Kind: api.PackageBundleKind, - APIVersion: api.SchemeBuilder.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "1.20", - Namespace: "eksa-packages", - Annotations: map[string]string{ - "eksa.aws.com/signature": "signature-456", - }, - }, - }, - signature: "signature-123", - wantBool: false, - wantErr: true, - }, - } - for _, tc := range tests { - t.Run(tc.testname, func(tt *testing.T) { - got, err := CheckSignature(tc.testbundle, tc.signature) - if (err != nil) != tc.wantErr { - tt.Fatalf("CheckSignature() error = %v, wantErr %v", err, tc.wantErr) - } - if got != tc.wantBool { - tt.Fatalf("CheckSignature() = %#v\n\n\n, want %#v", got, tc.wantBool) - } - }) - } -} - type mockPublicRegistryClientBundle struct { err error } diff --git a/generatebundlefile/bundle_types.go b/generatebundlefile/bundle_types.go index e71edb6e5..8248b0598 100644 --- a/generatebundlefile/bundle_types.go +++ b/generatebundlefile/bundle_types.go @@ -29,15 +29,6 @@ type SigningPackageBundle struct { Status interface{} `json:"status,omitempty"` } -// newSigningPackageBundle is api.PackageBundle using SigningObjectMeta instead of metav1.ObjectMeta -func newSigningPackageBundle(bundle *api.PackageBundle) *SigningPackageBundle { - return &SigningPackageBundle{ - PackageBundle: bundle, - SigningObjectMeta: newSigningObjectMeta(&bundle.ObjectMeta), - Status: nil, - } -} - // SigningObjectMeta removes fields that shouldn't be included when signing. type SigningObjectMeta struct { *metav1.ObjectMeta @@ -48,14 +39,6 @@ type SigningObjectMeta struct { CreationTimestamp interface{} `json:"creationTimestamp,omitempty"` } -// newSigningObjectMeta is metav1.ObjectMeta without the CreationTimestamp since it gets added the yaml as null otherwise. -func newSigningObjectMeta(meta *metav1.ObjectMeta) *SigningObjectMeta { - return &SigningObjectMeta{ - ObjectMeta: meta, - CreationTimestamp: nil, - } -} - // Input is the schema for the input file // +kubebuilder:object:root=true // +kubebuilder:object:generate=false @@ -82,7 +65,6 @@ type Project struct { Repository string `json:"repository,omitempty"` Versions []Tag `json:"versions,omitempty"` WorkloadOnly bool `json:"workloadonly,omitempty"` - CopyImages bool `json:"copyimages,omitempty"` } // Tag is the release tag diff --git a/generatebundlefile/data/promote/autoscaler/promote.yaml b/generatebundlefile/data/promote/autoscaler/promote.yaml deleted file mode 100644 index f459f3cd7..000000000 --- a/generatebundlefile/data/promote/autoscaler/promote.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This file is only used to determine which Helm charts to move from Dev to Prod during a release. -name: "v1-22-1001" -kubernetesVersion: "1.22" -packages: - - org: cluster-autoscaler - projects: - - name: cluster-autoscaler - repository: cluster-autoscaler/charts/cluster-autoscaler - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 9.37.0-1.30-latest - - org: cluster-autoscaler - projects: - - name: cluster-autoscaler - repository: cluster-autoscaler/charts/cluster-autoscaler - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 9.37.0-1.29-latest - - org: cluster-autoscaler - projects: - - name: cluster-autoscaler - repository: cluster-autoscaler/charts/cluster-autoscaler - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 9.37.0-1.28-latest - - org: cluster-autoscaler - projects: - - name: cluster-autoscaler - repository: cluster-autoscaler/charts/cluster-autoscaler - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 9.37.0-1.27-latest - - org: cluster-autoscaler - projects: - - name: cluster-autoscaler - repository: cluster-autoscaler/charts/cluster-autoscaler - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 9.37.0-1.26-latest diff --git a/generatebundlefile/data/promote/eks-anywhere-packages/promote.yaml b/generatebundlefile/data/promote/eks-anywhere-packages/promote.yaml deleted file mode 100644 index ebc49c208..000000000 --- a/generatebundlefile/data/promote/eks-anywhere-packages/promote.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# This file is only used to determine which Helm charts to move from Dev to Prod during a release. -name: "v1-23-1001" -kubernetesVersion: "1.23" -packages: - - org: aws - projects: - - name: eks-anywhere-packages-crds - repository: eks-anywhere-packages-crds - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: latest - - - name: eks-anywhere-packages-migrations - repository: eks-anywhere-packages-migrations - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: latest - - - name: eks-anywhere-packages - repository: eks-anywhere-packages - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.0.0-latest - - name: latest - - - name: credential-provider-package - repository: credential-provider-package - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.0.0-latest - - name: latest diff --git a/generatebundlefile/data/promote/emissary/promote.yaml b/generatebundlefile/data/promote/emissary/promote.yaml deleted file mode 100644 index 03750b393..000000000 --- a/generatebundlefile/data/promote/emissary/promote.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This file is only used to determine which Helm charts to move from Dev to Prod during a release. -name: "v1-23-1001" -kubernetesVersion: "1.23" -packages: - - org: emissary - projects: - - name: emissary - repository: emissary-ingress/emissary - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 3.9.1-latest - - name: emissary-crds - repository: emissary-ingress/crds - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 3.9.1-latest diff --git a/generatebundlefile/data/promote/metallb/promote.yaml b/generatebundlefile/data/promote/metallb/promote.yaml deleted file mode 100644 index 6a14ebb94..000000000 --- a/generatebundlefile/data/promote/metallb/promote.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This file is only used to determine which Helm charts to move from Dev to Prod during a release. -name: "v1-23-1001" -kubernetesVersion: "1.23" -packages: - - org: metallb - projects: - - name: crds - repository: metallb/crds - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.14.5-latest - - org: metallb - projects: - - name: metallb - repository: metallb/metallb - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.14.5-latest diff --git a/generatebundlefile/data/promote/metrics-server/promote.yaml b/generatebundlefile/data/promote/metrics-server/promote.yaml deleted file mode 100644 index 4c5416ee3..000000000 --- a/generatebundlefile/data/promote/metrics-server/promote.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This file is only used to determine which Helm charts to move from Dev to Prod during a release. -name: "v1-22-1001" -kubernetesVersion: "1.22" -packages: - - org: metrics-server - projects: - - name: metrics-server - repository: metrics-server/charts/metrics-server - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.7.1-eks-1-30-latest - - org: metrics-server - projects: - - name: metrics-server - repository: metrics-server/charts/metrics-server - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.7.1-eks-1-29-latest - - org: metrics-server - projects: - - name: metrics-server - repository: metrics-server/charts/metrics-server - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.7.1-eks-1-28-latest - - org: metrics-server - projects: - - name: metrics-server - repository: metrics-server/charts/metrics-server - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.7.1-eks-1-27-latest - - org: metrics-server - projects: - - name: metrics-server - repository: metrics-server/charts/metrics-server - registry: 067575901363.dkr.ecr.us-west-2.amazonaws.com - versions: - - name: 0.7.1-eks-1-26-latest diff --git a/generatebundlefile/ecr.go b/generatebundlefile/ecr.go index 5c4595a3a..515116262 100644 --- a/generatebundlefile/ecr.go +++ b/generatebundlefile/ecr.go @@ -31,11 +31,6 @@ type registryClient interface { GetAuthorizationToken(ctx context.Context, params *ecr.GetAuthorizationTokenInput, optFns ...func(*ecr.Options)) (*ecr.GetAuthorizationTokenOutput, error) } -type CheckECR interface { - tagExistsInRepository(repository, tag string) (bool, error) - shaExistsInRepository(repository, tag string) (bool, error) -} - // NewECRClient Creates a new ECR Client client func NewECRClient(client registryClient, needsCreds bool) (*ecrClient, error) { ecrClient := &ecrClient{ @@ -146,7 +141,7 @@ func (c *ecrClient) GetShaForInputs(project Project) ([]api.SourceVersion, error // tagFromSha Looks up the Tag of an ECR artifact from a sha func (c *ecrClient) tagFromSha(repository, sha, substringTag string) (string, error) { if repository == "" || sha == "" { - return "", fmt.Errorf("Emtpy repository, or sha passed to the function") + return "", fmt.Errorf("empty repository, or sha passed to the function") } var imagelookup []ecrtypes.ImageIdentifier imagelookup = append(imagelookup, ecrtypes.ImageIdentifier{ImageDigest: &sha}) @@ -156,7 +151,7 @@ func (c *ecrClient) tagFromSha(repository, sha, substringTag string) (string, er ImageIds: imagelookup, }) if err != nil { - if strings.Contains(err.Error(), "does not exist within the repository") == true { + if strings.Contains(err.Error(), "does not exist within the repository") { return "", nil } else { return "", fmt.Errorf("looking up image details %v", err) @@ -192,11 +187,11 @@ func NewAuthFile(dockerstruct *DockerAuth) (DockerAuthFile, error) { jsonbytes, err := json.Marshal(*dockerstruct) auth := DockerAuthFile{} if err != nil { - return auth, fmt.Errorf("Marshalling docker auth file to json %w", err) + return auth, fmt.Errorf("marshalling docker auth file to json %w", err) } f, err := os.CreateTemp("", "dockerAuth") if err != nil { - return auth, fmt.Errorf("Creating tempfile %w", err) + return auth, fmt.Errorf("creating tempfile %w", err) } defer f.Close() fmt.Fprint(f, string(jsonbytes)) @@ -206,126 +201,8 @@ func NewAuthFile(dockerstruct *DockerAuth) (DockerAuthFile, error) { func (d DockerAuthFile) Remove() error { if d.Authfile == "" { - return fmt.Errorf("No Authfile in DockerAuthFile given") + return fmt.Errorf("no Authfile in DockerAuthFile given") } defer os.Remove(d.Authfile) return nil } - -// getNameAndVersion looks up the latest pushed helm chart's tag from a given repo name from ECR. -func (c *SDKClients) getNameAndVersion(repoName, tag, accountID string) (string, string, string, error) { - var version string - var sha string - splitname := strings.Split(repoName, ":") // TODO add a regex filter - name := splitname[0] - ecrname := fmt.Sprintf("%s.dkr.ecr.us-west-2.amazonaws.com/%s", accountID, name) - BundleLog.Info("Looking up ECR for Helm chart", "Repository", ecrname) - if len(splitname) > 0 { - // If for promotion, we use a named tag instead of latest we do a lookup for that tag. - if !strings.HasSuffix(tag, "latest") { - imageIDs := []ecrtypes.ImageIdentifier{{ImageTag: &tag}} - ImageDetails, err := c.ecrClient.Describe(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repoName), - ImageIds: imageIDs, - }) - if err != nil { - return "", "", "", fmt.Errorf("DescribeImagesRequest to ECR failed: %w", err) - } - if len(ImageDetails) == 1 { - return ecrname, tag, *ImageDetails[0].ImageDigest, nil - } - } - // If tag is -latest we do a timestamp lookup - if tag == "latest" { - ImageDetails, err := c.ecrClient.Describe(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repoName), - }) - if err != nil { - return "", "", "", err - } - var images []ImageDetailsBothECR - for _, image := range ImageDetails { - details, err := createECRImageDetails(ImageDetailsECR{PrivateImageDetails: image}) - if err != nil { - return "", "", "", err - } - images = append(images, details) - } - version, sha, err = getLatestHelmTagandSha(images) - return ecrname, version, sha, err - } - // If tag contains -latest we do timestamp lookup of any tags matching a regexp of the specified tag. - if strings.Contains(tag, "-latest") { - regex := regexp.MustCompile(`-latest`) - splitVersion := regex.Split(tag, -1) // extract out the version without the latest - ImageDetails, err := c.ecrClient.Describe(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repoName), - }) - if err != nil { - return "", "", "", fmt.Errorf("Unable to complete DescribeImagesRequest to ECR: %s", err) - } - var images []ImageDetailsBothECR - for _, image := range ImageDetails { - details, _ := createECRImageDetails(ImageDetailsECR{PrivateImageDetails: image}) - images = append(images, details) - } - filteredImageDetails := ImageTagFilter(images, splitVersion[0]) - version, sha, err = getLatestHelmTagandSha(filteredImageDetails) - if err != nil { - return "", "", "", err - } - return ecrname, version, sha, err - } - } - return "", "", "", fmt.Errorf("invalid repository: %q", repoName) -} - -// shaExistsInRepository checks if a given OCI artifact exists in a destination repo using the sha sum. -func (c *ecrClient) shaExistsInRepository(repository, sha string) (bool, error) { - if repository == "" || sha == "" { - return false, fmt.Errorf("Emtpy repository, or sha passed to the function") - } - var imagelookup []ecrtypes.ImageIdentifier - imagelookup = append(imagelookup, ecrtypes.ImageIdentifier{ImageDigest: &sha}) - ImageDetails, err := c.Describe(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repository), - ImageIds: imagelookup, - }) - if err != nil { - if strings.Contains(err.Error(), "does not exist within the repository") == true { - return false, nil - } - } - for _, detail := range ImageDetails { - if detail.ImageDigest != nil && *detail.ImageDigest == sha { - return true, nil - } - } - return false, nil -} - -// tagExistsInRepository checks if a given OCI artifact exists in a destination repo using the sha sum. -func (c *ecrClient) tagExistsInRepository(repository, tag string) (bool, error) { - if repository == "" || tag == "" { - return false, fmt.Errorf("Emtpy repository, or tag passed to the function") - } - var imagelookup []ecrtypes.ImageIdentifier - imagelookup = append(imagelookup, ecrtypes.ImageIdentifier{ImageTag: &tag}) - ImageDetails, err := c.Describe(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repository), - ImageIds: imagelookup, - }) - if err != nil { - if strings.Contains(err.Error(), "does not exist within the repository") == true { - return false, nil - } - } - for _, detail := range ImageDetails { - for _, Imagetag := range detail.ImageTags { - if tag == Imagetag { - return true, nil - } - } - } - return false, nil -} diff --git a/generatebundlefile/ecr_helper.go b/generatebundlefile/ecr_helper.go index 406bc722f..da1016ff9 100644 --- a/generatebundlefile/ecr_helper.go +++ b/generatebundlefile/ecr_helper.go @@ -1,28 +1,20 @@ package main import ( - "crypto/sha256" "fmt" - "os" "os/exec" - "path/filepath" "reflect" - "regexp" "strings" "time" ecrtypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" ecrpublictypes "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" - "github.com/go-logr/logr" "github.com/jinzhu/copier" "github.com/pkg/errors" - "golang.org/x/exp/slices" api "github.com/aws/eks-anywhere-packages/api/v1alpha1" ) -const imageIndexMediaType = "application/vnd.oci.image.index.v1+json" - // removeDuplicates removes any duplicates from Version list, useful for scenarios when // multiple tags for an image are present, this would cause duplicates on the bundle CRD, // so we remove the first one in this case since they are the same thing. @@ -39,16 +31,6 @@ func removeDuplicates(s []api.SourceVersion) []api.SourceVersion { return l } -// stringInSlice checks to see if a string is in a slice -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - // removeStringSlice removes a named string from a slice, without knowing it's index or it being ordered. func removeStringSlice(l []string, item string) []string { for i, other := range l { @@ -69,23 +51,6 @@ func deleteEmptyStringSlice(s []string) []string { return r } -func printSlice[v any](s []v) { - if len(s) == 0 { - return - } - fmt.Println(s[0]) - printSlice(s[1:]) -} - -func printMap[k comparable, v any](m map[k]v) { - if len(m) == 0 { - return - } - for k, v := range m { - fmt.Println(k, ":", v) - } -} - // ExecCommand runs a given command, and constructs the log/output. func ExecCommand(cmd *exec.Cmd) (string, error) { commandOutput, err := cmd.CombinedOutput() @@ -106,7 +71,7 @@ func splitECRName(s string) (string, string, error) { if len(chartNameList) > 1 { return strings.Join(chartNameList[1:], "/"), chartNameList[len(chartNameList)-1], nil } - return "", "", fmt.Errorf("Error: %s", "Failed parsing chartName, check the input URI is a valid ECR URI") + return "", "", fmt.Errorf("parsing chartName, check the input URI is a valid ECR URI") } // imageTagFilter is used when filtering a list of images for a specific tag or tag substring @@ -158,44 +123,16 @@ func createECRImageDetails(images ImageDetailsECR) (ImageDetailsBothECR, error) copier.Copy(&t, &images.PrivateImageDetails) return *t, nil } - return ImageDetailsBothECR{}, fmt.Errorf("Error marshalling image details from ECR lookup.") + return ImageDetailsBothECR{}, fmt.Errorf("marshalling image details from ECR lookup") } if reflect.DeepEqual(images.PrivateImageDetails, ecrtypes.ImageDetail{}) { if images.PublicImageDetails.ImageDigest != nil && images.PublicImageDetails.ImagePushedAt != nil && images.PublicImageDetails.RegistryId != nil && images.PublicImageDetails.RepositoryName != nil { copier.Copy(&t, &images.PublicImageDetails) return *t, nil } - return ImageDetailsBothECR{}, fmt.Errorf("Error marshalling image details from ECR lookup.") + return ImageDetailsBothECR{}, fmt.Errorf("marshalling image details from ECR lookup") } - return ImageDetailsBothECR{}, fmt.Errorf("Error no data passed to createImageDetails") -} - -// getLatestHelmTagandSha Iterates list of ECR Helm Charts, to find latest pushed image and return tag/sha of the latest pushed image -func getLatestHelmTagandSha(details []ImageDetailsBothECR) (string, string, error) { - var latest ImageDetailsBothECR - latest.ImagePushedAt = &time.Time{} - for _, detail := range details { - if len(details) < 1 || detail.ImagePushedAt == nil || detail.ImageDigest == nil || detail.ImageTags == nil || len(detail.ImageTags) == 0 || *detail.ImageManifestMediaType != "application/vnd.oci.image.manifest.v1+json" { - continue - } - if detail.ImagePushedAt != nil && latest.ImagePushedAt.Before(*detail.ImagePushedAt) { - latest = detail - } - } - // Check if latest is equal to empty struct, and return error if that's the case. - if reflect.DeepEqual(latest, ImageDetailsBothECR{}) { - return "", "", fmt.Errorf("error no images found") - } - - var imageTag string - imageTagRegex := regexp.MustCompile(`^\d+\.\d+\.\d+.*[0-9a-f]{40}$`) - for _, tag := range latest.ImageTags { - if imageTagRegex.MatchString(tag) { - imageTag = tag - break - } - } - return imageTag, *latest.ImageDigest, nil + return ImageDetailsBothECR{}, fmt.Errorf("no data passed to createImageDetails") } // getLatestImageSha Iterates list of ECR Public Helm Charts, to find latest pushed image and return tag/sha of the latest pushed image @@ -216,147 +153,3 @@ func getLatestImageSha(details []ImageDetailsBothECR) (*api.SourceVersion, error } return &api.SourceVersion{Name: latest.ImageTags[0], Digest: *latest.ImageDigest}, nil } - -// copyImage will copy an OCI artifact from one registry to another registry. -func copyImage(log logr.Logger, authFile, source, destination string) error { - // Create temporary directory for copying image artifacts locally - currentDir, err := os.Getwd() - if err != nil { - return err - } - tempImageDir := filepath.Join(currentDir, "temp-image-dir") - defer os.RemoveAll(tempImageDir) - - // Copy image from source registry to local directory - log.Info("Copying source image to local directory", "Source", source, "Directory", fmt.Sprintf("dir://%s", tempImageDir)) - cmd := exec.Command("skopeo", "copy", "--authfile", authFile, source, fmt.Sprintf("dir://%s", tempImageDir), "-f", "oci", "--all") - stdout, err := ExecCommand(cmd) - fmt.Printf("%s\n", stdout) - if err != nil { - return err - } - - manifestFile := filepath.Join(tempImageDir, "manifest.json") - - // Fetch manifest media type from manifest.json present inside the - // copied image directory - log.Info("Getting media type from root manifest JSON") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq -r '.mediaType'", manifestFile)) - mediaType, err := ExecCommand(cmd) - if err != nil { - return err - } - - if mediaType == imageIndexMediaType { - // Remove manifest.json files corresponding to all artifacts that are //not images. - // These might be SBOMs, attributions which `skopeo copy` cannot handle. We filter - // these out by checking for artifacts that have an "unknown" architecture value. - log.Info("Removing manifest JSON files for all non-image artifacts") - log.Info("Getting non-image artifact digests") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq -r '.manifests[] | select(.platform.architecture == \"unknown\").digest'", manifestFile)) - stdout, err = ExecCommand(cmd) - if err != nil { - return err - } - nonImageDigests := strings.Split(stdout, "\n") - for _, digest := range nonImageDigests { - if digest != "" { - trimmedDigest := strings.TrimPrefix(digest, "sha256:") - manifestFile := filepath.Join(tempImageDir, fmt.Sprintf("%s.manifest.json", trimmedDigest)) - err = os.Remove(manifestFile) - if err != nil { - return err - } - } - } - - // Next we move on to the gzipped files representing artifact layers. We need to delete - // the layer files for all artifacts expect those corresponding to the images. So we filter - // each artifact that is of image type and compile the list of layer files to retain. Then we - // iterate over all the files in the local directory and delete everything except this list. - log.Info("Removing compressed layer files for all non-image artifacts") - filesToRetain := []string{"manifest.json", "version"} - log.Info("Getting image artifact digests") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq -r '.manifests[] | select(.platform.architecture != \"unknown\").digest'", manifestFile)) - stdout, err = ExecCommand(cmd) - if err != nil { - return err - } - imageDigests := strings.Split(stdout, "\n") - for _, digest := range imageDigests { - trimmedDigest := strings.TrimPrefix(digest, "sha256:") - filesToRetain = append(filesToRetain, fmt.Sprintf("%s.manifest.json", trimmedDigest)) - manifestFile := filepath.Join(tempImageDir, fmt.Sprintf("%s.manifest.json", trimmedDigest)) - log.Info("Getting config digest for image") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq -r '.config.digest'", manifestFile)) - stdout, err = ExecCommand(cmd) - if err != nil { - return err - } - configDigest := strings.TrimPrefix(stdout, "sha256:") - filesToRetain = append(filesToRetain, configDigest) - - log.Info("Getting layer digests for image") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq -r '.layers[].digest'", manifestFile)) - stdout, err = ExecCommand(cmd) - if err != nil { - return err - } - layerDigests := strings.Split(stdout, "\n") - for _, digest := range layerDigests { - layerDigest := strings.TrimPrefix(digest, "sha256:") - if !slices.Contains(filesToRetain, layerDigest) { - filesToRetain = append(filesToRetain, layerDigest) - } - } - } - - tempImageDirFiles, err := os.ReadDir(tempImageDir) - if err != nil { - return err - } - for _, file := range tempImageDirFiles { - if !slices.Contains(filesToRetain, file.Name()) { - err = os.Remove(filepath.Join(tempImageDir, file.Name())) - if err != nil { - return err - } - } - } - - // Finally we update the root manifest.json to include only the image artifacts - // by deleting all other media types. - log.Info("Updating root manifest JSON contents to remove all non-image artifacts") - cmd = exec.Command("bash", "-c", fmt.Sprintf("cat %s | jq 'del(.manifests[] | select(.platform.architecture == \"unknown\"))'", manifestFile)) - updatedManifestContents, err := ExecCommand(cmd) - if err != nil { - return err - } - - err = os.WriteFile(manifestFile, []byte(updatedManifestContents), 0o644) - if err != nil { - return err - } - - // When using digest references as URIs, Skopeo complains if the manifest digest - // does not not match the destination reference. So we update the destination - // digest reference to the actual digest of the manifest to avoid this issue. - if strings.Contains(destination, "@sha256:") { - imageDigestRegex := regexp.MustCompile("sha256:.*") - h := sha256.New() - h.Write([]byte(updatedManifestContents)) - updatedManifestDigest := fmt.Sprintf("%x", h.Sum(nil)) - destination = imageDigestRegex.ReplaceAllString(destination, fmt.Sprintf("sha256:%s", updatedManifestDigest)) - } - } - - // Copy image from local directory to destination registry - log.Info("Copying image from local directory to destination", "Directory", fmt.Sprintf("dir://%s", tempImageDir), "Destination", destination) - cmd = exec.Command("skopeo", "copy", "--authfile", authFile, fmt.Sprintf("dir://%s", tempImageDir), destination, "-f", "oci", "--all") - stdout, err = ExecCommand(cmd) - fmt.Printf("%s\n", stdout) - if err != nil { - return err - } - return nil -} diff --git a/generatebundlefile/ecr_public.go b/generatebundlefile/ecr_public.go index fdd8d516a..234ced7cd 100644 --- a/generatebundlefile/ecr_public.go +++ b/generatebundlefile/ecr_public.go @@ -135,56 +135,6 @@ func (c *SDKClients) GetShaForPublicInputs(project Project) ([]api.SourceVersion return sourceVersion, nil } -// shaExistsInRepositoryPublic checks if a given OCI artifact exists in a destination repo using the sha sum. -func (c *ecrPublicClient) shaExistsInRepository(repository, sha string) (bool, error) { - if repository == "" || sha == "" { - return false, fmt.Errorf("Emtpy repository, or sha passed to the function") - } - var imagelookup []ecrpublictypes.ImageIdentifier - imagelookup = append(imagelookup, ecrpublictypes.ImageIdentifier{ImageDigest: &sha}) - ImageDetails, err := c.DescribePublic(&ecrpublic.DescribeImagesInput{ - RepositoryName: aws.String(repository), - ImageIds: imagelookup, - }) - if err != nil { - if strings.Contains(err.Error(), "does not exist within the repository") == true { - return false, nil - } - } - for _, detail := range ImageDetails { - if detail.ImageDigest != nil && *detail.ImageDigest == sha { - return true, nil - } - } - return false, nil -} - -// tagExistsInRepository checks if a given OCI artifact exists in a destination repo using the sha sum. -func (c *ecrPublicClient) tagExistsInRepository(repository, tag string) (bool, error) { - if repository == "" || tag == "" { - return false, fmt.Errorf("Emtpy repository, or tag passed to the function") - } - var imagelookup []ecrpublictypes.ImageIdentifier - imagelookup = append(imagelookup, ecrpublictypes.ImageIdentifier{ImageTag: &tag}) - ImageDetails, err := c.DescribePublic(&ecrpublic.DescribeImagesInput{ - RepositoryName: aws.String(repository), - ImageIds: imagelookup, - }) - if err != nil { - if strings.Contains(err.Error(), "does not exist within the repository") == true { - return false, nil - } - } - for _, detail := range ImageDetails { - for _, Imagetag := range detail.ImageTags { - if tag == Imagetag { - return true, nil - } - } - } - return false, nil -} - // GetRegistryURI gets the current account's AWS ECR Public registry URI func (c *ecrPublicClient) GetRegistryURI() (string, error) { registries, err := c.DescribeRegistries(context.TODO(), (&ecrpublic.DescribeRegistriesInput{})) @@ -194,7 +144,7 @@ func (c *ecrPublicClient) GetRegistryURI() (string, error) { if len(registries.Registries) > 0 && registries.Registries[0].RegistryUri != nil && *registries.Registries[0].RegistryUri != "" { return *registries.Registries[0].RegistryUri, nil } - return "", fmt.Errorf("Emtpy list of registries for the account") + return "", fmt.Errorf("empty list of registries for the account") } // GetPublicAuthToken gets an authorization token from ECR public @@ -207,48 +157,3 @@ func (c *ecrPublicClient) GetPublicAuthToken() (string, error) { return authToken, nil } - -// getNameAndVersionPublic looks up the latest pushed helm chart's tag from a given repo name Full name in Public ECR OCI format. -func (c *SDKClients) getNameAndVersionPublic(repoName, tag, registryURI string) (string, string, string, error) { - var version string - var sha string - splitname := strings.Split(repoName, ":") // TODO add a regex filter - name := splitname[0] - ecrname := fmt.Sprintf("%s/%s", c.ecrPublicClient.SourceRegistry, name) - if len(splitname) > 0 { - if !strings.HasSuffix(tag, "latest") { - imageIDs := []ecrpublictypes.ImageIdentifier{{ImageTag: &tag}} - ImageDetails, err := c.ecrPublicClient.DescribePublic(&ecrpublic.DescribeImagesInput{ - RepositoryName: aws.String(repoName), - ImageIds: imageIDs, - }) - if err != nil { - return "", "", "", fmt.Errorf("DescribeImagesRequest to public ECR failed: %w", err) - } - if len(ImageDetails) == 1 { - return ecrname, tag, *ImageDetails[0].ImageDigest, nil - } - } - ImageDetails, err := c.ecrPublicClient.DescribePublic(&ecrpublic.DescribeImagesInput{ - RepositoryName: aws.String(repoName), - }) - if err != nil { - return "", "", "", err - } - var images []ImageDetailsBothECR - for _, image := range ImageDetails { - details, err := createECRImageDetails(ImageDetailsECR{PublicImageDetails: image}) - if err != nil { - return "", "", "", err - } - images = append(images, details) - } - version, sha, err = getLatestHelmTagandSha(images) - if err != nil { - return "", "", "", err - } - ecrname := fmt.Sprintf("%s/%s", c.ecrPublicClient.SourceRegistry, name) - return ecrname, version, sha, err - } - return "", "", "", fmt.Errorf("invalid repository: %q", repoName) -} diff --git a/generatebundlefile/ecr_test.go b/generatebundlefile/ecr_test.go index 9dc3c0c1a..bcc8aeb86 100644 --- a/generatebundlefile/ecr_test.go +++ b/generatebundlefile/ecr_test.go @@ -10,8 +10,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ecr" ecrtypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/aws/aws-sdk-go-v2/service/ecrpublic" - ecrpublictypes "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" ) func TestSplitECRName(t *testing.T) { @@ -86,7 +84,7 @@ func TestUnTarHelmChart(t *testing.T) { t.Fatal("Error creating test dir:", err) } defer os.RemoveAll("hello-eks-anywhere") - f, err := os.Create("hello-eks-anywhere/Chart.yaml") + f, _ := os.Create("hello-eks-anywhere/Chart.yaml") content := []byte("apiVersion: v2\nversion: 0.1.0\nappVersion: 0.1.0\nname: hello-eks-anywhere\n") err = os.WriteFile("hello-eks-anywhere/Chart.yaml", content, 0o644) if err != nil { @@ -141,54 +139,6 @@ func TestUnTarHelmChart(t *testing.T) { } } -func TestShaExistsInRepository(t *testing.T) { - client := newMockPublicRegistryClient(nil) - tests := []struct { - client *mockPublicRegistryClient - testName string - testRepository string - testVersion string - checkPass bool - wantErr bool - }{ - { - testName: "Test empty Repository", - testRepository: "", - testVersion: "sha256:0526725a65691944e831add6b247b25a93b8eeb1033dddadeaa089e95b021172", - wantErr: true, - }, - { - testName: "Test empty Version", - testRepository: "hello-eks-anywhere", - testVersion: "", - wantErr: true, - }, - { - testName: "Test valid Repository and Version", - testRepository: "hello-eks-anywhere", - testVersion: "sha256:0526725a65691944e831add6b247b25a93b8eeb1033dddadeaa089e95b021172", - checkPass: true, - wantErr: false, - }, - } - for _, tc := range tests { - t.Run(tc.testName, func(tt *testing.T) { - clients := &SDKClients{ - ecrPublicClient: &ecrPublicClient{ - publicRegistryClient: client, - }, - } - got, err := clients.ecrPublicClient.shaExistsInRepository(tc.testRepository, tc.testVersion) - if (err != nil) != tc.wantErr { - tt.Fatalf("shaExistsInRepositoryPublic() error = %v, wantErr %v", err, tc.wantErr) - } - if got != tc.checkPass { - tt.Fatalf("shaExistsInRepositoryPublic() = %#v\n\n\n, want %#v", got, tc.checkPass) - } - }) - } -} - func TestTagFromSha(t *testing.T) { client := newMockRegistryClient(nil) tests := []struct { @@ -250,42 +200,6 @@ func TestTagFromSha(t *testing.T) { } } -// Helper funcions -// to create the mocks "impl 'r *mockRegistryClient' registryClient" - -type mockPublicRegistryClient struct { - err error -} - -func newMockPublicRegistryClient(err error) *mockPublicRegistryClient { - return &mockPublicRegistryClient{ - err: err, - } -} - -var testSha string = "sha256:0526725a65691944e831add6b247b25a93b8eeb1033dddadeaa089e95b021172" - -func (r *mockPublicRegistryClient) DescribeImages(ctx context.Context, params *ecrpublic.DescribeImagesInput, optFns ...func(*ecrpublic.Options)) (*ecrpublic.DescribeImagesOutput, error) { - if r.err != nil { - return nil, r.err - } - return &ecrpublic.DescribeImagesOutput{ - ImageDetails: []ecrpublictypes.ImageDetail{ - { - ImageDigest: &testSha, - }, - }, - }, nil -} - -func (r *mockPublicRegistryClient) DescribeRegistries(ctx context.Context, params *ecrpublic.DescribeRegistriesInput, optFns ...func(*ecrpublic.Options)) (*ecrpublic.DescribeRegistriesOutput, error) { - panic("not implemented") // TODO: Implement -} - -func (r *mockPublicRegistryClient) GetAuthorizationToken(ctx context.Context, params *ecrpublic.GetAuthorizationTokenInput, optFns ...func(*ecrpublic.Options)) (*ecrpublic.GetAuthorizationTokenOutput, error) { - panic("not implemented") // TODO: Implement -} - // ECR type mockRegistryClient struct { diff --git a/generatebundlefile/go.mod b/generatebundlefile/go.mod index fd029409b..2226b8a0a 100644 --- a/generatebundlefile/go.mod +++ b/generatebundlefile/go.mod @@ -7,7 +7,6 @@ replace github.com/aws/eks-anywhere-packages => ../ require ( github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.39 - github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.22.1 github.com/aws/aws-sdk-go-v2/service/ecr v1.19.5 github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.3 github.com/aws/aws-sdk-go-v2/service/kms v1.17.1 @@ -16,7 +15,6 @@ require ( github.com/go-logr/logr v1.3.0 github.com/jinzhu/copier v0.3.5 github.com/pkg/errors v0.9.1 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 gopkg.in/yaml.v2 v2.4.0 helm.sh/helm/v3 v3.14.3 k8s.io/apimachinery v0.29.0 @@ -154,6 +152,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.6.0 // indirect diff --git a/generatebundlefile/go.sum b/generatebundlefile/go.sum index a437e17c2..9d7535633 100644 --- a/generatebundlefile/go.sum +++ b/generatebundlefile/go.sum @@ -33,7 +33,6 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ= @@ -44,18 +43,14 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8D github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.22.1 h1:X5wW/v/Lk++7ZKIee9wCs1uT4ENIJyuancNe5hHY7rg= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.22.1/go.mod h1:HL0nIKWDYBdEhBDS4bldwL3NWi2++f8l1n+mQnTrr9Q= github.com/aws/aws-sdk-go-v2/service/ecr v1.19.5 h1:hg2/a7rE9dwYr+/DPNzHQ+IsHXLNt1NsQVUecBtA8os= github.com/aws/aws-sdk-go-v2/service/ecr v1.19.5/go.mod h1:pGwmNL8hN0jpBfKfTbmu+Rl0bJkDhaGl+9PQLrZ4KLo= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.13.3 h1:2XpcXse156FZfnvnrzqTb8uwJuWUcT1ryiU7dZOzBYc= @@ -71,7 +66,6 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3 github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/generatebundlefile/hack/common.sh b/generatebundlefile/hack/common.sh index 7eb080c08..f56791028 100755 --- a/generatebundlefile/hack/common.sh +++ b/generatebundlefile/hack/common.sh @@ -51,13 +51,6 @@ function generate () { --output "output-${version}" } -function regionCheck () { - local version=$1 - cd "${BASE_DIRECTORY}/generatebundlefile" - ./bin/generatebundlefile --bundle "output-${version}/bundle.yaml" \ - --region-check true || true -} - function push () { local version=${1?:no version specified} local stage=${2?:no version specified} diff --git a/generatebundlefile/hack/promote.sh b/generatebundlefile/hack/promote.sh deleted file mode 100755 index 234cb64e9..000000000 --- a/generatebundlefile/hack/promote.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -e -set -x -set -o pipefail - -export LANG=C.UTF-8 - -BASE_DIRECTORY=$(git rev-parse --show-toplevel) -HELM_REPO=${HELM_REPO} -echo ${HELM_REPO} - -make build -chmod +x ${BASE_DIRECTORY}/generatebundlefile/bin - -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -aws ecr get-login-password --region us-west-2 | HELM_EXPERIMENTAL_OCI=1 helm registry login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com -aws ecr-public get-login-password --region us-east-1 | HELM_EXPERIMENTAL_OCI=1 helm registry login --username AWS --password-stdin public.ecr.aws - -cd "${BASE_DIRECTORY}/generatebundlefile" - -KMS_KEY="arn:aws:kms:us-west-2:857151390494:alias/signingPackagesKey" - -if [[ -n "${PROMOTE_FILE}" ]]; then - if [[ "${HELM_REPO}" == "eks-anywhere-packages" ]]; then - # We need to copy the images for the packages helm chart, but no other packages. - ./bin/generatebundlefile \ - --promote ${HELM_REPO} --input ${PROMOTE_FILE} --copy-images --key ${KMS_KEY} - else - ./bin/generatebundlefile \ - --promote ${HELM_REPO} --input ${PROMOTE_FILE} --key ${KMS_KEY} - fi -else - ./bin/generatebundlefile \ - --promote ${HELM_REPO} --key ${KMS_KEY} - -fi diff --git a/generatebundlefile/helm.go b/generatebundlefile/helm.go index 9d8bfc138..05587c01d 100644 --- a/generatebundlefile/helm.go +++ b/generatebundlefile/helm.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -71,7 +70,7 @@ func helmLog(log logr.Logger) action.DebugLog { // UnTarHelmChart will attempt to move the helm chart out of the helm cache, by untaring it to the pwd and creating the filesystem to unpack it into. func UnTarHelmChart(chartRef, chartPath, dest string) error { if chartRef == "" || chartPath == "" || dest == "" { - return fmt.Errorf("Empty input value given for UnTarHelmChart") + return fmt.Errorf("empty input value given for UnTarHelmChart") } _, err := os.Stat(dest) if os.IsNotExist(err) { @@ -91,7 +90,7 @@ func UnTarHelmChart(chartRef, chartPath, dest string) error { return chartutil.ExpandFile(dest, chartRef) } -// hasRequires checks for the existance of the requires.yaml within the helm directory +// hasRequires checks for the existence of the requires.yaml within the helm directory func hasRequires(helmdir string) (string, error) { requires := filepath.Join(helmdir, "requires.yaml") info, err := os.Stat(requires) @@ -99,7 +98,7 @@ func hasRequires(helmdir string) (string, error) { return "", err } if info.IsDir() { - return "", fmt.Errorf("Found Dir, not requires.yaml file") + return "", fmt.Errorf("found Dir, not requires.yaml file") } return requires, nil } @@ -144,14 +143,14 @@ func validateHelmRequiresName(helmrequires *Requires) error { func (helmrequires *Requires) validateHelmRequiresNotEmpty() error { // Check if Projects are listed if len(helmrequires.Spec.Images) < 1 { - return fmt.Errorf("Should use non-empty list of images for requires") + return fmt.Errorf("should use non-empty list of images for requires") } return nil } // parseHelmRequires will attempt to unpack the requires.yaml into the Go struct `Requires` func parseHelmRequires(fileName string, helmrequires *Requires) error { - content, err := ioutil.ReadFile(fileName) + content, err := os.ReadFile(fileName) if err != nil { return fmt.Errorf("unable to read file due to: %v", err) } diff --git a/generatebundlefile/input.go b/generatebundlefile/input.go index f0e9a093c..526165737 100644 --- a/generatebundlefile/input.go +++ b/generatebundlefile/input.go @@ -3,7 +3,7 @@ package main import ( "bytes" "fmt" - "io/ioutil" + "os" "path/filepath" "strings" "time" @@ -55,7 +55,7 @@ func validateInputConfigName(inputConfig *Input) error { func (inputConfig *Input) ValidateInputNotEmpty() error { // Check if Projects are listed if len(inputConfig.Packages) < 1 { - return fmt.Errorf("Should use non-empty list of projects for input") + return fmt.Errorf("should use non-empty list of projects for input") } return nil } @@ -96,13 +96,13 @@ func validateBundleName(bundle *api.PackageBundle) error { func ValidateBundleNoSignature(bundle *api.PackageBundle) error { // Check if Projects are listed if len(bundle.Spec.Packages) < 1 { - return fmt.Errorf("Should use non-empty list of projects for input") + return fmt.Errorf("should use non-empty list of projects for input") } return nil } func ParseBundle(filename string) (*api.PackageBundle, error) { - content, err := ioutil.ReadFile(filepath.Clean(filename)) + content, err := os.ReadFile(filepath.Clean(filename)) if err != nil { return nil, fmt.Errorf("reading package bundle file %q: %w", filename, err) } @@ -121,7 +121,7 @@ func ParseBundle(filename string) (*api.PackageBundle, error) { } func ParseInputConfig(fileName string, inputConfig *Input) error { - content, err := ioutil.ReadFile(filepath.Clean(fileName)) + content, err := os.ReadFile(filepath.Clean(fileName)) if err != nil { return fmt.Errorf("unable to read file due to: %v", err) } @@ -138,10 +138,10 @@ func ParseInputConfig(fileName string, inputConfig *Input) error { return fmt.Errorf("cluster spec file %s is invalid or does not contain kind %v", fileName, inputConfig) } -func (c *SDKClients) NewBundleFromInput(Input *Input) (api.PackageBundleSpec, string, map[string]bool, error) { +func (c *SDKClients) NewBundleFromInput(Input *Input) (api.PackageBundleSpec, string, error) { packageBundleSpec := api.PackageBundleSpec{} if Input.Name == "" || Input.KubernetesVersion == "" { - return packageBundleSpec, "", nil, fmt.Errorf("error: Empty input field from `Name` or `KubernetesVersion`.") + return packageBundleSpec, "", fmt.Errorf("empty input field from `Name` or `KubernetesVersion`") } currentTime := time.Now() @@ -153,17 +153,15 @@ func (c *SDKClients) NewBundleFromInput(Input *Input) (api.PackageBundleSpec, st if Input.MinVersion != "" { packageBundleSpec.MinVersion = Input.MinVersion } - copyImages := map[string]bool{} for _, org := range Input.Packages { for _, project := range org.Projects { - copyImages[project.Repository] = project.CopyImages bundlePkg, err := c.NewPackageFromInput(project) if err != nil { BundleLog.Error(err, "Unable to complete NewBundleFromInput from ecr lookup failure") - return packageBundleSpec, "", nil, err + return packageBundleSpec, "", err } packageBundleSpec.Packages = append(packageBundleSpec.Packages, *bundlePkg) } } - return packageBundleSpec, name, copyImages, nil + return packageBundleSpec, name, nil } diff --git a/generatebundlefile/main.go b/generatebundlefile/main.go index 18d58623e..ea1e24cb4 100644 --- a/generatebundlefile/main.go +++ b/generatebundlefile/main.go @@ -6,11 +6,8 @@ import ( "io" "os" "path/filepath" - "strings" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecrpublic" ctrl "sigs.k8s.io/controller-runtime" @@ -21,8 +18,6 @@ import ( var BundleLog = ctrl.Log.WithName("BundleGenerator") -var Profile = "default" - func main() { opts := NewOptions() opts.SetupLogger() @@ -46,22 +41,7 @@ func main() { return } - err := cmdPromote(opts) - if err != nil { - BundleLog.Error(err, "promoting curated package") - os.Exit(1) - } - - if opts.regionCheck { - err := cmdRegion(opts) - if err != nil { - BundleLog.Error(err, "checking bundle across region") - os.Exit(1) - } - return - } - - err = cmdGenerate(opts) + err := cmdGenerate(opts) if err != nil { BundleLog.Error(err, "generating bundle") os.Exit(1) @@ -84,216 +64,6 @@ func cmdGenerateSample(w io.Writer) error { return nil } -func cmdPromote(opts *Options) error { - BundleLog.Info("Starting Promote from private ECR to Public ECR....") - - promoteCharts := make(map[string][]string) - - if opts.tag == "" { - opts.tag = "latest" - } - - // If we are promoting an individual chart with the --promote flag like we do for most charts. - if opts.promote != "" { - promoteCharts[opts.promote] = append(promoteCharts[opts.promote], opts.tag) - } - - // If we are promoting multiple chart with the --input file flag we override the struct with files inputs from the file. - if opts.inputFile != "" { - packages, err := opts.ValidateInput() - if err != nil { - return err - } - for _, f := range packages { - Inputs, err := ValidateInputConfig(f) - if err != nil { - BundleLog.Error(err, "Unable to validate input file") - os.Exit(1) - } - delete(promoteCharts, opts.promote) // Clear the promote map to pull values only from file - for _, p := range Inputs.Packages { - for _, project := range p.Projects { - for _, version := range project.Versions { - promoteCharts[project.Repository] = append(promoteCharts[project.Repository], version.Name) - } - } - } - } - } - - clients, err := GetSDKClients() - if err != nil { - return fmt.Errorf("getting SDK clients: %w", err) - } - - dockerStruct := &DockerAuth{ - Auths: map[string]DockerAuthRegistry{ - fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", clients.stsClient.AccountID, ecrRegion): {clients.ecrClient.AuthConfig}, - }, - } - - Profile := "default" - val, ok := os.LookupEnv("AWS_PROFILE") - if ok { - Profile = val - } - if Profile != "default" { - clients, err = clients.GetProfileSDKConnection("ecrpublic", Profile, ecrPublicRegion) - if err != nil { - BundleLog.Error(err, "Unable create SDK Client connections") - os.Exit(1) - } - - clients.ecrPublicClientRelease.SourceRegistry, err = clients.ecrPublicClientRelease.GetRegistryURI() - if err != nil { - BundleLog.Error(err, "Unable create find Public ECR URI for destination account") - os.Exit(1) - } - dockerStruct.Auths["public.ecr.aws"] = DockerAuthRegistry{clients.ecrPublicClientRelease.AuthConfig} - } - - clients.ecrPublicClient.SourceRegistry, err = clients.ecrPublicClient.GetRegistryURI() - if err != nil { - return fmt.Errorf("getting registry URI: %w", err) - } - dockerStruct.Auths["public.ecr.aws"] = DockerAuthRegistry{clients.ecrPublicClient.AuthConfig} - - dockerAuth, err := NewAuthFile(dockerStruct) - if err != nil { - return fmt.Errorf("creating auth file: %w", err) - } - if dockerAuth.Authfile == "" { - return fmt.Errorf("no authfile generated") - } - clients.helmDriver, err = NewHelm(BundleLog, dockerAuth.Authfile) - if err != nil { - BundleLog.Error(err, "Unable to create Helm driver") - os.Exit(1) - } - for repoName, versions := range promoteCharts { - for _, version := range versions { - err = clients.PromoteHelmChart(repoName, dockerAuth.Authfile, version, opts.copyImages) - if err != nil { - return fmt.Errorf("promoting Helm chart: %w", err) - } - } - } - err = dockerAuth.Remove() - if err != nil { - return fmt.Errorf("cleaning up docker auth file: %w", err) - } - BundleLog.Info("Promote Finished, exiting gracefully") - return nil -} - -func cmdRegion(opts *Options) error { - BundleLog.Info("Starting Region Check Process") - if opts.bundleFile == "" { - BundleLog.Info("Please use the --bundle flag when running region check") - os.Exit(1) - } - Bundle, err := ValidateBundle(opts.bundleFile) - if err != nil { - BundleLog.Error(err, "Unable to validate input file") - os.Exit(1) - } - - d := &RepositoryCloudWatch{} - k8sVersionSlice := strings.Split(Bundle.ObjectMeta.Name, "-") - K8sVersion := fmt.Sprintf("%s-%s", k8sVersionSlice[0], k8sVersionSlice[1]) - - cloudWatchDataStruct := []RepositoryCloudWatch{} - - BundleLog.Info("Getting list of images to Region Check") - for _, packages := range Bundle.Spec.Packages { - for _, versions := range packages.Source.Versions { - for _, images := range versions.Images { - d = &RepositoryCloudWatch{ - Repository: images.Repository, - Digest: images.Digest, - TotalHits: 0, - K8sVersion: K8sVersion, - } - cloudWatchDataStruct = append(cloudWatchDataStruct, *d) - } - } - } - - // Deduplicate the Array of structs for the CRDs packages which contain the same image reference twice. - m := map[RepositoryCloudWatch]struct{}{} - uniquecloudWatchDataStruct := []RepositoryCloudWatch{} - for _, d := range cloudWatchDataStruct { - if _, ok := m[d]; !ok { - uniquecloudWatchDataStruct = append(uniquecloudWatchDataStruct, d) - m[d] = struct{}{} - } - } - - // Creating AWS Clients with profile - Profile := "default" - val, ok := os.LookupEnv("AWS_PROFILE") - if ok { - Profile = val - } - BundleLog.Info("Using Env", "AWS_PROFILE", Profile) - BundleLog.Info("Creating SDK connections to Region Check") - clients := &SDKClients{} - conf, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(ecrRegion), - config.WithSharedConfigProfile(Profile), - ) - if err != nil { - return fmt.Errorf("loading default AWS config: %w", err) - } - cloudwatchC := cloudwatch.NewFromConfig(conf) - clients, err = clients.GetProfileSDKConnection("ecr", Profile, ecrRegion) - if err != nil { - BundleLog.Error(err, "getting SDK connection") - os.Exit(1) - } - clients, err = clients.GetProfileSDKConnection("sts", Profile, ecrRegion) - if err != nil { - BundleLog.Error(err, "getting profile SDK connection") - os.Exit(1) - } - - var cloudwatchData []cloudwatchtypes.MetricDatum - var missingList []string - for _, region := range RegionList { - BundleLog.Info("Starting Check for", "Region", region) - clients, err = clients.GetProfileSDKConnection("ecr", Profile, region) - if err != nil { - BundleLog.Error(err, "getting ECR SDK connection") - os.Exit(1) - } - for i, image := range uniquecloudWatchDataStruct { - check, err := clients.ecrClientRelease.shaExistsInRepository(image.Repository, image.Digest) - if err != nil { - BundleLog.Error(err, "finding ECR images") - } - if check { - uniquecloudWatchDataStruct[i].TotalHits++ - } else { - missingList = append(missingList, fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s", clients.stsClientRelease.AccountID, region, image.Repository, image.Digest)) - } - } - } - BundleLog.Info("Missing Region List:") - printSlice(missingList) - for i, image := range uniquecloudWatchDataStruct { - percent := (float64(image.TotalHits) / float64(len(RegionList))) * 100 - uniquecloudWatchDataStruct[i].Percent = percent - cloudwatchData = FormCloudWatchData(cloudwatchData, image.Repository, uniquecloudWatchDataStruct[i].Percent) - } - err = PushCloudWatchRegionCheckData(cloudwatchC, cloudwatchData, uniquecloudWatchDataStruct[0].K8sVersion) - if err != nil { - BundleLog.Error(err, "pushing cloudwatch data") - os.Exit(1) - } - BundleLog.Info("Finished Region Check!") - return nil -} - func cmdGenerate(opts *Options) error { // grab local path to caller, and make new caller pwd, err := os.Getwd() @@ -374,7 +144,7 @@ func cmdGenerate(opts *Options) error { } BundleLog.Info("In Progress: Populating Bundles and looking up Sha256 tags") - addOnBundleSpec, name, copyImages, err := clients.NewBundleFromInput(Inputs) + addOnBundleSpec, name, err := clients.NewBundleFromInput(Inputs) if err != nil { BundleLog.Error(err, "Unable to create bundle from input file") os.Exit(1) @@ -440,130 +210,6 @@ func cmdGenerate(opts *Options) error { } bundle := AddMetadata(addOnBundleSpec, name) - // We will make a compound check for public and private profile after the launch once we want to stop - // push packages to private ECR. - if opts.publicProfile != "" { - BundleLog.Info("Starting release public ECR process....") - clients, err := GetSDKClients() - if err != nil { - BundleLog.Error(err, "getting sdk clients") - os.Exit(1) - } - - clients, err = clients.GetProfileSDKConnection("ecrpublic", opts.publicProfile, ecrPublicRegion) - if err != nil { - BundleLog.Error(err, "Unable create SDK Client connections") - os.Exit(1) - } - - clients.ecrPublicClient.SourceRegistry, err = clients.ecrPublicClient.GetRegistryURI() - if err != nil { - BundleLog.Error(err, "Unable create find Public ECR URI from calling account") - os.Exit(1) - } - clients.ecrPublicClientRelease.SourceRegistry, err = clients.ecrPublicClientRelease.GetRegistryURI() - if err != nil { - BundleLog.Error(err, "Unable create find Public ECR URI for destination account") - os.Exit(1) - } - dockerReleaseStruct = &DockerAuth{ - Auths: map[string]DockerAuthRegistry{ - fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", clients.stsClient.AccountID, ecrRegion): { - clients.ecrClient.AuthConfig, - }, - fmt.Sprintf("public.ecr.aws/%s", clients.ecrPublicClient.SourceRegistry): { - clients.ecrPublicClient.AuthConfig, - }, - "public.ecr.aws": {clients.ecrPublicClientRelease.AuthConfig}, - }, - } - dockerAuth, err = NewAuthFile(dockerReleaseStruct) - if err != nil || dockerAuth.Authfile == "" { - BundleLog.Error(err, "Unable create AuthFile") - os.Exit(1) - } - clients.helmDriver, err = NewHelm(BundleLog, dockerAuth.Authfile) - if err != nil { - BundleLog.Error(err, "Unable to create Helm driver") - os.Exit(1) - } - for _, charts := range addOnBundleSpec.Packages { - for _, versions := range charts.Source.Versions { - err = clients.PromoteHelmChart(charts.Source.Repository, dockerAuth.Authfile, versions.Name, copyImages[charts.Source.Repository]) - if err != nil { - BundleLog.Error(err, "promoting helm chart", - "name", charts.Source.Repository) - os.Exit(1) - } - } - } - err = dockerAuth.Remove() - if err != nil { - BundleLog.Error(err, "removing docker auth file") - os.Exit(1) - } - - return nil - } - - // See above comment about compound check when we want to cutover - // if o.publicProfile != "" && if o.privateProfile != "" {} - if opts.privateProfile != "" { - BundleLog.Info("Starting release to private ECR process....") - clients, err := GetSDKClients() - if err != nil { - BundleLog.Error(err, "getting SDK clients") - os.Exit(1) - } - - clients, err = clients.GetProfileSDKConnection("ecr", opts.privateProfile, ecrRegion) - if err != nil { - BundleLog.Error(err, "getting SDK connection") - os.Exit(1) - } - clients, err = clients.GetProfileSDKConnection("sts", opts.privateProfile, ecrRegion) - if err != nil { - BundleLog.Error(err, "getting profile SDK connection") - os.Exit(1) - } - dockerReleaseStruct = &DockerAuth{ - Auths: map[string]DockerAuthRegistry{ - fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", clients.stsClient.AccountID, ecrRegion): { - clients.ecrClient.AuthConfig, - }, - fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", clients.stsClientRelease.AccountID, ecrRegion): { - clients.ecrClientRelease.AuthConfig, - }, - }, - } - dockerAuth, err = NewAuthFile(dockerReleaseStruct) - if err != nil { - BundleLog.Error(err, "Unable create AuthFile") - os.Exit(1) - } - clients.helmDriver, err = NewHelm(BundleLog, dockerAuth.Authfile) - if err != nil { - BundleLog.Error(err, "Unable to create Helm driver") - os.Exit(1) - } - for _, charts := range addOnBundleSpec.Packages { - for _, versions := range charts.Source.Versions { - err = clients.PromoteHelmChart(charts.Source.Repository, dockerAuth.Authfile, versions.Name, true) - if err != nil { - BundleLog.Error(err, "promoting helm chart", - "name", charts.Source.Repository) - os.Exit(1) - } - } - } - err = dockerAuth.Remove() - if err != nil { - BundleLog.Error(err, "removing docker auth file") - os.Exit(1) - } - return nil - } - bundle.Annotations[FullExcludesAnnotation] = Excludes BundleLog.Info("Generating bundle signature", "key", opts.key) signature, err := GetBundleSignature(context.Background(), bundle, opts.key) diff --git a/generatebundlefile/options.go b/generatebundlefile/options.go index 77bc99d8c..daf9d77d2 100644 --- a/generatebundlefile/options.go +++ b/generatebundlefile/options.go @@ -13,17 +13,11 @@ import ( // Options represents the flag for the current plugin type Options struct { - inputFile string - outputFolder string - generateSample bool - promote string - tag string - copyImages bool - key string - publicProfile string - privateProfile string - bundleFile string - regionCheck bool + inputFile string + outputFolder string + generateSample bool + key string + bundleFile string } func (o *Options) SetupLogger() { @@ -78,14 +72,8 @@ func NewOptions() *Options { fs.BoolVar(&o.generateSample, "generate-sample", false, "Whether you want to generate a sample bundle for yourself") fs.StringVar(&o.inputFile, "input", "", "The path where the input bundle generation file lives") fs.StringVar(&o.outputFolder, "output", "output", "The path where to write the output bundle files") - fs.StringVar(&o.promote, "promote", "", "The Helm chart private ECR OCI uri to pull and promote") - fs.StringVar(&o.tag, "tag", "", "The tag of the helm chart used with promote flag") - fs.BoolVar(&o.copyImages, "copy-images", false, "Whether to copy images during a helm promotion, the default is to Not copy.") fs.StringVar(&o.key, "key", "k", "The key to sign with") - fs.StringVar(&o.publicProfile, "public-profile", "", "The AWS Public Profile to release the prod bundle into") - fs.StringVar(&o.privateProfile, "private-profile", "", "The AWS Private Profile to release all packages into") fs.StringVar(&o.bundleFile, "bundle", "", "The path where the bundle file lives") - fs.BoolVar(&o.regionCheck, "region-check", false, "Check the passed in bundle resource and if they exist in the supported ECR regions") err := fs.Parse(os.Args[1:]) if err != nil { BundleLog.Error(err, "Error parsing input flags") diff --git a/generatebundlefile/promote.go b/generatebundlefile/promote.go index 452151bcc..73ce92fb5 100644 --- a/generatebundlefile/promote.go +++ b/generatebundlefile/promote.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" - "strings" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ecr" @@ -20,7 +18,6 @@ type SDKClients struct { ecrClientRelease *ecrClient ecrPublicClientRelease *ecrPublicClient stsClientRelease *stsClient - helmDriver *helmDriver } // GetSDKClients is used to handle the creation of different SDK clients. @@ -38,13 +35,13 @@ func GetSDKClients() (*SDKClients, error) { stsclient := sts.NewFromConfig(conf) clients.stsClient, err = NewStsClient(stsclient, true) if err != nil { - return nil, fmt.Errorf("Unable to create SDK connection to STS %s", err) + return nil, fmt.Errorf("creating SDK connection to STS %s", err) } ecrClient := ecr.NewFromConfig(conf) clients.ecrClient, err = NewECRClient(ecrClient, true) if err != nil { - return nil, fmt.Errorf("Unable to create SDK connection to ECR %s", err) + return nil, fmt.Errorf("creating SDK connection to ECR %s", err) } clients.stsClientRelease = clients.stsClient @@ -80,242 +77,23 @@ func (c *SDKClients) GetProfileSDKConnection(service, profile, region string) (* clientWithProfile := ecrpublic.NewFromConfig(confWithProfile) c.ecrPublicClientRelease, err = NewECRPublicClient(clientWithProfile, true) if err != nil { - return nil, fmt.Errorf("Unable to create SDK connection to ECR Public using another profile %s", err) + return nil, fmt.Errorf("creating SDK connection to ECR Public using another profile %s", err) } return c, nil case "ecr": clientWithProfile := ecr.NewFromConfig(confWithProfile) c.ecrClientRelease, err = NewECRClient(clientWithProfile, true) if err != nil { - return nil, fmt.Errorf("Unable to create SDK connection to ECR using another profile %s", err) + return nil, fmt.Errorf("creating SDK connection to ECR using another profile %s", err) } return c, nil case "sts": clientWithProfile := sts.NewFromConfig(confWithProfile) c.stsClientRelease, err = NewStsClient(clientWithProfile, true) if err != nil { - return nil, fmt.Errorf("Unable to create SDK connection to STS using another profile %s", err) + return nil, fmt.Errorf("creating SDK connection to STS using another profile %s", err) } return c, nil } return nil, fmt.Errorf("gave service not supported by GetProfileSDKConnection(), consider adding it to the switch case") } - -// PromoteHelmChart will take a given repository, and authFile and handle helm and image promotion for the mentioned chart. -func (c *SDKClients) PromoteHelmChart(repository, authFile, tag string, copyImages bool) error { - var name, version, sha string - pwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("Error getting pwd %s", err) - } - - // If we are not moving artifacts to the Private ECR Packages artifact account, get information from public ECR instead. - // The first scenario runs for flags - // --private-profile - // --promote. - // The 2nd scenario runs for flags - // --public-profile. - if Profile != "default" { - name, version, sha, err = c.getNameAndVersionPublic(repository, tag, c.stsClientRelease.AccountID) - if err != nil { - return fmt.Errorf("Error getting name and version from helm chart: %s", err) - } - } else { - name, version, sha, err = c.getNameAndVersion(repository, tag, c.stsClient.AccountID) - if err != nil { - return fmt.Errorf("Error getting name and version from helm chart: %s", err) - } - } - - // Pull the Helm chart to Helm Cache - BundleLog.Info("Found Helm Chart to read requires.yaml for image information", "Chart", fmt.Sprintf("%s:%s", name, version)) - semVer := strings.Replace(version, "_", "+", 1) // TODO use the Semvar library instead of this hack. - chartPath, err := c.helmDriver.PullHelmChart(name, semVer) - if err != nil { - return fmt.Errorf("Error pulling helmchart %s", err) - } - // Get the correct Repo Name from the flag based on ECR repo name formatting - // since we prepend the github org on some repo's, and not on others. - chartName, helmname, err := splitECRName(name) - if err != nil { - return fmt.Errorf("Failed splitECRName %s", err) - } - // Untar the helm .tgz to pwd and name the folder to the helm chart Name - dest := filepath.Join(pwd, chartName) - err = UnTarHelmChart(chartPath, chartName, dest) - if err != nil { - return fmt.Errorf("failed pulling helm release %s", err) - } - // Check for requires.yaml in the unpacked helm chart - helmDest := filepath.Join(pwd, chartName, helmname) - defer os.RemoveAll(helmDest) - f, err := hasRequires(helmDest) - if err != nil { - return fmt.Errorf("Helm chart doesn't have requires.yaml inside %s", err) - } - // Unpack requires.yaml into a GO struct - helmRequiresImages, err := validateHelmRequires(f) - if err != nil { - return fmt.Errorf("Unable to parse requires.yaml file to Go Struct %s", err) - } - - // Create a 2nd struct since the the helm chart is going to Public ECR while the images are going to Private ECR. - helmRequires := &Requires{ - Spec: RequiresSpec{ - Images: []Image{ - { - Repository: chartName, - Tag: version, - Digest: sha, - }, - }, - }, - } - // Loop through each image, and the helm chart itself and check for existence in ECR Public, skip if we find the SHA already exists in destination. - // If we don't find the SHA in public, we lookup the tag from Private Dev account, and move to Private Artifact account. - // This runs for flags - // --private-profile - // --promote - // --copy-images - if copyImages { - accountID := c.activeAccountID() - for _, image := range helmRequiresImages.Spec.Images { - checkSha, checkTag, err := c.CheckDestinationECR(image, image.Tag) - // This is going to run a copy if only 1 check passes because there are scenarios where the correct SHA exists, but the tag is not in sync. - // Copy with the correct image SHA, but a new tag will just write a new tag to ECR so it's safe to run. - if checkSha && checkTag { - BundleLog.Info("Image Digest and Tag already exists in destination location, skipping image copy", "Destination", fmt.Sprintf("docker://%s.dkr.ecr.us-west-2.amazonaws.com/%s:%s@%s", accountID, image.Repository, image.Tag, image.Digest)) - continue - } else { - BundleLog.Info("Image Digest and Tag don't exist in destination location, copying images to destination", "Location", fmt.Sprintf("%s/%s@%s", image.Repository, image.Tag, image.Digest)) - // We have cases with tag mismatch where the SHA is accurate, but the tag in the destination repo is not synced, this will sync it. - if image.Tag == "" { - image.Tag, err = c.ecrClient.tagFromSha(image.Repository, image.Digest, tag) - } - if err != nil { - BundleLog.Error(err, "Unable to find Tag from Digest") - } - err = c.moveImage(image, authFile) - if err != nil { - return err - } - - continue - } - } - } - - // If we have the profile for the artifact account present, we skip moving helm charts since they go to the public ECR only. - // This will move the Helm chart from Private ECR to Public ECR if it doesn't exist. This goes to either dev or prod depending on the flags passed in. - // This runs for flags - // --public-profile - // --promote. - if c.ecrClientRelease == nil { - for _, image := range helmRequires.Spec.Images { - // Check if the Helm chart is going to Prod, or dev account. - destinationRegistry := c.ecrPublicClient.SourceRegistry - if c.ecrPublicClientRelease != nil { - destinationRegistry = c.ecrPublicClientRelease.SourceRegistry - } - checkSha, checkTag, err := c.CheckDestinationECR(image, image.Tag) - if checkSha && checkTag { - BundleLog.Info("Image Digest and Tag already exist in destination location, skipping", "Destination", fmt.Sprintf("docker://%s/%s:%s@%s", destinationRegistry, image.Repository, image.Tag, image.Digest)) - continue - } else { - if c.ecrPublicClientRelease == nil { - source := fmt.Sprintf("docker://%s.dkr.ecr.us-west-2.amazonaws.com/%s:%s", c.stsClient.AccountID, image.Repository, image.Tag) - destination := fmt.Sprintf("docker://%s/%s:%s", destinationRegistry, image.Repository, image.Tag) - BundleLog.Info("Copying Helm Digest and Tag to destination location", "Location", fmt.Sprintf("%s/%s:%s %s", c.ecrPublicClient.SourceRegistry, image.Repository, image.Tag, image.Digest)) - err = copyImage(BundleLog, authFile, source, destination) - if err != nil { - return fmt.Errorf("Unable to copy image from source to destination repo %s", err) - } - } else { - source := fmt.Sprintf("docker://%s/%s:%s", c.ecrPublicClient.SourceRegistry, image.Repository, image.Tag) - destination := fmt.Sprintf("docker://%s/%s:%s", destinationRegistry, image.Repository, image.Tag) - BundleLog.Info("Copying Helm Digest and Tag to destination location", "Location", fmt.Sprintf("%s/%s:%s %s", c.ecrPublicClient.SourceRegistry, image.Repository, image.Tag, image.Digest)) - err = copyImage(BundleLog, authFile, source, destination) - if err != nil { - return fmt.Errorf("Unable to copy image from source to destination repo %s", err) - } - } - } - } - } - return nil -} - -func (c *SDKClients) CheckDestinationECR(image Image, version string) (bool, bool, error) { - var checkSha, checkTag bool - var err error - var check CheckECR - - // Change the source destination check depending on release to dev or prod. - destination := c.ecrPublicClient - if c.ecrPublicClientRelease != nil { - destination = c.ecrPublicClientRelease - } - - // We either check in private ECR or public ECR depending on what's passed in. - if c.stsClientRelease != nil { - check = c.ecrClientRelease - } else { - check = destination - } - - check = c.ecrPublicClient - if c.ecrPublicClientRelease != nil { - check = c.ecrPublicClientRelease - } - - checkSha, err = check.shaExistsInRepository(image.Repository, image.Digest) - if err != nil { - return false, false, err - } - checkTag, err = check.tagExistsInRepository(image.Repository, version) - if err != nil { - return false, false, err - } - - return checkSha, checkTag, nil -} - -func (c *SDKClients) activeAccountID() string { - if c.stsClientRelease != nil && c.stsClient != nil { - return c.stsClient.AccountID - } - if c.stsClientRelease != nil { - return c.stsClientRelease.AccountID - } - if c.stsClient != nil { - return c.stsClient.AccountID - } - return "" -} - -func (c *SDKClients) moveImage(image Image, authFile string) error { - var destinationPrefix string - accountID := c.activeAccountID() - BundleLog.Info("Moving container images to ECR") - sourcePrefix := fmt.Sprintf("docker://%s.dkr.ecr.us-west-2.amazonaws.com/%s", accountID, image.Repository) - if c.ecrClientRelease != nil { - destinationPrefix = fmt.Sprintf("docker://%s.dkr.ecr.us-west-2.amazonaws.com/%s", c.stsClientRelease.AccountID, image.Repository) - } else if c.ecrPublicClientRelease != nil { - destinationPrefix = fmt.Sprintf("docker://%s/%s", c.ecrPublicClientRelease.SourceRegistry, image.Repository) - } else { - destinationPrefix = fmt.Sprintf("docker://%s/%s", c.ecrPublicClient.SourceRegistry, image.Repository) - } - - err := copyImage(BundleLog, authFile, sourcePrefix+":"+image.Tag, destinationPrefix+":"+image.Tag) - if err != nil { - return fmt.Errorf("unable to copy image tag from source to destination repo: %w", err) - } - - // when the image.Tag doesn't match image.Digest in the source ECR, we need to move the Digest directly - err = copyImage(BundleLog, authFile, sourcePrefix+"@"+image.Digest, destinationPrefix+"@"+image.Digest) - if err != nil { - return fmt.Errorf("unable to copy image digest from source to destination repo: %w", err) - } - - return nil -} diff --git a/generatebundlefile/region.go b/generatebundlefile/region.go deleted file mode 100644 index 63b2dd6d3..000000000 --- a/generatebundlefile/region.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" -) - -var RegionList = []string{ - "us-east-2", - "us-east-1", - "us-west-1", - "us-west-2", - "ap-northeast-3", - "ap-northeast-2", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ca-central-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-north-1", - "sa-east-1", -} - -type RepositoryCloudWatch struct { - Repository string `json:"repository,omitempty"` - Digest string `json:"digest,omitempty"` - Region string `json:"region,omitempty"` - TotalHits int `json:"totalhits,omitempty"` - Percent float64 `json:"percent,omitempty"` - K8sVersion string `json:"k8sversion,omitempty"` -} - -func FormCloudWatchData(metricData []cloudwatchtypes.MetricDatum, name string, value float64) []cloudwatchtypes.MetricDatum { - regionName := "region-availibity" - d := []cloudwatchtypes.Dimension{ - { - Name: ®ionName, - Value: &name, - }, - } - metricDataPoint := &cloudwatchtypes.MetricDatum{ - Dimensions: d, - MetricName: &name, - Value: &value, - } - metricData = append(metricData, *metricDataPoint) - return metricData -} - -func PushCloudWatchRegionCheckData(c *cloudwatch.Client, data []cloudwatchtypes.MetricDatum, k8s_version string) error { - Input := &cloudwatch.PutMetricDataInput{ - MetricData: data, - Namespace: &k8s_version, - } - err := pushMetricData(c, *Input) - if err != nil { - return err - } - return nil -} - -func pushMetricData(c *cloudwatch.Client, input cloudwatch.PutMetricDataInput) error { - _, err := c.PutMetricData(context.TODO(), &input) - if err != nil { - return fmt.Errorf("sending cloudwatch PutMetricData(): %w", err) - } - return nil -} diff --git a/generatebundlefile/sts.go b/generatebundlefile/sts.go index baaec1919..bbbccecb2 100644 --- a/generatebundlefile/sts.go +++ b/generatebundlefile/sts.go @@ -27,7 +27,7 @@ func NewStsClient(client stsClientInterface, account bool) (*stsClient, error) { stsClient.AccountID = *stslookup.Account return stsClient, nil } - return nil, fmt.Errorf("Empty Account ID from stslookup call") + return nil, fmt.Errorf("empty Account ID from stslookup call") } return stsClient, nil }