From c63528c6d2e03f9e698138b4c00cfdbd01cc7e7d Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 29 Aug 2023 16:20:41 +0200 Subject: [PATCH 01/29] Add sign-only mode. --- prow/jobs/kyma-project/test-infra/go-validation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prow/jobs/kyma-project/test-infra/go-validation.yaml b/prow/jobs/kyma-project/test-infra/go-validation.yaml index 320b13a89f55..4e289ac16c59 100644 --- a/prow/jobs/kyma-project/test-infra/go-validation.yaml +++ b/prow/jobs/kyma-project/test-infra/go-validation.yaml @@ -11,10 +11,11 @@ presubmits: # runs on PRs prow.k8s.io/pubsub.project: "sap-kyma-prow" prow.k8s.io/pubsub.runID: "pull-unit-test-go" prow.k8s.io/pubsub.topic: "prowjobs" + preset-bot-github-token: "true" run_if_changed: '.*\.go$' skip_report: false decorate: true - cluster: untrusted-workload + cluster: trusted-workload max_concurrency: 10 branches: - ^main$ @@ -69,4 +70,3 @@ presubmits: # runs on PRs requests: memory: 1Gi cpu: 2000m - \ No newline at end of file From 16e5bd86b8d20e072395072339da65aca79ebeca Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 28 Sep 2023 14:23:02 +0200 Subject: [PATCH 02/29] POC of image building in ADO. --- cmd/image-builder/config.go | 14 ++++- cmd/image-builder/config_test.go | 10 +-- cmd/image-builder/main.go | 1 + configs/kaniko-build-config-ado.yaml | 43 +++++++++++++ go.mod | 3 + .../test-infra/image-builder-ado-example.yaml | 62 +++++++++++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 configs/kaniko-build-config-ado.yaml create mode 100644 prow/jobs/test-infra/image-builder-ado-example.yaml diff --git a/cmd/image-builder/config.go b/cmd/image-builder/config.go index 2a9d4b7ded44..4ca7bca017dc 100644 --- a/cmd/image-builder/config.go +++ b/cmd/image-builder/config.go @@ -31,6 +31,14 @@ type Config struct { // SignConfig contains custom configuration of signers // as well as org/repo mapping of enabled signers in specific repository SignConfig SignConfig `yaml:"sign-config" json:"sign-config"` + // ADO organization URL to call for triggering ADO pipeline + ADOOrganizationURL string `yaml:"ado-organization-url" json:"ado-organization-url"` + // ADO project name to call for triggering ADO pipeline + ADOProjectName string `yaml:"ado-project-name" json:"ado-project-name"` + // ADO pipeline ID to call for triggering ADO pipeline + ADOPipelineID int `yaml:"ado-pipeline-id" json:"ado-pipeline-id"` + // ADO pipeline version to call for triggering ADO pipeline + ADOPipelineVersion int `yaml:"ado-pipeline-version,omitempty" json:"ado-pipeline-version,omitempty"` } type SignConfig struct { @@ -60,7 +68,7 @@ func (c *Config) ParseConfig(f []byte) error { type Variants map[string]map[string]string // GetVariants fetches variants from provided file. -// If variant flag is used, it fetches the requested variant. +// If Variant flag is used, it fetches the requested Variant. func GetVariants(variant string, f string, fileGetter func(string) ([]byte, error)) (Variants, error) { var v Variants b, err := fileGetter(f) @@ -68,7 +76,7 @@ func GetVariants(variant string, f string, fileGetter func(string) ([]byte, erro if !os.IsNotExist(err) { return nil, err } - // variant file not found, skipping + // Variant file not found, skipping return nil, nil } if err := yaml.Unmarshal(b, &v); err != nil { @@ -77,7 +85,7 @@ func GetVariants(variant string, f string, fileGetter func(string) ([]byte, erro if variant != "" { va, ok := v[variant] if !ok { - return nil, fmt.Errorf("requested variant '%s', but it's not present in variants.yaml file", variant) + return nil, fmt.Errorf("requested Variant '%s', but it's not present in variants.yaml file", variant) } return Variants{variant: va}, nil } diff --git a/cmd/image-builder/config_test.go b/cmd/image-builder/config_test.go index 9f34120c1800..804fa8d479c0 100644 --- a/cmd/image-builder/config_test.go +++ b/cmd/image-builder/config_test.go @@ -80,7 +80,7 @@ func Test_getVariants(t *testing.T) { }, }, { - name: "variant file does not exist, pass", + name: "Variant file does not exist, pass", expectErr: false, variantsFile: "", expectVariants: nil, @@ -92,7 +92,7 @@ func Test_getVariants(t *testing.T) { expectVariants: nil, }, { - name: "get only single variant, pass", + name: "get only single Variant, pass", expectErr: false, variantsFile: "variants.yaml", variant: "main", @@ -101,10 +101,10 @@ func Test_getVariants(t *testing.T) { }, }, { - name: "variant is not present in variants file, fail", + name: "Variant is not present in variants file, fail", expectErr: true, variantsFile: "variants.yaml", - variant: "missing-variant", + variant: "missing-Variant", expectVariants: nil, }, { @@ -118,7 +118,7 @@ func Test_getVariants(t *testing.T) { t.Run(c.name, func(t *testing.T) { fakeFileGetter := func(f string) ([]byte, error) { if f == "malformed-variants.yaml" { - return []byte("'asd':\n- malformed variant as list`"), nil + return []byte("'asd':\n- malformed Variant as list`"), nil } if f == "err-deadline-exceeded" { return nil, os.ErrDeadlineExceeded diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index cccffd91be32..454b9e77ddfd 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -24,6 +24,7 @@ import ( "github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines" "golang.org/x/net/context" errutil "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/ptr" ) type options struct { diff --git a/configs/kaniko-build-config-ado.yaml b/configs/kaniko-build-config-ado.yaml new file mode 100644 index 000000000000..b53c4fa78115 --- /dev/null +++ b/configs/kaniko-build-config-ado.yaml @@ -0,0 +1,43 @@ +tag-template: 'v{{ .Date }}-{{ .ShortSHA }}' +registry: + - europe-docker.pkg.dev/kyma-project/prod +dev-registry: + - europe-docker.pkg.dev/kyma-project/dev +reproducible: false +log-format: json +ado-organization-url: https://dev.azure.com/hyperspace-pipelines +ado-project-name: neighbors-team +ado-pipeline-id: 13736 +cache: + enabled: true + cache-repo: europe-docker.pkg.dev/kyma-project/cache/cache + cache-run-layers: true +sign-config: + enabled-signers: + '*': + - signify-dev + - signify-prod + signers: + - name: signify-dev + type: notary + job-type: + - presubmit + - postsubmit + config: + endpoint: https://signing-dev.repositories.cloud.sap/signingsvc/sign + timeout: 5m + retry-timeout: 10s + secret: + path: /secret/secret.yaml + type: signify + - name: signify-prod + type: notary + job-type: + - postsubmit + config: + endpoint: https://signing.repositories.cloud.sap/signingsvc/sign + timeout: 5m + retry-timeout: 10s + secret: + path: /secret-prod/secret.yaml + type: signify diff --git a/go.mod b/go.mod index 303818c51cf1..eac08f61b6c5 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,9 @@ require ( github.com/google/uuid v1.4.0 github.com/imdario/mergo v0.3.16 github.com/jamiealquiza/envy v1.1.0 + github.com/jinzhu/copier v0.3.5 + github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 + github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 github.com/jinzhu/copier v0.4.0 github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 github.com/onsi/ginkgo/v2 v2.13.2 diff --git a/prow/jobs/test-infra/image-builder-ado-example.yaml b/prow/jobs/test-infra/image-builder-ado-example.yaml new file mode 100644 index 000000000000..2d3d51b6c9ac --- /dev/null +++ b/prow/jobs/test-infra/image-builder-ado-example.yaml @@ -0,0 +1,62 @@ +# Code generated by rendertemplates. DO NOT EDIT. + + +presubmits: # runs on PRs + kyma-project/test-infra: + - name: pull-image-builder-test + annotations: + description: "image-builder test job" + owner: "neighbors" + labels: + prow.k8s.io/pubsub.project: "sap-kyma-prow" + prow.k8s.io/pubsub.runID: "pull-gitserver-build" + prow.k8s.io/pubsub.topic: "prowjobs" + preset-sa-kyma-push-images: "true" + run_if_changed: '^tools/gitserver' + optional: true + skip_report: false + decorate: true + cluster: untrusted-workload + max_concurrency: 10 + spec: + containers: + - image: "eu.gcr.io/sap-kyma-neighbors-dev/image-builder:latest" + imagePullPolicy: Always + securityContext: + privileged: false + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + env: + - name: ADO_PAT + valueFrom: + secretKeyRef: + name: image-builder-ado-token + key: token + command: + - "/image-builder" + args: + - "--name=test-build" + - "--config=/config/kaniko-build-config.yaml" + - "--context=." + - "--dockerfile=development/secrets-rotator/cloud-run/rotate-service-account/Dockerfile" + - "--build-in-ado=true" + - "--export-Tags=true" + resources: + requests: + memory: 500Mi + cpu: 500m + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: signify-secret + mountPath: /secret + readOnly: true + volumes: + - name: config + configMap: + name: kaniko-build-config-ado + - name: signify-secret + secret: + secretName: signify-dev-secret From d3e9b4750b422c0b0d3a112f6c8b0bdc1b1e2858 Mon Sep 17 00:00:00 2001 From: dekiel Date: Fri, 29 Sep 2023 00:44:00 +0200 Subject: [PATCH 03/29] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index eac08f61b6c5..6decd0a0c027 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/google/uuid v1.4.0 github.com/imdario/mergo v0.3.16 github.com/jamiealquiza/envy v1.1.0 - github.com/jinzhu/copier v0.3.5 + github.com/jinzhu/copier v0.4.0 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 github.com/jinzhu/copier v0.4.0 From cb2f7a25953c2d62931720c72ea2830122fa9b27 Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 15 Nov 2023 15:33:39 +0100 Subject: [PATCH 04/29] Use default build config. --- configs/kaniko-build-config-ado.yaml | 43 ---------------------------- 1 file changed, 43 deletions(-) delete mode 100644 configs/kaniko-build-config-ado.yaml diff --git a/configs/kaniko-build-config-ado.yaml b/configs/kaniko-build-config-ado.yaml deleted file mode 100644 index b53c4fa78115..000000000000 --- a/configs/kaniko-build-config-ado.yaml +++ /dev/null @@ -1,43 +0,0 @@ -tag-template: 'v{{ .Date }}-{{ .ShortSHA }}' -registry: - - europe-docker.pkg.dev/kyma-project/prod -dev-registry: - - europe-docker.pkg.dev/kyma-project/dev -reproducible: false -log-format: json -ado-organization-url: https://dev.azure.com/hyperspace-pipelines -ado-project-name: neighbors-team -ado-pipeline-id: 13736 -cache: - enabled: true - cache-repo: europe-docker.pkg.dev/kyma-project/cache/cache - cache-run-layers: true -sign-config: - enabled-signers: - '*': - - signify-dev - - signify-prod - signers: - - name: signify-dev - type: notary - job-type: - - presubmit - - postsubmit - config: - endpoint: https://signing-dev.repositories.cloud.sap/signingsvc/sign - timeout: 5m - retry-timeout: 10s - secret: - path: /secret/secret.yaml - type: signify - - name: signify-prod - type: notary - job-type: - - postsubmit - config: - endpoint: https://signing.repositories.cloud.sap/signingsvc/sign - timeout: 5m - retry-timeout: 10s - secret: - path: /secret-prod/secret.yaml - type: signify From c200c1df44c5bc091e4a0b4124493cbe9a121f93 Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 28 Sep 2023 14:23:02 +0200 Subject: [PATCH 05/29] POC of image building in ADO. --- configs/kaniko-build-config-ado.yaml | 43 +++++++++++++++++++ .../image-builder/images/kaniko/Dockerfile | 5 +++ 2 files changed, 48 insertions(+) create mode 100644 configs/kaniko-build-config-ado.yaml create mode 100644 development/image-builder/images/kaniko/Dockerfile diff --git a/configs/kaniko-build-config-ado.yaml b/configs/kaniko-build-config-ado.yaml new file mode 100644 index 000000000000..b53c4fa78115 --- /dev/null +++ b/configs/kaniko-build-config-ado.yaml @@ -0,0 +1,43 @@ +tag-template: 'v{{ .Date }}-{{ .ShortSHA }}' +registry: + - europe-docker.pkg.dev/kyma-project/prod +dev-registry: + - europe-docker.pkg.dev/kyma-project/dev +reproducible: false +log-format: json +ado-organization-url: https://dev.azure.com/hyperspace-pipelines +ado-project-name: neighbors-team +ado-pipeline-id: 13736 +cache: + enabled: true + cache-repo: europe-docker.pkg.dev/kyma-project/cache/cache + cache-run-layers: true +sign-config: + enabled-signers: + '*': + - signify-dev + - signify-prod + signers: + - name: signify-dev + type: notary + job-type: + - presubmit + - postsubmit + config: + endpoint: https://signing-dev.repositories.cloud.sap/signingsvc/sign + timeout: 5m + retry-timeout: 10s + secret: + path: /secret/secret.yaml + type: signify + - name: signify-prod + type: notary + job-type: + - postsubmit + config: + endpoint: https://signing.repositories.cloud.sap/signingsvc/sign + timeout: 5m + retry-timeout: 10s + secret: + path: /secret-prod/secret.yaml + type: signify diff --git a/development/image-builder/images/kaniko/Dockerfile b/development/image-builder/images/kaniko/Dockerfile new file mode 100644 index 000000000000..37b41720e2a6 --- /dev/null +++ b/development/image-builder/images/kaniko/Dockerfile @@ -0,0 +1,5 @@ +FROM gcr.io/kaniko-project/executor:v1.14.0 + +COPY ./image-builder /image-builder + +ENTRYPOINT ["/image-builder"] From b5b9cd8046c0a4f5e3033ab2f7d7ddbebb888669 Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 15 Nov 2023 15:33:39 +0100 Subject: [PATCH 06/29] Use default build config. --- configs/kaniko-build-config-ado.yaml | 43 ---------------------------- 1 file changed, 43 deletions(-) delete mode 100644 configs/kaniko-build-config-ado.yaml diff --git a/configs/kaniko-build-config-ado.yaml b/configs/kaniko-build-config-ado.yaml deleted file mode 100644 index b53c4fa78115..000000000000 --- a/configs/kaniko-build-config-ado.yaml +++ /dev/null @@ -1,43 +0,0 @@ -tag-template: 'v{{ .Date }}-{{ .ShortSHA }}' -registry: - - europe-docker.pkg.dev/kyma-project/prod -dev-registry: - - europe-docker.pkg.dev/kyma-project/dev -reproducible: false -log-format: json -ado-organization-url: https://dev.azure.com/hyperspace-pipelines -ado-project-name: neighbors-team -ado-pipeline-id: 13736 -cache: - enabled: true - cache-repo: europe-docker.pkg.dev/kyma-project/cache/cache - cache-run-layers: true -sign-config: - enabled-signers: - '*': - - signify-dev - - signify-prod - signers: - - name: signify-dev - type: notary - job-type: - - presubmit - - postsubmit - config: - endpoint: https://signing-dev.repositories.cloud.sap/signingsvc/sign - timeout: 5m - retry-timeout: 10s - secret: - path: /secret/secret.yaml - type: signify - - name: signify-prod - type: notary - job-type: - - postsubmit - config: - endpoint: https://signing.repositories.cloud.sap/signingsvc/sign - timeout: 5m - retry-timeout: 10s - secret: - path: /secret-prod/secret.yaml - type: signify From f84740fac9ee3a480aa7dda37555bc769571131d Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 15 Nov 2023 16:50:27 +0100 Subject: [PATCH 07/29] fix linter errors --- prow/jobs/kyma-project/test-infra/go-validation.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prow/jobs/kyma-project/test-infra/go-validation.yaml b/prow/jobs/kyma-project/test-infra/go-validation.yaml index 4e289ac16c59..0bccbbe9f4b1 100644 --- a/prow/jobs/kyma-project/test-infra/go-validation.yaml +++ b/prow/jobs/kyma-project/test-infra/go-validation.yaml @@ -11,11 +11,10 @@ presubmits: # runs on PRs prow.k8s.io/pubsub.project: "sap-kyma-prow" prow.k8s.io/pubsub.runID: "pull-unit-test-go" prow.k8s.io/pubsub.topic: "prowjobs" - preset-bot-github-token: "true" run_if_changed: '.*\.go$' skip_report: false decorate: true - cluster: trusted-workload + cluster: untrusted-workload max_concurrency: 10 branches: - ^main$ From 4b65cbe81827cb1f09909b9ccc521c11b088c4e9 Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 15 Nov 2023 16:59:28 +0100 Subject: [PATCH 08/29] rendertemplates --- prow/jobs/kyma-project/test-infra/go-validation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/prow/jobs/kyma-project/test-infra/go-validation.yaml b/prow/jobs/kyma-project/test-infra/go-validation.yaml index 0bccbbe9f4b1..320b13a89f55 100644 --- a/prow/jobs/kyma-project/test-infra/go-validation.yaml +++ b/prow/jobs/kyma-project/test-infra/go-validation.yaml @@ -69,3 +69,4 @@ presubmits: # runs on PRs requests: memory: 1Gi cpu: 2000m + \ No newline at end of file From b3ea48d199fb30630d97b3d78b43bf41ffad4026 Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 21 Nov 2023 09:56:53 +0100 Subject: [PATCH 09/29] Move ado code to separate pacakge and make it testable. --- cmd/image-builder/config.go | 8 ---- pkg/azuredevops/pipelines/templateParams.go | 44 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 pkg/azuredevops/pipelines/templateParams.go diff --git a/cmd/image-builder/config.go b/cmd/image-builder/config.go index 4ca7bca017dc..9205a09cd973 100644 --- a/cmd/image-builder/config.go +++ b/cmd/image-builder/config.go @@ -31,14 +31,6 @@ type Config struct { // SignConfig contains custom configuration of signers // as well as org/repo mapping of enabled signers in specific repository SignConfig SignConfig `yaml:"sign-config" json:"sign-config"` - // ADO organization URL to call for triggering ADO pipeline - ADOOrganizationURL string `yaml:"ado-organization-url" json:"ado-organization-url"` - // ADO project name to call for triggering ADO pipeline - ADOProjectName string `yaml:"ado-project-name" json:"ado-project-name"` - // ADO pipeline ID to call for triggering ADO pipeline - ADOPipelineID int `yaml:"ado-pipeline-id" json:"ado-pipeline-id"` - // ADO pipeline version to call for triggering ADO pipeline - ADOPipelineVersion int `yaml:"ado-pipeline-version,omitempty" json:"ado-pipeline-version,omitempty"` } type SignConfig struct { diff --git a/pkg/azuredevops/pipelines/templateParams.go b/pkg/azuredevops/pipelines/templateParams.go new file mode 100644 index 000000000000..274f58a7e857 --- /dev/null +++ b/pkg/azuredevops/pipelines/templateParams.go @@ -0,0 +1,44 @@ +package pipelines + +import ( + "strconv" +) + +type OCIImageBuilderTemplateParams map[string]string + +func (p OCIImageBuilderTemplateParams) SetRepoName(repoName string) { + p["RepoName"] = repoName +} +func (p OCIImageBuilderTemplateParams) SetRepoOwner(repoOwner string) { + p["RepoOwner"] = repoOwner +} +func (p OCIImageBuilderTemplateParams) SetJobType(jobType string) { + p["JobType"] = jobType +} +func (p OCIImageBuilderTemplateParams) SetPullNumber(pullNumber string) { + p["PullNumber"] = pullNumber +} +func (p OCIImageBuilderTemplateParams) SetPullBaseSHA(pullBaseSHA string) { + p["PullBaseSHA"] = pullBaseSHA +} +func (p OCIImageBuilderTemplateParams) SetImageName(name string) { + p["ImageName"] = name +} +func (p OCIImageBuilderTemplateParams) SetDockerfilePath(dockerfile string) { + p["DockerfilePath"] = dockerfile +} +func (p OCIImageBuilderTemplateParams) SetBuildContext(context string) { + p["BuildContext"] = context +} +func (p OCIImageBuilderTemplateParams) SetExportTags(exportTags bool) { + p["ExportTags"] = strconv.FormatBool(exportTags) +} +func (p OCIImageBuilderTemplateParams) SetPlatforms(platforms string) { + p["Platforms"] = platforms +} +func (p OCIImageBuilderTemplateParams) SetBuildArgs(buildArgs string) { + p["BuildArgs"] = buildArgs +} +func (p OCIImageBuilderTemplateParams) SetImageTags(imageTags string) { + p["ImageTags"] = imageTags +} From 77d242a0a126e920885e70cd18947e04e6e31b4d Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 23 Nov 2023 17:49:34 +0100 Subject: [PATCH 10/29] Move build in ado code to separate package. This can be tested independent and reused in other tools. --- pkg/azuredevops/pipelines/templateParams.go | 35 ++++++++++----------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/pkg/azuredevops/pipelines/templateParams.go b/pkg/azuredevops/pipelines/templateParams.go index 274f58a7e857..7f4a340081f7 100644 --- a/pkg/azuredevops/pipelines/templateParams.go +++ b/pkg/azuredevops/pipelines/templateParams.go @@ -6,39 +6,36 @@ import ( type OCIImageBuilderTemplateParams map[string]string -func (p OCIImageBuilderTemplateParams) SetRepoName(repoName string) { - p["RepoName"] = repoName +func (p OCIImageBuilderTemplateParams) SetRepoName(repo string) { + p["RepoName"] = repo } -func (p OCIImageBuilderTemplateParams) SetRepoOwner(repoOwner string) { - p["RepoOwner"] = repoOwner +func (p OCIImageBuilderTemplateParams) SetRepoOwner(owner string) { + p["RepoOwner"] = owner } func (p OCIImageBuilderTemplateParams) SetJobType(jobType string) { p["JobType"] = jobType } -func (p OCIImageBuilderTemplateParams) SetPullNumber(pullNumber string) { - p["PullNumber"] = pullNumber +func (p OCIImageBuilderTemplateParams) SetPullNumber(number string) { + p["PullNumber"] = number } -func (p OCIImageBuilderTemplateParams) SetPullBaseSHA(pullBaseSHA string) { - p["PullBaseSHA"] = pullBaseSHA +func (p OCIImageBuilderTemplateParams) SetPullBaseSHA(sha string) { + p["PullBaseSHA"] = sha } func (p OCIImageBuilderTemplateParams) SetImageName(name string) { p["ImageName"] = name } -func (p OCIImageBuilderTemplateParams) SetDockerfilePath(dockerfile string) { - p["DockerfilePath"] = dockerfile +func (p OCIImageBuilderTemplateParams) SetDockerfilePath(path string) { + p["DockerfilePath"] = path } func (p OCIImageBuilderTemplateParams) SetBuildContext(context string) { p["BuildContext"] = context } -func (p OCIImageBuilderTemplateParams) SetExportTags(exportTags bool) { - p["ExportTags"] = strconv.FormatBool(exportTags) +func (p OCIImageBuilderTemplateParams) SetExportTags(export bool) { + p["ExportTags"] = strconv.FormatBool(export) } -func (p OCIImageBuilderTemplateParams) SetPlatforms(platforms string) { - p["Platforms"] = platforms +func (p OCIImageBuilderTemplateParams) SetBuildArgs(args string) { + p["BuildArgs"] = args } -func (p OCIImageBuilderTemplateParams) SetBuildArgs(buildArgs string) { - p["BuildArgs"] = buildArgs -} -func (p OCIImageBuilderTemplateParams) SetImageTags(imageTags string) { - p["ImageTags"] = imageTags +func (p OCIImageBuilderTemplateParams) SetImageTags(tags string) { + p["ImageTags"] = tags } From c7ce0a99d2eff8845bbda5af1b221137f4b592c2 Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 29 Nov 2023 11:00:47 +0100 Subject: [PATCH 11/29] Tests. TODOs for needed improvements. Comments. Small changes in code. --- pkg/azuredevops/pipelines/templateParams.go | 41 --------------------- 1 file changed, 41 deletions(-) delete mode 100644 pkg/azuredevops/pipelines/templateParams.go diff --git a/pkg/azuredevops/pipelines/templateParams.go b/pkg/azuredevops/pipelines/templateParams.go deleted file mode 100644 index 7f4a340081f7..000000000000 --- a/pkg/azuredevops/pipelines/templateParams.go +++ /dev/null @@ -1,41 +0,0 @@ -package pipelines - -import ( - "strconv" -) - -type OCIImageBuilderTemplateParams map[string]string - -func (p OCIImageBuilderTemplateParams) SetRepoName(repo string) { - p["RepoName"] = repo -} -func (p OCIImageBuilderTemplateParams) SetRepoOwner(owner string) { - p["RepoOwner"] = owner -} -func (p OCIImageBuilderTemplateParams) SetJobType(jobType string) { - p["JobType"] = jobType -} -func (p OCIImageBuilderTemplateParams) SetPullNumber(number string) { - p["PullNumber"] = number -} -func (p OCIImageBuilderTemplateParams) SetPullBaseSHA(sha string) { - p["PullBaseSHA"] = sha -} -func (p OCIImageBuilderTemplateParams) SetImageName(name string) { - p["ImageName"] = name -} -func (p OCIImageBuilderTemplateParams) SetDockerfilePath(path string) { - p["DockerfilePath"] = path -} -func (p OCIImageBuilderTemplateParams) SetBuildContext(context string) { - p["BuildContext"] = context -} -func (p OCIImageBuilderTemplateParams) SetExportTags(export bool) { - p["ExportTags"] = strconv.FormatBool(export) -} -func (p OCIImageBuilderTemplateParams) SetBuildArgs(args string) { - p["BuildArgs"] = args -} -func (p OCIImageBuilderTemplateParams) SetImageTags(tags string) { - p["ImageTags"] = tags -} From 5b9201e96d8c10b5593e4e1b0a49912987a7b90a Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 29 Nov 2023 11:40:36 +0100 Subject: [PATCH 12/29] go mod tidy --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 6decd0a0c027..332851239d5b 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/imdario/mergo v0.3.16 github.com/jamiealquiza/envy v1.1.0 github.com/jinzhu/copier v0.4.0 - github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 github.com/jinzhu/copier v0.4.0 github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 From d0cbeb8b934fe2cb43434a91b5a60d89464b6dbb Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 30 Nov 2023 00:38:41 +0100 Subject: [PATCH 13/29] Added building image-builder image to the prowjob. --- images/image-builder/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 images/image-builder/Dockerfile diff --git a/images/image-builder/Dockerfile b/images/image-builder/Dockerfile new file mode 100644 index 000000000000..6589ef5fe303 --- /dev/null +++ b/images/image-builder/Dockerfile @@ -0,0 +1,11 @@ +FROM cgr.dev/chainguard/go AS builder + +COPY ../.. /app/ +RUN cd /app/cmd/image-builder && CGO_ENABLED=0 go build -o /app/image-builder -a -ldflags '-extldflags "-static"' . + + +FROM gcr.io/kaniko-project/executor:v1.14.0 + +COPY --from=builder /app/image-builder /image-builder + +ENTRYPOINT ["/image-builder"] From 9552a0c6e6e334d072cdd83a23b7ebc32493766f Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 30 Nov 2023 10:27:34 +0100 Subject: [PATCH 14/29] Use go buildpack. Chainguard offers free access to the latest tag only. --- images/image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/image-builder/Dockerfile b/images/image-builder/Dockerfile index 6589ef5fe303..9fe74d8e3bfd 100644 --- a/images/image-builder/Dockerfile +++ b/images/image-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM cgr.dev/chainguard/go AS builder +FROM europe-docker.pkg.dev/kyma-project/prod/testimages/buildpack-go:v20231128-9bb59ac6 AS builder COPY ../.. /app/ RUN cd /app/cmd/image-builder && CGO_ENABLED=0 go build -o /app/image-builder -a -ldflags '-extldflags "-static"' . From b4266324d9b70d77b2ebd31bfa81688c53c2cce1 Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 30 Nov 2023 12:40:13 +0100 Subject: [PATCH 15/29] Moved Dockerfile back to original location. images directory is not suitable for building image-builder. --- images/image-builder/Dockerfile | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 images/image-builder/Dockerfile diff --git a/images/image-builder/Dockerfile b/images/image-builder/Dockerfile deleted file mode 100644 index 9fe74d8e3bfd..000000000000 --- a/images/image-builder/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM europe-docker.pkg.dev/kyma-project/prod/testimages/buildpack-go:v20231128-9bb59ac6 AS builder - -COPY ../.. /app/ -RUN cd /app/cmd/image-builder && CGO_ENABLED=0 go build -o /app/image-builder -a -ldflags '-extldflags "-static"' . - - -FROM gcr.io/kaniko-project/executor:v1.14.0 - -COPY --from=builder /app/image-builder /image-builder - -ENTRYPOINT ["/image-builder"] From b85baafd23548e8c959dca4521bb16b91d447572 Mon Sep 17 00:00:00 2001 From: dekiel Date: Fri, 1 Dec 2023 16:44:16 +0100 Subject: [PATCH 16/29] Unexport options fields. --- cmd/image-builder/config.go | 6 +++--- cmd/image-builder/config_test.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/image-builder/config.go b/cmd/image-builder/config.go index 9205a09cd973..2a9d4b7ded44 100644 --- a/cmd/image-builder/config.go +++ b/cmd/image-builder/config.go @@ -60,7 +60,7 @@ func (c *Config) ParseConfig(f []byte) error { type Variants map[string]map[string]string // GetVariants fetches variants from provided file. -// If Variant flag is used, it fetches the requested Variant. +// If variant flag is used, it fetches the requested variant. func GetVariants(variant string, f string, fileGetter func(string) ([]byte, error)) (Variants, error) { var v Variants b, err := fileGetter(f) @@ -68,7 +68,7 @@ func GetVariants(variant string, f string, fileGetter func(string) ([]byte, erro if !os.IsNotExist(err) { return nil, err } - // Variant file not found, skipping + // variant file not found, skipping return nil, nil } if err := yaml.Unmarshal(b, &v); err != nil { @@ -77,7 +77,7 @@ func GetVariants(variant string, f string, fileGetter func(string) ([]byte, erro if variant != "" { va, ok := v[variant] if !ok { - return nil, fmt.Errorf("requested Variant '%s', but it's not present in variants.yaml file", variant) + return nil, fmt.Errorf("requested variant '%s', but it's not present in variants.yaml file", variant) } return Variants{variant: va}, nil } diff --git a/cmd/image-builder/config_test.go b/cmd/image-builder/config_test.go index 804fa8d479c0..9f34120c1800 100644 --- a/cmd/image-builder/config_test.go +++ b/cmd/image-builder/config_test.go @@ -80,7 +80,7 @@ func Test_getVariants(t *testing.T) { }, }, { - name: "Variant file does not exist, pass", + name: "variant file does not exist, pass", expectErr: false, variantsFile: "", expectVariants: nil, @@ -92,7 +92,7 @@ func Test_getVariants(t *testing.T) { expectVariants: nil, }, { - name: "get only single Variant, pass", + name: "get only single variant, pass", expectErr: false, variantsFile: "variants.yaml", variant: "main", @@ -101,10 +101,10 @@ func Test_getVariants(t *testing.T) { }, }, { - name: "Variant is not present in variants file, fail", + name: "variant is not present in variants file, fail", expectErr: true, variantsFile: "variants.yaml", - variant: "missing-Variant", + variant: "missing-variant", expectVariants: nil, }, { @@ -118,7 +118,7 @@ func Test_getVariants(t *testing.T) { t.Run(c.name, func(t *testing.T) { fakeFileGetter := func(f string) ([]byte, error) { if f == "malformed-variants.yaml" { - return []byte("'asd':\n- malformed Variant as list`"), nil + return []byte("'asd':\n- malformed variant as list`"), nil } if f == "err-deadline-exceeded" { return nil, os.ErrDeadlineExceeded From 35557720edb1e50320594072b674e2786d823f9b Mon Sep 17 00:00:00 2001 From: dekiel Date: Mon, 4 Dec 2023 01:04:32 +0100 Subject: [PATCH 17/29] Remove testing prowjob. --- .../test-infra/image-builder-ado-example.yaml | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 prow/jobs/test-infra/image-builder-ado-example.yaml diff --git a/prow/jobs/test-infra/image-builder-ado-example.yaml b/prow/jobs/test-infra/image-builder-ado-example.yaml deleted file mode 100644 index 2d3d51b6c9ac..000000000000 --- a/prow/jobs/test-infra/image-builder-ado-example.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Code generated by rendertemplates. DO NOT EDIT. - - -presubmits: # runs on PRs - kyma-project/test-infra: - - name: pull-image-builder-test - annotations: - description: "image-builder test job" - owner: "neighbors" - labels: - prow.k8s.io/pubsub.project: "sap-kyma-prow" - prow.k8s.io/pubsub.runID: "pull-gitserver-build" - prow.k8s.io/pubsub.topic: "prowjobs" - preset-sa-kyma-push-images: "true" - run_if_changed: '^tools/gitserver' - optional: true - skip_report: false - decorate: true - cluster: untrusted-workload - max_concurrency: 10 - spec: - containers: - - image: "eu.gcr.io/sap-kyma-neighbors-dev/image-builder:latest" - imagePullPolicy: Always - securityContext: - privileged: false - seccompProfile: - type: RuntimeDefault - allowPrivilegeEscalation: false - env: - - name: ADO_PAT - valueFrom: - secretKeyRef: - name: image-builder-ado-token - key: token - command: - - "/image-builder" - args: - - "--name=test-build" - - "--config=/config/kaniko-build-config.yaml" - - "--context=." - - "--dockerfile=development/secrets-rotator/cloud-run/rotate-service-account/Dockerfile" - - "--build-in-ado=true" - - "--export-Tags=true" - resources: - requests: - memory: 500Mi - cpu: 500m - volumeMounts: - - name: config - mountPath: /config - readOnly: true - - name: signify-secret - mountPath: /secret - readOnly: true - volumes: - - name: config - configMap: - name: kaniko-build-config-ado - - name: signify-secret - secret: - secretName: signify-dev-secret From fbc83fbe8c5f13c3d7bfc2345a07533a9dab8903 Mon Sep 17 00:00:00 2001 From: dekiel Date: Mon, 4 Dec 2023 19:51:11 +0100 Subject: [PATCH 18/29] Add preview run flag. --- cmd/image-builder/main.go | 2 ++ pkg/azuredevops/pipelines/pipelines.go | 11 ++++++++++- pkg/azuredevops/pipelines/templatesParams.go | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 454b9e77ddfd..f06284fd258a 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -47,6 +47,7 @@ type options struct { signOnly bool imagesToSign sets.Strings buildInADO bool + adoPreviewRun bool parseTagsOnly bool } @@ -644,6 +645,7 @@ func (o *options) gatherOptions(flagSet *flag.FlagSet) *flag.FlagSet { flagSet.BoolVar(&o.signOnly, "sign-only", false, "Only sign the image, do not build it") flagSet.Var(&o.imagesToSign, "images-to-sign", "Comma-separated list of images to sign. Only used when sign-only flag is set") flagSet.BoolVar(&o.buildInADO, "build-in-ado", false, "Build in Azure DevOps pipeline environment") + flagSet.BoolVar(&o.adoPreviewRun, "ado-preview-run", false, "Trigger ADO pipeline in preview mode") flagSet.BoolVar(&o.parseTagsOnly, "parse-tags-only", false, "Only parse tags and print them to stdout") return flagSet diff --git a/pkg/azuredevops/pipelines/pipelines.go b/pkg/azuredevops/pipelines/pipelines.go index 896b057a4e7c..2cce2c1d8227 100644 --- a/pkg/azuredevops/pipelines/pipelines.go +++ b/pkg/azuredevops/pipelines/pipelines.go @@ -109,7 +109,7 @@ func GetRunLogs(ctx context.Context, buildClient BuildClient, httpClient HTTPCli return string(body), nil } -func Run(ctx context.Context, adoClient Client, templateParameters map[string]string, adoConfig Config) (*pipelines.Run, error) { +func Run(ctx context.Context, adoClient Client, templateParameters map[string]string, adoConfig Config, pipelineRunArgs ...RunPipelineArgs) (*pipelines.Run, error) { adoRunPipelineArgs := pipelines.RunPipelineArgs{ Project: &adoConfig.ADOProjectName, PipelineId: &adoConfig.ADOPipelineID, @@ -121,7 +121,16 @@ func Run(ctx context.Context, adoClient Client, templateParameters map[string]st if adoConfig.ADOPipelineVersion != 0 { adoRunPipelineArgs.PipelineVersion = &adoConfig.ADOPipelineVersion } + for _, arg := range pipelineRunArgs { + arg(&adoRunPipelineArgs) + } // TODO: use structured logging with debug severity fmt.Printf("Using TemplateParameters: %+v\n", adoRunPipelineArgs.RunParameters.TemplateParameters) return adoClient.RunPipeline(ctx, adoRunPipelineArgs) } + +type RunPipelineArgs func(*pipelines.RunPipelineArgs) + +func PipelinePreviewRun(args *pipelines.RunPipelineArgs) { + args.RunParameters.PreviewRun = ptr.To(true) +} diff --git a/pkg/azuredevops/pipelines/templatesParams.go b/pkg/azuredevops/pipelines/templatesParams.go index a2617cb9d237..af5d9f8e3e63 100644 --- a/pkg/azuredevops/pipelines/templatesParams.go +++ b/pkg/azuredevops/pipelines/templatesParams.go @@ -14,6 +14,7 @@ func (e ErrRequiredParamNotSet) Error() string { return "required parameter not set: " + string(e) } +// TODO: Rename, remove Template, as this is are parameters for pipeline execution. // OCIImageBuilderTemplateParams is a map of parameters for OCIImageBuilderTemplate type OCIImageBuilderTemplateParams map[string]string From 32ae4255be36c5bd7a8140db34e77042dfcb1316 Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 00:23:50 +0100 Subject: [PATCH 19/29] Add ADO pipeline preview run feature and test Added a feature in the image builder that allows ADO (Azure DevOps) pipeline to run in preview mode. This allows users to see the final YAML of the pipeline before executing it. The use of this option can be flagged with adoPreviewRun and it has been limited to work only when running in ADO and not locally. Made companion changes to the tests for these features ensuring all new code is covered. This addition was made to aid debug and development efforts by providing more comprehensive information about pipeline executions in ADO. --- cmd/image-builder/main.go | 9 ++++++ pkg/azuredevops/pipelines/pipelines_test.go | 35 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index f06284fd258a..f96ae736e1da 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -267,6 +267,11 @@ func buildInADO(o options) error { return fmt.Errorf("build in ADO failed, failed running ADO pipeline, err: %s", err) } + if o.adoPreviewRun { + fmt.Printf("ADO pipeline preview run result\n: %s", *pipelineRun.FinalYaml) + return nil + } + pipelineRunResult, err := adopipelines.GetRunResult(ctx, adoClient, o.AdoConfig.GetADOConfig(), pipelineRun.Id, 30*time.Second) if err != nil { return fmt.Errorf("build in ADO failed, failed getting ADO pipeline run result, err: %s", err) @@ -574,6 +579,10 @@ func validateOptions(o options) error { errs = append(errs, fmt.Errorf("variant flag is not supported when running in ADO")) } + if o.adoPreviewRun && !o.buildInADO { + errs = append(errs, fmt.Errorf("adoPreviewRun flag is not supported when running locally")) + } + return errutil.NewAggregate(errs) } diff --git a/pkg/azuredevops/pipelines/pipelines_test.go b/pkg/azuredevops/pipelines/pipelines_test.go index d0414ef59eaa..a39ddedf0b64 100644 --- a/pkg/azuredevops/pipelines/pipelines_test.go +++ b/pkg/azuredevops/pipelines/pipelines_test.go @@ -13,7 +13,7 @@ import ( "strings" "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines" - "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" + pipelinesmocks "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" "github.com/microsoft/azure-devops-go-api/azuredevops/v7/build" adoPipelines "github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines" @@ -201,6 +201,8 @@ var _ = Describe("Pipelines", func() { run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) Expect(err).ToNot(HaveOccurred()) Expect(run.Id).To(Equal(ptr.To(123))) + mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) + mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) mockADOClient.AssertExpectations(GinkgoT()) }) @@ -209,7 +211,38 @@ var _ = Describe("Pipelines", func() { _, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) Expect(err).To(HaveOccurred()) + mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) + mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) mockADOClient.AssertExpectations(GinkgoT()) }) + + It("should run the pipeline in preview mode", func() { + finalYaml := "pipeline:\n stages:\n - stage: Build\n jobs:\n - job: Build\n steps:\n - script: echo Hello, world!\n displayName: 'Run a one-line script'" + runPipelineArgs.RunParameters.PreviewRun = ptr.To(true) + mockRun := &adoPipelines.Run{Id: ptr.To(123), FinalYaml: &finalYaml} + mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) + + run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig, pipelines.PipelinePreviewRun) + Expect(err).ToNot(HaveOccurred()) + Expect(run.Id).To(Equal(ptr.To(123))) + Expect(run.FinalYaml).To(Equal(&finalYaml)) + mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) + mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) + mockADOClient.AssertExpectations(GinkgoT()) + }) + }) + + Describe("PipelinePreviewRun", func() { + It("should set PreviewRun to true", func() { + args := &adoPipelines.RunPipelineArgs{ + RunParameters: &adoPipelines.RunPipelineParameters{ + PreviewRun: ptr.To(false), + }, + } + + pipelines.PipelinePreviewRun(args) + + Expect(args.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + }) }) }) From de2bb8a5a6329cb839486032c7b5ac84647447ee Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 02:32:16 +0100 Subject: [PATCH 20/29] package not used in image-builder --- cmd/image-builder/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index f96ae736e1da..a591bdef44ba 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -24,7 +24,6 @@ import ( "github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines" "golang.org/x/net/context" errutil "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/utils/ptr" ) type options struct { From e5dc1c2a037ec81615108a5c784c9d74cb626c27 Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 02:56:56 +0100 Subject: [PATCH 21/29] File committed by mistake. --- development/image-builder/images/kaniko/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 development/image-builder/images/kaniko/Dockerfile diff --git a/development/image-builder/images/kaniko/Dockerfile b/development/image-builder/images/kaniko/Dockerfile deleted file mode 100644 index 37b41720e2a6..000000000000 --- a/development/image-builder/images/kaniko/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM gcr.io/kaniko-project/executor:v1.14.0 - -COPY ./image-builder /image-builder - -ENTRYPOINT ["/image-builder"] From 2eaba3ba9c0a9feb547f40c6456d3ae2b97d2dcc Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 02:58:49 +0100 Subject: [PATCH 22/29] go mod tidy --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 332851239d5b..303818c51cf1 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,6 @@ require ( github.com/jamiealquiza/envy v1.1.0 github.com/jinzhu/copier v0.4.0 github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 - github.com/jinzhu/copier v0.4.0 - github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0 github.com/onsi/ginkgo/v2 v2.13.2 github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 From 2c26d410d29ffe8cd827b80028a2816716c436be Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 13:01:42 +0100 Subject: [PATCH 23/29] Pull number should be expect only for presubmit job types. --- cmd/image-builder/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index a591bdef44ba..0ca1e17ba235 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -202,10 +202,12 @@ func prepareADOTemplateParameters(options options) (adopipelines.OCIImageBuilder } number, present := os.LookupEnv("PULL_NUMBER") - if !present { + if jobType == "presubmit" && !present { return nil, fmt.Errorf("PULL_NUMBER environment variable is not set, please set it to valid pull request number") } - templateParameters.SetPullNumber(number) + if present { + templateParameters.SetPullNumber(number) + } baseSHA, present := os.LookupEnv("PULL_BASE_SHA") if !present { From b85ab4cf6077e7bae2bc3067e14fc36003f59ab3 Mon Sep 17 00:00:00 2001 From: dekiel Date: Tue, 5 Dec 2023 13:03:47 +0100 Subject: [PATCH 24/29] Print exit code in new line. --- cmd/image-builder/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 0ca1e17ba235..53ee59d9f267 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -728,13 +728,13 @@ func main() { fmt.Println(err) os.Exit(1) } - fmt.Printf("%s", jsonTags) + fmt.Printf("%s\n", jsonTags) os.Exit(0) } if o.buildInADO { err = buildInADO(o) if err != nil { - fmt.Printf("Image build failed with error: %s", err) + fmt.Printf("Image build failed with error: %s\n", err) os.Exit(1) } } From aeab1a62e7fa06e7568baef727ba13dc2a06a25b Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 6 Dec 2023 13:49:54 +0100 Subject: [PATCH 25/29] Align with naming standard. --- pkg/azuredevops/pipelines/pipelines_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/azuredevops/pipelines/pipelines_test.go b/pkg/azuredevops/pipelines/pipelines_test.go index a39ddedf0b64..db77cdf05f5e 100644 --- a/pkg/azuredevops/pipelines/pipelines_test.go +++ b/pkg/azuredevops/pipelines/pipelines_test.go @@ -13,7 +13,7 @@ import ( "strings" "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines" - pipelinesmocks "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" + pipelinesMocks "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" "github.com/microsoft/azure-devops-go-api/azuredevops/v7/build" adoPipelines "github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines" @@ -32,7 +32,7 @@ func (t ginkgoT) Cleanup(f func()) { var _ = Describe("Pipelines", func() { var ( ctx context.Context - mockADOClient *pipelinesmocks.MockClient + mockADOClient *pipelinesMocks.MockClient adoConfig pipelines.Config t ginkgoT ) @@ -41,7 +41,7 @@ var _ = Describe("Pipelines", func() { ctx = context.Background() t = ginkgoT{} t.GinkgoTInterface = GinkgoT() - mockADOClient = pipelinesmocks.NewMockClient(t) + mockADOClient = pipelinesMocks.NewMockClient(t) adoConfig = pipelines.Config{ ADOOrganizationURL: "https://dev.azure.com", ADOProjectName: "example-project", @@ -106,15 +106,15 @@ var _ = Describe("Pipelines", func() { Describe("GetRunLogs", func() { var ( - mockBuildClient *pipelinesmocks.MockBuildClient - mockHTTPClient *pipelinesmocks.MockHTTPClient + mockBuildClient *pipelinesMocks.MockBuildClient + mockHTTPClient *pipelinesMocks.MockHTTPClient getBuildLogsArgs build.GetBuildLogsArgs mockBuildLogs *[]build.BuildLog ) BeforeEach(func() { - mockBuildClient = pipelinesmocks.NewMockBuildClient(t) - mockHTTPClient = pipelinesmocks.NewMockHTTPClient(t) + mockBuildClient = pipelinesMocks.NewMockBuildClient(t) + mockHTTPClient = pipelinesMocks.NewMockHTTPClient(t) getBuildLogsArgs = build.GetBuildLogsArgs{ Project: &adoConfig.ADOProjectName, BuildId: ptr.To(42), From f9a1ec046bf9e4099365aaee9ca60f1c52db4c24 Mon Sep 17 00:00:00 2001 From: dekiel Date: Wed, 6 Dec 2023 14:06:30 +0100 Subject: [PATCH 26/29] Add error handling for nil final yaml in ADO pipeline preview run This update adds an error message when the final yaml in the Azure DevOps (ADO) pipeline preview run is nil. This handling is necessary to give clearer feedback when the pipeline preview run fails due to a nil final yaml and to prevent runtime errors. --- cmd/image-builder/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 53ee59d9f267..d8277087abba 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -268,10 +268,13 @@ func buildInADO(o options) error { return fmt.Errorf("build in ADO failed, failed running ADO pipeline, err: %s", err) } - if o.adoPreviewRun { + if o.adoPreviewRun && pipelineRun.FinalYaml != nil { fmt.Printf("ADO pipeline preview run result\n: %s", *pipelineRun.FinalYaml) return nil } + if o.adoPreviewRun && pipelineRun.FinalYaml == nil { + return fmt.Errorf("ADO pipeline preview run failed, final yaml is nil") + } pipelineRunResult, err := adopipelines.GetRunResult(ctx, adoClient, o.AdoConfig.GetADOConfig(), pipelineRun.Id, 30*time.Second) if err != nil { From a4dfb4e31fe208df08b455fbdb71eea6fc5e243b Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 7 Dec 2023 13:12:35 +0100 Subject: [PATCH 27/29] Add preview run option for Azure DevOps pipelines Refactored the Azure DevOps pipeline trigger functionality to support a "preview run" mode where users can see the generated pipeline yaml before running the actual pipeline. This feature is useful in creating or troubleshooting pipelines in Azure DevOps as it enables users to verify and adjust the yaml configuration before executing a pipeline run. This reduces the risk of pipeline failures due to misconfigurations. Added a new flag that allows the user to specify the path of a yaml file that contains the pipeline definition for the preview run. Made necessary changes in cmd/image-builder/main.go and pkg/azuredevops/pipelines/pipelines.go. Also modified some tests to accommodate these changes and ensure the functionality is working as expected in both preview run and standard run modes. --- cmd/image-builder/main.go | 54 +++++--- cmd/image-builder/main_test.go | 59 ++++++++ pkg/azuredevops/pipelines/pipelines.go | 27 +++- pkg/azuredevops/pipelines/pipelines_test.go | 141 +++++++++++++------- 4 files changed, 210 insertions(+), 71 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index d8277087abba..f4aa2a8c60ff 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -43,11 +43,12 @@ type options struct { platforms sets.Strings exportTags bool // signOnly only sign images. No build will be performed. - signOnly bool - imagesToSign sets.Strings - buildInADO bool - adoPreviewRun bool - parseTagsOnly bool + signOnly bool + imagesToSign sets.Strings + buildInADO bool + adoPreviewRun bool + adoPreviewRunYamlPath string + parseTagsOnly bool } const ( @@ -257,30 +258,43 @@ func buildInADO(o options) error { if err != nil { return fmt.Errorf("build in ADO failed, failed preparing ADO template parameters, err: %s", err) } + fmt.Printf("Using TemplateParameters: %+v\n", templateParameters) - fmt.Println("Running ADO pipeline.") adoClient := adopipelines.NewClient(o.AdoConfig.ADOOrganizationURL, adoPAT) + var opts []adopipelines.RunPipelineArgsOptions + if o.adoPreviewRun { + fmt.Println("Running in preview mode.") + opts = append(opts, adopipelines.PipelinePreviewRun(o.adoPreviewRunYamlPath)) + } + + fmt.Println("Preparing ADO pipeline run arguments.") + runPipelineArgs, err := adopipelines.NewRunPipelineArgs(templateParameters, o.AdoConfig.GetADOConfig(), opts...) + if err != nil { + return fmt.Errorf("build in ADO failed, failed creating ADO pipeline run args, err: %s", err) + } + fmt.Println("Triggering ADO build pipeline") ctx := context.Background() - pipelineRun, err := adopipelines.Run(ctx, adoClient, templateParameters, o.AdoConfig.GetADOConfig()) + pipelineRun, err := adoClient.RunPipeline(ctx, runPipelineArgs) if err != nil { return fmt.Errorf("build in ADO failed, failed running ADO pipeline, err: %s", err) } - if o.adoPreviewRun && pipelineRun.FinalYaml != nil { - fmt.Printf("ADO pipeline preview run result\n: %s", *pipelineRun.FinalYaml) + if o.adoPreviewRun { + if pipelineRun.FinalYaml != nil { + fmt.Printf("ADO pipeline preview run final yaml\n: %s", *pipelineRun.FinalYaml) + } else { + fmt.Println("ADO pipeline preview run final yaml is empty") + } return nil } - if o.adoPreviewRun && pipelineRun.FinalYaml == nil { - return fmt.Errorf("ADO pipeline preview run failed, final yaml is nil") - } pipelineRunResult, err := adopipelines.GetRunResult(ctx, adoClient, o.AdoConfig.GetADOConfig(), pipelineRun.Id, 30*time.Second) if err != nil { return fmt.Errorf("build in ADO failed, failed getting ADO pipeline run result, err: %s", err) } - fmt.Printf("ADO pipeline run finished with status: %s", *pipelineRunResult) + fmt.Printf("ADO pipeline run finished with status: %s\n", *pipelineRunResult) fmt.Println("Getting ADO pipeline run logs.") adoBuildClient, err := adopipelines.NewBuildClient(o.AdoConfig.ADOOrganizationURL, adoPAT) @@ -576,7 +590,7 @@ func validateOptions(o options) error { } if o.envFile != "" && o.buildInADO { - errs = append(errs, fmt.Errorf("envFile flag is not supported when running in ADO")) + errs = append(errs, fmt.Errorf("env-file flag is not supported when running in ADO")) } if o.variant != "" && o.buildInADO { @@ -584,7 +598,15 @@ func validateOptions(o options) error { } if o.adoPreviewRun && !o.buildInADO { - errs = append(errs, fmt.Errorf("adoPreviewRun flag is not supported when running locally")) + errs = append(errs, fmt.Errorf("ado-preview-run flag is not supported when running locally")) + } + + if o.adoPreviewRun && o.adoPreviewRunYamlPath == "" { + errs = append(errs, fmt.Errorf("ado-preview-run-yaml-path flag is missing, please provide path to yaml file with ADO pipeline definition")) + } + + if o.adoPreviewRunYamlPath != "" && !o.adoPreviewRun { + errs = append(errs, fmt.Errorf("ado-preview-run-yaml-path flag is provided, but adoPreviewRun flag is not set to true")) } return errutil.NewAggregate(errs) @@ -659,6 +681,7 @@ func (o *options) gatherOptions(flagSet *flag.FlagSet) *flag.FlagSet { flagSet.Var(&o.imagesToSign, "images-to-sign", "Comma-separated list of images to sign. Only used when sign-only flag is set") flagSet.BoolVar(&o.buildInADO, "build-in-ado", false, "Build in Azure DevOps pipeline environment") flagSet.BoolVar(&o.adoPreviewRun, "ado-preview-run", false, "Trigger ADO pipeline in preview mode") + flagSet.StringVar(&o.adoPreviewRunYamlPath, "ado-preview-run-yaml-path", "", "Path to yaml file with ADO pipeline definition to be used in preview mode") flagSet.BoolVar(&o.parseTagsOnly, "parse-tags-only", false, "Only parse tags and print them to stdout") return flagSet @@ -740,6 +763,7 @@ func main() { fmt.Printf("Image build failed with error: %s\n", err) os.Exit(1) } + os.Exit(0) } err = buildLocally(o) if err != nil { diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 18dfbeb78375..5e6959d91c70 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -566,3 +566,62 @@ type mockSigner struct { func (m *mockSigner) Sign(images []string) error { return m.signFunc(images) } + +// TODO: add tests for functions related to execution in ado. +// Test copied from pkg/azuredevops/pipelines/pipelines_test.go, rewrite to run it here. +// Describe("Run", func() { +// var ( +// templateParams map[string]string +// runPipelineArgs adoPipelines.RunPipelineArgs +// ) +// +// BeforeEach(func() { +// templateParams = map[string]string{"param1": "value1", "param2": "value2"} +// runPipelineArgs = adoPipelines.RunPipelineArgs{ +// Project: &adoConfig.ADOProjectName, +// PipelineId: &adoConfig.ADOPipelineID, +// RunParameters: &adoPipelines.RunPipelineParameters{ +// PreviewRun: ptr.To(false), +// TemplateParameters: &templateParams, +// }, +// PipelineVersion: &adoConfig.ADOPipelineVersion, +// } +// }) +// +// It("should run the pipeline", func() { +// mockRun := &adoPipelines.Run{Id: ptr.To(123)} +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) +// +// run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) +// Expect(err).ToNot(HaveOccurred()) +// Expect(run.Id).To(Equal(ptr.To(123))) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// +// It("should handle ADO client error", func() { +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(nil, fmt.Errorf("ADO client error")) +// +// _, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) +// Expect(err).To(HaveOccurred()) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// +// It("should run the pipeline in preview mode", func() { +// finalYaml := "pipeline:\n stages:\n - stage: Build\n jobs:\n - job: Build\n steps:\n - script: echo Hello, world!\n displayName: 'Run a one-line script'" +// runPipelineArgs.RunParameters.PreviewRun = ptr.To(true) +// mockRun := &adoPipelines.Run{Id: ptr.To(123), FinalYaml: &finalYaml} +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) +// +// run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig, pipelines.PipelinePreviewRun) +// Expect(err).ToNot(HaveOccurred()) +// Expect(run.Id).To(Equal(ptr.To(123))) +// Expect(run.FinalYaml).To(Equal(&finalYaml)) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// }) diff --git a/pkg/azuredevops/pipelines/pipelines.go b/pkg/azuredevops/pipelines/pipelines.go index 2cce2c1d8227..a12976c404c1 100644 --- a/pkg/azuredevops/pipelines/pipelines.go +++ b/pkg/azuredevops/pipelines/pipelines.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "time" adov7 "github.com/microsoft/azure-devops-go-api/azuredevops/v7" @@ -109,7 +110,7 @@ func GetRunLogs(ctx context.Context, buildClient BuildClient, httpClient HTTPCli return string(body), nil } -func Run(ctx context.Context, adoClient Client, templateParameters map[string]string, adoConfig Config, pipelineRunArgs ...RunPipelineArgs) (*pipelines.Run, error) { +func NewRunPipelineArgs(templateParameters map[string]string, adoConfig Config, pipelineRunArgs ...RunPipelineArgsOptions) (pipelines.RunPipelineArgs, error) { adoRunPipelineArgs := pipelines.RunPipelineArgs{ Project: &adoConfig.ADOProjectName, PipelineId: &adoConfig.ADOPipelineID, @@ -122,15 +123,27 @@ func Run(ctx context.Context, adoClient Client, templateParameters map[string]st adoRunPipelineArgs.PipelineVersion = &adoConfig.ADOPipelineVersion } for _, arg := range pipelineRunArgs { - arg(&adoRunPipelineArgs) + err := arg(&adoRunPipelineArgs) + if err != nil { + return pipelines.RunPipelineArgs{}, fmt.Errorf("failed setting pipeline run args, err: %w", err) + } } // TODO: use structured logging with debug severity - fmt.Printf("Using TemplateParameters: %+v\n", adoRunPipelineArgs.RunParameters.TemplateParameters) - return adoClient.RunPipeline(ctx, adoRunPipelineArgs) + // fmt.Printf("Using TemplateParameters: %+v\n", adoRunPipelineArgs.RunParameters.TemplateParameters) + return adoRunPipelineArgs, nil } -type RunPipelineArgs func(*pipelines.RunPipelineArgs) +type RunPipelineArgsOptions func(*pipelines.RunPipelineArgs) error -func PipelinePreviewRun(args *pipelines.RunPipelineArgs) { - args.RunParameters.PreviewRun = ptr.To(true) +func PipelinePreviewRun(overrideYamlPath string) func(args *pipelines.RunPipelineArgs) error { + return func(args *pipelines.RunPipelineArgs) error { + args.RunParameters.PreviewRun = ptr.To(true) + data, err := os.ReadFile(overrideYamlPath) + if err != nil { + return fmt.Errorf("failed reading override yaml file, err: %w", err) + } + overrideYaml := string(data) + args.RunParameters.YamlOverride = &overrideYaml + return nil + } } diff --git a/pkg/azuredevops/pipelines/pipelines_test.go b/pkg/azuredevops/pipelines/pipelines_test.go index db77cdf05f5e..b9b368fb9473 100644 --- a/pkg/azuredevops/pipelines/pipelines_test.go +++ b/pkg/azuredevops/pipelines/pipelines_test.go @@ -1,6 +1,7 @@ package pipelines_test import ( + "os" "time" . "github.com/onsi/ginkgo/v2" @@ -175,74 +176,116 @@ var _ = Describe("Pipelines", func() { }) }) - Describe("Run", func() { + Describe("NewRunPipelineArgs", func() { var ( - templateParams map[string]string - runPipelineArgs adoPipelines.RunPipelineArgs + templateParameters map[string]string + pipelineRunArgs []pipelines.RunPipelineArgsOptions ) BeforeEach(func() { - templateParams = map[string]string{"param1": "value1", "param2": "value2"} - runPipelineArgs = adoPipelines.RunPipelineArgs{ - Project: &adoConfig.ADOProjectName, - PipelineId: &adoConfig.ADOPipelineID, - RunParameters: &adoPipelines.RunPipelineParameters{ - PreviewRun: ptr.To(false), - TemplateParameters: &templateParams, - }, - PipelineVersion: &adoConfig.ADOPipelineVersion, + templateParameters = map[string]string{ + "key1": "value1", + "key2": "value2", } }) - It("should run the pipeline", func() { - mockRun := &adoPipelines.Run{Id: ptr.To(123)} - mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) - - run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) - Expect(err).ToNot(HaveOccurred()) - Expect(run.Id).To(Equal(ptr.To(123))) - mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) - mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) - mockADOClient.AssertExpectations(GinkgoT()) + Context("when NewRunPipelineArgs is successful", func() { + It("should return the correct PipelineArgs and no error", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.Project).To(Equal(&adoConfig.ADOProjectName)) + Expect(pipelineArgs.PipelineId).To(Equal(&adoConfig.ADOPipelineID)) + Expect(pipelineArgs.PipelineVersion).To(Equal(&adoConfig.ADOPipelineVersion)) + Expect(pipelineArgs.RunParameters.TemplateParameters).To(Equal(&templateParameters)) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(false))) + Expect(pipelineArgs).To(BeAssignableToTypeOf(adoPipelines.RunPipelineArgs{})) + }) + Context("when PipelinePreviewRun option is passed", func() { + var dummyOverrideYamlPath = "./dummyOverride.yaml" + BeforeEach(func() { + pipelineRunArgs = []pipelines.RunPipelineArgsOptions{ + pipelines.PipelinePreviewRun(dummyOverrideYamlPath), + } + err := os.WriteFile(dummyOverrideYamlPath, []byte("dummyYamlContent"), 0644) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := os.Remove(dummyOverrideYamlPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should enable preview run and set YamlOverride according to file content", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig, pipelineRunArgs...) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.Project).To(Equal(&adoConfig.ADOProjectName)) + Expect(pipelineArgs.PipelineId).To(Equal(&adoConfig.ADOPipelineID)) + Expect(pipelineArgs.PipelineVersion).To(Equal(&adoConfig.ADOPipelineVersion)) + Expect(pipelineArgs.RunParameters.TemplateParameters).To(Equal(&templateParameters)) + Expect(pipelineArgs).To(BeAssignableToTypeOf(adoPipelines.RunPipelineArgs{})) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + Expect(pipelineArgs.RunParameters.YamlOverride).To(Equal(ptr.To("dummyYamlContent"))) + }) + }) }) - It("should handle ADO client error", func() { - mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(nil, fmt.Errorf("ADO client error")) - - _, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) - Expect(err).To(HaveOccurred()) - mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) - mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) - mockADOClient.AssertExpectations(GinkgoT()) - }) - - It("should run the pipeline in preview mode", func() { - finalYaml := "pipeline:\n stages:\n - stage: Build\n jobs:\n - job: Build\n steps:\n - script: echo Hello, world!\n displayName: 'Run a one-line script'" - runPipelineArgs.RunParameters.PreviewRun = ptr.To(true) - mockRun := &adoPipelines.Run{Id: ptr.To(123), FinalYaml: &finalYaml} - mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) - - run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig, pipelines.PipelinePreviewRun) - Expect(err).ToNot(HaveOccurred()) - Expect(run.Id).To(Equal(ptr.To(123))) - Expect(run.FinalYaml).To(Equal(&finalYaml)) - mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) - mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) - mockADOClient.AssertExpectations(GinkgoT()) + Context("when NewRunPipelineArgs fails", func() { + BeforeEach(func() { + pipelineRunArgs = []pipelines.RunPipelineArgsOptions{ + func(args *adoPipelines.RunPipelineArgs) error { + return fmt.Errorf("dummy error") + }, + } + }) + + It("should return an error", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig, pipelineRunArgs...) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("failed setting pipeline run args, err: dummy error")) + Expect(pipelineArgs).To(BeEquivalentTo(adoPipelines.RunPipelineArgs{})) + }) }) }) Describe("PipelinePreviewRun", func() { - It("should set PreviewRun to true", func() { - args := &adoPipelines.RunPipelineArgs{ + var ( + dummyOverrideYamlPath = "./dummyOverride.yaml" + err error + pipelineArgs *adoPipelines.RunPipelineArgs + ) + + BeforeEach(func() { + err = os.WriteFile(dummyOverrideYamlPath, []byte("dummyYamlContent"), 0644) + Expect(err).NotTo(HaveOccurred()) + pipelineArgs = &adoPipelines.RunPipelineArgs{ RunParameters: &adoPipelines.RunPipelineParameters{ PreviewRun: ptr.To(false), }, } + }) + + AfterEach(func() { + err = os.Remove(dummyOverrideYamlPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should prepare function that sets PreviewRun to true and reads override yaml", func() { - pipelines.PipelinePreviewRun(args) + pipelinePreviewRun := pipelines.PipelinePreviewRun(dummyOverrideYamlPath) - Expect(args.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + err := pipelinePreviewRun(pipelineArgs) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + Expect(pipelineArgs.RunParameters.YamlOverride).To(Equal(ptr.To("dummyYamlContent"))) + }) + Context("when the override yaml file does not exist", func() { + It("should return an error", func() { + nonExistentFilePath := "/path/to/non-existent/file.yaml" + pipelinePreviewRun := pipelines.PipelinePreviewRun(nonExistentFilePath) + + err := pipelinePreviewRun(pipelineArgs) + Expect(err).To(HaveOccurred()) + }) }) }) }) From cdc54688febb3645f682757bbdef7c6a1c46e68b Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 7 Dec 2023 22:57:08 +0100 Subject: [PATCH 28/29] Review comments. --- cmd/image-builder/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index f4aa2a8c60ff..66539d634233 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -202,12 +202,12 @@ func prepareADOTemplateParameters(options options) (adopipelines.OCIImageBuilder return nil, fmt.Errorf("JOB_TYPE environment variable is not set to valid value, please set it to either 'presubmit' or 'postsubmit'") } - number, present := os.LookupEnv("PULL_NUMBER") - if jobType == "presubmit" && !present { + pullNumber, isPullNumberSet := os.LookupEnv("PULL_NUMBER") + if jobType == "presubmit" && isPullNumberSet { return nil, fmt.Errorf("PULL_NUMBER environment variable is not set, please set it to valid pull request number") } if present { - templateParameters.SetPullNumber(number) + templateParameters.SetPullNumber(pullNumber) } baseSHA, present := os.LookupEnv("PULL_BASE_SHA") From ef98fc8e21d3c76e4ba58cf1fe327ec01ceb87a6 Mon Sep 17 00:00:00 2001 From: dekiel Date: Thu, 7 Dec 2023 23:32:24 +0100 Subject: [PATCH 29/29] Added missing negation. Aligned variable name in if condition. --- cmd/image-builder/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 66539d634233..b709d3b83cab 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -203,10 +203,10 @@ func prepareADOTemplateParameters(options options) (adopipelines.OCIImageBuilder } pullNumber, isPullNumberSet := os.LookupEnv("PULL_NUMBER") - if jobType == "presubmit" && isPullNumberSet { + if jobType == "presubmit" && !isPullNumberSet { return nil, fmt.Errorf("PULL_NUMBER environment variable is not set, please set it to valid pull request number") } - if present { + if isPullNumberSet { templateParameters.SetPullNumber(pullNumber) }