diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index cccffd91be32..b709d3b83cab 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -43,10 +43,12 @@ type options struct { platforms sets.Strings exportTags bool // signOnly only sign images. No build will be performed. - signOnly bool - imagesToSign sets.Strings - buildInADO bool - parseTagsOnly bool + signOnly bool + imagesToSign sets.Strings + buildInADO bool + adoPreviewRun bool + adoPreviewRunYamlPath string + parseTagsOnly bool } const ( @@ -200,11 +202,13 @@ func prepareADOTemplateParameters(options options) (adopipelines.OCIImageBuilder return nil, fmt.Errorf("JOB_TYPE environment variable is not set to valid value, please set it to either 'presubmit' or 'postsubmit'") } - number, present := os.LookupEnv("PULL_NUMBER") - if !present { + pullNumber, isPullNumberSet := os.LookupEnv("PULL_NUMBER") + if jobType == "presubmit" && !isPullNumberSet { return nil, fmt.Errorf("PULL_NUMBER environment variable is not set, please set it to valid pull request number") } - templateParameters.SetPullNumber(number) + if isPullNumberSet { + templateParameters.SetPullNumber(pullNumber) + } baseSHA, present := os.LookupEnv("PULL_BASE_SHA") if !present { @@ -254,22 +258,43 @@ func buildInADO(o options) error { if err != nil { return fmt.Errorf("build in ADO failed, failed preparing ADO template parameters, err: %s", err) } + fmt.Printf("Using TemplateParameters: %+v\n", templateParameters) - fmt.Println("Running ADO pipeline.") adoClient := adopipelines.NewClient(o.AdoConfig.ADOOrganizationURL, adoPAT) + var opts []adopipelines.RunPipelineArgsOptions + if o.adoPreviewRun { + fmt.Println("Running in preview mode.") + opts = append(opts, adopipelines.PipelinePreviewRun(o.adoPreviewRunYamlPath)) + } + + fmt.Println("Preparing ADO pipeline run arguments.") + runPipelineArgs, err := adopipelines.NewRunPipelineArgs(templateParameters, o.AdoConfig.GetADOConfig(), opts...) + if err != nil { + return fmt.Errorf("build in ADO failed, failed creating ADO pipeline run args, err: %s", err) + } + fmt.Println("Triggering ADO build pipeline") ctx := context.Background() - pipelineRun, err := adopipelines.Run(ctx, adoClient, templateParameters, o.AdoConfig.GetADOConfig()) + pipelineRun, err := adoClient.RunPipeline(ctx, runPipelineArgs) if err != nil { return fmt.Errorf("build in ADO failed, failed running ADO pipeline, err: %s", err) } + if o.adoPreviewRun { + if pipelineRun.FinalYaml != nil { + fmt.Printf("ADO pipeline preview run final yaml\n: %s", *pipelineRun.FinalYaml) + } else { + fmt.Println("ADO pipeline preview run final yaml is empty") + } + return nil + } + pipelineRunResult, err := adopipelines.GetRunResult(ctx, adoClient, o.AdoConfig.GetADOConfig(), pipelineRun.Id, 30*time.Second) if err != nil { return fmt.Errorf("build in ADO failed, failed getting ADO pipeline run result, err: %s", err) } - fmt.Printf("ADO pipeline run finished with status: %s", *pipelineRunResult) + fmt.Printf("ADO pipeline run finished with status: %s\n", *pipelineRunResult) fmt.Println("Getting ADO pipeline run logs.") adoBuildClient, err := adopipelines.NewBuildClient(o.AdoConfig.ADOOrganizationURL, adoPAT) @@ -565,13 +590,25 @@ func validateOptions(o options) error { } if o.envFile != "" && o.buildInADO { - errs = append(errs, fmt.Errorf("envFile flag is not supported when running in ADO")) + errs = append(errs, fmt.Errorf("env-file flag is not supported when running in ADO")) } if o.variant != "" && o.buildInADO { errs = append(errs, fmt.Errorf("variant flag is not supported when running in ADO")) } + if o.adoPreviewRun && !o.buildInADO { + errs = append(errs, fmt.Errorf("ado-preview-run flag is not supported when running locally")) + } + + if o.adoPreviewRun && o.adoPreviewRunYamlPath == "" { + errs = append(errs, fmt.Errorf("ado-preview-run-yaml-path flag is missing, please provide path to yaml file with ADO pipeline definition")) + } + + if o.adoPreviewRunYamlPath != "" && !o.adoPreviewRun { + errs = append(errs, fmt.Errorf("ado-preview-run-yaml-path flag is provided, but adoPreviewRun flag is not set to true")) + } + return errutil.NewAggregate(errs) } @@ -643,6 +680,8 @@ func (o *options) gatherOptions(flagSet *flag.FlagSet) *flag.FlagSet { flagSet.BoolVar(&o.signOnly, "sign-only", false, "Only sign the image, do not build it") flagSet.Var(&o.imagesToSign, "images-to-sign", "Comma-separated list of images to sign. Only used when sign-only flag is set") flagSet.BoolVar(&o.buildInADO, "build-in-ado", false, "Build in Azure DevOps pipeline environment") + flagSet.BoolVar(&o.adoPreviewRun, "ado-preview-run", false, "Trigger ADO pipeline in preview mode") + flagSet.StringVar(&o.adoPreviewRunYamlPath, "ado-preview-run-yaml-path", "", "Path to yaml file with ADO pipeline definition to be used in preview mode") flagSet.BoolVar(&o.parseTagsOnly, "parse-tags-only", false, "Only parse tags and print them to stdout") return flagSet @@ -715,15 +754,16 @@ func main() { fmt.Println(err) os.Exit(1) } - fmt.Printf("%s", jsonTags) + fmt.Printf("%s\n", jsonTags) os.Exit(0) } if o.buildInADO { err = buildInADO(o) if err != nil { - fmt.Printf("Image build failed with error: %s", err) + fmt.Printf("Image build failed with error: %s\n", err) os.Exit(1) } + os.Exit(0) } err = buildLocally(o) if err != nil { diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 18dfbeb78375..5e6959d91c70 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -566,3 +566,62 @@ type mockSigner struct { func (m *mockSigner) Sign(images []string) error { return m.signFunc(images) } + +// TODO: add tests for functions related to execution in ado. +// Test copied from pkg/azuredevops/pipelines/pipelines_test.go, rewrite to run it here. +// Describe("Run", func() { +// var ( +// templateParams map[string]string +// runPipelineArgs adoPipelines.RunPipelineArgs +// ) +// +// BeforeEach(func() { +// templateParams = map[string]string{"param1": "value1", "param2": "value2"} +// runPipelineArgs = adoPipelines.RunPipelineArgs{ +// Project: &adoConfig.ADOProjectName, +// PipelineId: &adoConfig.ADOPipelineID, +// RunParameters: &adoPipelines.RunPipelineParameters{ +// PreviewRun: ptr.To(false), +// TemplateParameters: &templateParams, +// }, +// PipelineVersion: &adoConfig.ADOPipelineVersion, +// } +// }) +// +// It("should run the pipeline", func() { +// mockRun := &adoPipelines.Run{Id: ptr.To(123)} +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) +// +// run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) +// Expect(err).ToNot(HaveOccurred()) +// Expect(run.Id).To(Equal(ptr.To(123))) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// +// It("should handle ADO client error", func() { +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(nil, fmt.Errorf("ADO client error")) +// +// _, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) +// Expect(err).To(HaveOccurred()) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// +// It("should run the pipeline in preview mode", func() { +// finalYaml := "pipeline:\n stages:\n - stage: Build\n jobs:\n - job: Build\n steps:\n - script: echo Hello, world!\n displayName: 'Run a one-line script'" +// runPipelineArgs.RunParameters.PreviewRun = ptr.To(true) +// mockRun := &adoPipelines.Run{Id: ptr.To(123), FinalYaml: &finalYaml} +// mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) +// +// run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig, pipelines.PipelinePreviewRun) +// Expect(err).ToNot(HaveOccurred()) +// Expect(run.Id).To(Equal(ptr.To(123))) +// Expect(run.FinalYaml).To(Equal(&finalYaml)) +// mockADOClient.AssertCalled(t, "RunPipeline", ctx, runPipelineArgs) +// mockADOClient.AssertNumberOfCalls(t, "RunPipeline", 1) +// mockADOClient.AssertExpectations(GinkgoT()) +// }) +// }) diff --git a/pkg/azuredevops/pipelines/pipelines.go b/pkg/azuredevops/pipelines/pipelines.go index 896b057a4e7c..a12976c404c1 100644 --- a/pkg/azuredevops/pipelines/pipelines.go +++ b/pkg/azuredevops/pipelines/pipelines.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "time" adov7 "github.com/microsoft/azure-devops-go-api/azuredevops/v7" @@ -109,7 +110,7 @@ func GetRunLogs(ctx context.Context, buildClient BuildClient, httpClient HTTPCli return string(body), nil } -func Run(ctx context.Context, adoClient Client, templateParameters map[string]string, adoConfig Config) (*pipelines.Run, error) { +func NewRunPipelineArgs(templateParameters map[string]string, adoConfig Config, pipelineRunArgs ...RunPipelineArgsOptions) (pipelines.RunPipelineArgs, error) { adoRunPipelineArgs := pipelines.RunPipelineArgs{ Project: &adoConfig.ADOProjectName, PipelineId: &adoConfig.ADOPipelineID, @@ -121,7 +122,28 @@ func Run(ctx context.Context, adoClient Client, templateParameters map[string]st if adoConfig.ADOPipelineVersion != 0 { adoRunPipelineArgs.PipelineVersion = &adoConfig.ADOPipelineVersion } + for _, arg := range pipelineRunArgs { + err := arg(&adoRunPipelineArgs) + if err != nil { + return pipelines.RunPipelineArgs{}, fmt.Errorf("failed setting pipeline run args, err: %w", err) + } + } // TODO: use structured logging with debug severity - fmt.Printf("Using TemplateParameters: %+v\n", adoRunPipelineArgs.RunParameters.TemplateParameters) - return adoClient.RunPipeline(ctx, adoRunPipelineArgs) + // fmt.Printf("Using TemplateParameters: %+v\n", adoRunPipelineArgs.RunParameters.TemplateParameters) + return adoRunPipelineArgs, nil +} + +type RunPipelineArgsOptions func(*pipelines.RunPipelineArgs) error + +func PipelinePreviewRun(overrideYamlPath string) func(args *pipelines.RunPipelineArgs) error { + return func(args *pipelines.RunPipelineArgs) error { + args.RunParameters.PreviewRun = ptr.To(true) + data, err := os.ReadFile(overrideYamlPath) + if err != nil { + return fmt.Errorf("failed reading override yaml file, err: %w", err) + } + overrideYaml := string(data) + args.RunParameters.YamlOverride = &overrideYaml + return nil + } } diff --git a/pkg/azuredevops/pipelines/pipelines_test.go b/pkg/azuredevops/pipelines/pipelines_test.go index d0414ef59eaa..b9b368fb9473 100644 --- a/pkg/azuredevops/pipelines/pipelines_test.go +++ b/pkg/azuredevops/pipelines/pipelines_test.go @@ -1,6 +1,7 @@ package pipelines_test import ( + "os" "time" . "github.com/onsi/ginkgo/v2" @@ -13,7 +14,7 @@ import ( "strings" "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines" - "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" + pipelinesMocks "github.com/kyma-project/test-infra/pkg/azuredevops/pipelines/mocks" "github.com/microsoft/azure-devops-go-api/azuredevops/v7/build" adoPipelines "github.com/microsoft/azure-devops-go-api/azuredevops/v7/pipelines" @@ -32,7 +33,7 @@ func (t ginkgoT) Cleanup(f func()) { var _ = Describe("Pipelines", func() { var ( ctx context.Context - mockADOClient *pipelinesmocks.MockClient + mockADOClient *pipelinesMocks.MockClient adoConfig pipelines.Config t ginkgoT ) @@ -41,7 +42,7 @@ var _ = Describe("Pipelines", func() { ctx = context.Background() t = ginkgoT{} t.GinkgoTInterface = GinkgoT() - mockADOClient = pipelinesmocks.NewMockClient(t) + mockADOClient = pipelinesMocks.NewMockClient(t) adoConfig = pipelines.Config{ ADOOrganizationURL: "https://dev.azure.com", ADOProjectName: "example-project", @@ -106,15 +107,15 @@ var _ = Describe("Pipelines", func() { Describe("GetRunLogs", func() { var ( - mockBuildClient *pipelinesmocks.MockBuildClient - mockHTTPClient *pipelinesmocks.MockHTTPClient + mockBuildClient *pipelinesMocks.MockBuildClient + mockHTTPClient *pipelinesMocks.MockHTTPClient getBuildLogsArgs build.GetBuildLogsArgs mockBuildLogs *[]build.BuildLog ) BeforeEach(func() { - mockBuildClient = pipelinesmocks.NewMockBuildClient(t) - mockHTTPClient = pipelinesmocks.NewMockHTTPClient(t) + mockBuildClient = pipelinesMocks.NewMockBuildClient(t) + mockHTTPClient = pipelinesMocks.NewMockHTTPClient(t) getBuildLogsArgs = build.GetBuildLogsArgs{ Project: &adoConfig.ADOProjectName, BuildId: ptr.To(42), @@ -175,41 +176,116 @@ var _ = Describe("Pipelines", func() { }) }) - Describe("Run", func() { + Describe("NewRunPipelineArgs", func() { var ( - templateParams map[string]string - runPipelineArgs adoPipelines.RunPipelineArgs + templateParameters map[string]string + pipelineRunArgs []pipelines.RunPipelineArgsOptions ) BeforeEach(func() { - templateParams = map[string]string{"param1": "value1", "param2": "value2"} - runPipelineArgs = adoPipelines.RunPipelineArgs{ - Project: &adoConfig.ADOProjectName, - PipelineId: &adoConfig.ADOPipelineID, + templateParameters = map[string]string{ + "key1": "value1", + "key2": "value2", + } + }) + + Context("when NewRunPipelineArgs is successful", func() { + It("should return the correct PipelineArgs and no error", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.Project).To(Equal(&adoConfig.ADOProjectName)) + Expect(pipelineArgs.PipelineId).To(Equal(&adoConfig.ADOPipelineID)) + Expect(pipelineArgs.PipelineVersion).To(Equal(&adoConfig.ADOPipelineVersion)) + Expect(pipelineArgs.RunParameters.TemplateParameters).To(Equal(&templateParameters)) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(false))) + Expect(pipelineArgs).To(BeAssignableToTypeOf(adoPipelines.RunPipelineArgs{})) + }) + Context("when PipelinePreviewRun option is passed", func() { + var dummyOverrideYamlPath = "./dummyOverride.yaml" + BeforeEach(func() { + pipelineRunArgs = []pipelines.RunPipelineArgsOptions{ + pipelines.PipelinePreviewRun(dummyOverrideYamlPath), + } + err := os.WriteFile(dummyOverrideYamlPath, []byte("dummyYamlContent"), 0644) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := os.Remove(dummyOverrideYamlPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should enable preview run and set YamlOverride according to file content", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig, pipelineRunArgs...) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.Project).To(Equal(&adoConfig.ADOProjectName)) + Expect(pipelineArgs.PipelineId).To(Equal(&adoConfig.ADOPipelineID)) + Expect(pipelineArgs.PipelineVersion).To(Equal(&adoConfig.ADOPipelineVersion)) + Expect(pipelineArgs.RunParameters.TemplateParameters).To(Equal(&templateParameters)) + Expect(pipelineArgs).To(BeAssignableToTypeOf(adoPipelines.RunPipelineArgs{})) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + Expect(pipelineArgs.RunParameters.YamlOverride).To(Equal(ptr.To("dummyYamlContent"))) + }) + }) + }) + + Context("when NewRunPipelineArgs fails", func() { + BeforeEach(func() { + pipelineRunArgs = []pipelines.RunPipelineArgsOptions{ + func(args *adoPipelines.RunPipelineArgs) error { + return fmt.Errorf("dummy error") + }, + } + }) + + It("should return an error", func() { + pipelineArgs, err := pipelines.NewRunPipelineArgs(templateParameters, adoConfig, pipelineRunArgs...) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("failed setting pipeline run args, err: dummy error")) + Expect(pipelineArgs).To(BeEquivalentTo(adoPipelines.RunPipelineArgs{})) + }) + }) + }) + + Describe("PipelinePreviewRun", func() { + var ( + dummyOverrideYamlPath = "./dummyOverride.yaml" + err error + pipelineArgs *adoPipelines.RunPipelineArgs + ) + + BeforeEach(func() { + err = os.WriteFile(dummyOverrideYamlPath, []byte("dummyYamlContent"), 0644) + Expect(err).NotTo(HaveOccurred()) + pipelineArgs = &adoPipelines.RunPipelineArgs{ RunParameters: &adoPipelines.RunPipelineParameters{ - PreviewRun: ptr.To(false), - TemplateParameters: &templateParams, + PreviewRun: ptr.To(false), }, - PipelineVersion: &adoConfig.ADOPipelineVersion, } }) - It("should run the pipeline", func() { - mockRun := &adoPipelines.Run{Id: ptr.To(123)} - mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(mockRun, nil) - - run, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) - Expect(err).ToNot(HaveOccurred()) - Expect(run.Id).To(Equal(ptr.To(123))) - mockADOClient.AssertExpectations(GinkgoT()) + AfterEach(func() { + err = os.Remove(dummyOverrideYamlPath) + Expect(err).NotTo(HaveOccurred()) }) - It("should handle ADO client error", func() { - mockADOClient.On("RunPipeline", ctx, runPipelineArgs).Return(nil, fmt.Errorf("ADO client error")) + It("should prepare function that sets PreviewRun to true and reads override yaml", func() { - _, err := pipelines.Run(ctx, mockADOClient, templateParams, adoConfig) - Expect(err).To(HaveOccurred()) - mockADOClient.AssertExpectations(GinkgoT()) + pipelinePreviewRun := pipelines.PipelinePreviewRun(dummyOverrideYamlPath) + + err := pipelinePreviewRun(pipelineArgs) + Expect(err).NotTo(HaveOccurred()) + Expect(pipelineArgs.RunParameters.PreviewRun).To(Equal(ptr.To(true))) + Expect(pipelineArgs.RunParameters.YamlOverride).To(Equal(ptr.To("dummyYamlContent"))) + }) + Context("when the override yaml file does not exist", func() { + It("should return an error", func() { + nonExistentFilePath := "/path/to/non-existent/file.yaml" + pipelinePreviewRun := pipelines.PipelinePreviewRun(nonExistentFilePath) + + err := pipelinePreviewRun(pipelineArgs) + Expect(err).To(HaveOccurred()) + }) }) }) }) diff --git a/pkg/azuredevops/pipelines/templatesParams.go b/pkg/azuredevops/pipelines/templatesParams.go index a2617cb9d237..af5d9f8e3e63 100644 --- a/pkg/azuredevops/pipelines/templatesParams.go +++ b/pkg/azuredevops/pipelines/templatesParams.go @@ -14,6 +14,7 @@ func (e ErrRequiredParamNotSet) Error() string { return "required parameter not set: " + string(e) } +// TODO: Rename, remove Template, as this is are parameters for pipeline execution. // OCIImageBuilderTemplateParams is a map of parameters for OCIImageBuilderTemplate type OCIImageBuilderTemplateParams map[string]string