From 160da4d95708f0f677469f07a187f7b8c72c273e Mon Sep 17 00:00:00 2001 From: Robert Cerven Date: Tue, 5 Nov 2024 23:40:16 +0100 Subject: [PATCH] allow custom user renovate configs, in config maps, there can be one global one for all components, and also custom config for specific components we are also now creating renovate config in json format STONEBLD-2916 Signed-off-by: Robert Cerven --- ...onent_dependency_update_controller_test.go | 253 ++++++++++++--- controllers/renovate_util.go | 298 +++++++++++------- controllers/suite_util_test.go | 27 ++ pkg/k8s/credentials.go | 5 +- 4 files changed, 412 insertions(+), 171 deletions(-) diff --git a/controllers/component_dependency_update_controller_test.go b/controllers/component_dependency_update_controller_test.go index 66f01636..69461a64 100644 --- a/controllers/component_dependency_update_controller_test.go +++ b/controllers/component_dependency_update_controller_test.go @@ -18,7 +18,6 @@ package controllers import ( "context" "encoding/base64" - "encoding/json" "fmt" "strings" "time" @@ -231,10 +230,6 @@ var _ = Describe("Component nudge controller", func() { for _, renovateConfig := range renovateConfigMaps { if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { renovateConfigMapFound = true - - for _, renovateConfigData := range renovateConfig.Data { - Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username"`)).Should(BeTrue()) - } } if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-ca") { renovateCaConfigMapFound = true @@ -245,6 +240,8 @@ var _ = Describe("Component nudge controller", func() { renovatePipelines := getRenovatePipelineRunList() Expect(len(renovatePipelines)).Should(Equal(1)) + renovateCommand := strings.Join(renovatePipelines[0].Spec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Command, ";") + Expect(strings.Contains(renovateCommand, `'username':'image_repo_username'`)).Should(BeTrue()) caMounted := false for _, volumeMount := range renovatePipelines[0].Spec.PipelineSpec.Tasks[0].TaskSpec.TaskSpec.Steps[0].VolumeMounts { if volumeMount.Name == CaVolumeMountName { @@ -298,15 +295,10 @@ var _ = Describe("Component nudge controller", func() { return renovatePipelinesCreated == 1 && renovateConfigsCreated == 2 && failureCount == 0 }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) - renovateConfigMaps := getRenovateConfigMapList() - Expect(len(renovateConfigMaps)).Should(Equal(2)) - for _, renovateConfig := range renovateConfigMaps { - if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { - for _, renovateConfigData := range renovateConfig.Data { - Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username_partial"`)).Should(BeTrue()) - } - } - } + renovatePipelines := getRenovatePipelineRunList() + Expect(len(renovatePipelines)).Should(Equal(1)) + renovateCommand := strings.Join(renovatePipelines[0].Spec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Command, ";") + Expect(strings.Contains(renovateCommand, `'username':'image_repo_username_partial'`)).Should(BeTrue()) }) It("Test build performs nudge on success, image repository partial auth from multiple secrets", func() { @@ -368,15 +360,10 @@ var _ = Describe("Component nudge controller", func() { return renovatePipelinesCreated == 1 && renovateConfigsCreated == 2 && failureCount == 0 }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) - renovateConfigMaps := getRenovateConfigMapList() - Expect(len(renovateConfigMaps)).Should(Equal(2)) - for _, renovateConfig := range renovateConfigMaps { - if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { - for _, renovateConfigData := range renovateConfig.Data { - Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username_2"`)).Should(BeTrue()) - } - } - } + renovatePipelines := getRenovatePipelineRunList() + Expect(len(renovatePipelines)).Should(Equal(1)) + renovateCommand := strings.Join(renovatePipelines[0].Spec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Command, ";") + Expect(strings.Contains(renovateCommand, `'username':'image_repo_username_2'`)).Should(BeTrue()) }) It("Test stale pipeline not nudged", func() { @@ -416,19 +403,186 @@ var _ = Describe("Component nudge controller", func() { }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) }) + It("Test build performs nudge on success, only global renovate config provided", func() { + customConfigMapName := types.NamespacedName{Namespace: UserNamespace, Name: GlobalRenovateConfigName} + customConfigString := `{"username":"globalconfiguserjson"}` + customConfigMapData := map[string]string{ConfigKeyJson: customConfigString} + createCustomRenovateConfigMap(customConfigMapName, customConfigMapData) + customConfigType := "json" + + createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) + Eventually(func() bool { + pr := getPipelineRun("test-pipeline-1", UserNamespace) + return controllerutil.ContainsFinalizer(pr, NudgeFinalizer) + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + pr := getPipelineRun("test-pipeline-1", UserNamespace) + pr.Status.SetCondition(&apis.Condition{ + Type: apis.ConditionSucceeded, + Status: "True", + LastTransitionTime: apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, + }) + pr.Status.Results = []tektonapi.PipelineRunResult{ + {Name: ImageDigestParamName, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "sha256:12345"}}, + {Name: ImageUrl, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "quay.io.foo/bar:latest"}}, + } + pr.Status.CompletionTime = &metav1.Time{Time: time.Now()} + Expect(k8sClient.Status().Update(ctx, pr)).Should(BeNil()) + + Eventually(func() bool { + // check that no nudgeerror event was reported + failureCount := getRenovateFailedEventCount() + // check that renovate config was created + renovateConfigsCreated := len(getRenovateConfigMapList()) + // check that renovate pipeline run was created + renovatePipelinesCreated := len(getRenovatePipelineRunList()) + return renovatePipelinesCreated == 1 && renovateConfigsCreated == 2 && failureCount == 0 + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(2)) + + for _, renovateConfig := range renovateConfigMaps { + if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { + for key, val := range renovateConfig.Data { + Expect(val).Should(Equal(customConfigString)) + Expect(strings.HasSuffix(key, customConfigType)) + } + break + } + } + deleteConfigMap(customConfigMapName) + }) + + It("Test build performs nudge on success, only component renovate config provided", func() { + customConfigName := fmt.Sprintf("nudging-renovate-config-%s", Operator1) + customConfigMapName1 := types.NamespacedName{Namespace: UserNamespace, Name: customConfigName} + customConfigString := `{"username":"componentconfiguserjs"}` + customConfigMapData := map[string]string{ConfigKeyJs: customConfigString} + createCustomRenovateConfigMap(customConfigMapName1, customConfigMapData) + customConfigName = fmt.Sprintf("nudging-renovate-config-%s", Operator2) + customConfigMapName2 := types.NamespacedName{Namespace: UserNamespace, Name: customConfigName} + createCustomRenovateConfigMap(customConfigMapName2, customConfigMapData) + customConfigType := "js" + + createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) + Eventually(func() bool { + pr := getPipelineRun("test-pipeline-1", UserNamespace) + return controllerutil.ContainsFinalizer(pr, NudgeFinalizer) + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + pr := getPipelineRun("test-pipeline-1", UserNamespace) + pr.Status.SetCondition(&apis.Condition{ + Type: apis.ConditionSucceeded, + Status: "True", + LastTransitionTime: apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, + }) + pr.Status.Results = []tektonapi.PipelineRunResult{ + {Name: ImageDigestParamName, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "sha256:12345"}}, + {Name: ImageUrl, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "quay.io.foo/bar:latest"}}, + } + pr.Status.CompletionTime = &metav1.Time{Time: time.Now()} + Expect(k8sClient.Status().Update(ctx, pr)).Should(BeNil()) + + Eventually(func() bool { + // check that no nudgeerror event was reported + failureCount := getRenovateFailedEventCount() + // check that renovate config was created + renovateConfigsCreated := len(getRenovateConfigMapList()) + // check that renovate pipeline run was created + renovatePipelinesCreated := len(getRenovatePipelineRunList()) + return renovatePipelinesCreated == 1 && renovateConfigsCreated == 2 && failureCount == 0 + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(2)) + + for _, renovateConfig := range renovateConfigMaps { + if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { + for key, val := range renovateConfig.Data { + Expect(val).Should(Equal(customConfigString)) + Expect(strings.HasSuffix(key, customConfigType)) + } + break + } + } + deleteConfigMap(customConfigMapName1) + deleteConfigMap(customConfigMapName2) + }) + + It("Test build performs nudge on success, global and component renovate config provided", func() { + customConfigMapName := types.NamespacedName{Namespace: UserNamespace, Name: GlobalRenovateConfigName} + customConfigString := `{"username":"globalconfiguserjson"}` + customConfigMapData := map[string]string{ConfigKeyJson: customConfigString} + createCustomRenovateConfigMap(customConfigMapName, customConfigMapData) + + customConfigName := fmt.Sprintf("nudging-renovate-config-%s", Operator1) + customConfigMapName1 := types.NamespacedName{Namespace: UserNamespace, Name: customConfigName} + customConfigString1 := `{"username":"componentconfiguserjs"}` + customConfigString2 := `{"username":"componentconfiguserjson"}` + customConfigMapData = map[string]string{ConfigKeyJs: customConfigString1, ConfigKeyJson: customConfigString2} + createCustomRenovateConfigMap(customConfigMapName1, customConfigMapData) + customConfigName = fmt.Sprintf("nudging-renovate-config-%s", Operator2) + customConfigMapName2 := types.NamespacedName{Namespace: UserNamespace, Name: customConfigName} + createCustomRenovateConfigMap(customConfigMapName2, customConfigMapData) + customConfigType := "json" + + createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) + Eventually(func() bool { + pr := getPipelineRun("test-pipeline-1", UserNamespace) + return controllerutil.ContainsFinalizer(pr, NudgeFinalizer) + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + pr := getPipelineRun("test-pipeline-1", UserNamespace) + pr.Status.SetCondition(&apis.Condition{ + Type: apis.ConditionSucceeded, + Status: "True", + LastTransitionTime: apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, + }) + pr.Status.Results = []tektonapi.PipelineRunResult{ + {Name: ImageDigestParamName, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "sha256:12345"}}, + {Name: ImageUrl, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "quay.io.foo/bar:latest"}}, + } + pr.Status.CompletionTime = &metav1.Time{Time: time.Now()} + Expect(k8sClient.Status().Update(ctx, pr)).Should(BeNil()) + + Eventually(func() bool { + // check that no nudgeerror event was reported + failureCount := getRenovateFailedEventCount() + // check that renovate config was created + renovateConfigsCreated := len(getRenovateConfigMapList()) + // check that renovate pipeline run was created + renovatePipelinesCreated := len(getRenovatePipelineRunList()) + return renovatePipelinesCreated == 1 && renovateConfigsCreated == 2 && failureCount == 0 + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(2)) + + for _, renovateConfig := range renovateConfigMaps { + if strings.HasPrefix(renovateConfig.ObjectMeta.Name, "renovate-pipeline") { + for key, val := range renovateConfig.Data { + Expect(val).Should(Equal(customConfigString2)) + Expect(strings.HasSuffix(key, customConfigType)) + } + break + } + } + deleteConfigMap(customConfigMapName) + deleteConfigMap(customConfigMapName1) + deleteConfigMap(customConfigMapName2) + }) }) Context("Test nudge failure handling", func() { It("Test single failure results in retry", func() { failures = 1 // mock function to produce error - GenerateRenovateConfigForNudge = func(target updateTarget, buildResult *BuildResult) (string, error) { + + GenerateRenovateConfigForNudge = func(target updateTarget, buildResult *BuildResult) (RenovateConfig, error) { if failures == 0 { log.Info("components nudged") - return "\nrenovate_config\nrenovate_line1\nrenovate_line2\n", nil + return RenovateConfig{}, nil } failures = failures - 1 - return "", fmt.Errorf("failure") + return RenovateConfig{}, fmt.Errorf("failure") } createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) @@ -469,13 +623,13 @@ var _ = Describe("Component nudge controller", func() { It("Test retries exceeded", func() { failures = 10 // mock function to produce error - GenerateRenovateConfigForNudge = func(target updateTarget, buildResult *BuildResult) (string, error) { + GenerateRenovateConfigForNudge = func(target updateTarget, buildResult *BuildResult) (RenovateConfig, error) { if failures == 0 { log.Info("components nudged") - return "\nrenovate_config\nrenovate_line1\nrenovate_line2\n", nil + return RenovateConfig{}, nil } failures = failures - 1 - return "", fmt.Errorf("failure") + return RenovateConfig{}, fmt.Errorf("failure") } createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) @@ -521,13 +675,14 @@ var _ = Describe("Component nudge controller", func() { imageRepositoryHost := "quay.io" imageRepositoryUsername := "repository_username" fileMatches := "file1, file2, file3" - fileMatchesList := `["file1","file2","file3"]` + fileMatchesList := []string{"file1", "file2", "file3"} repositories := []renovateRepository{{BaseBranches: []string{"base_branch"}, Repository: "some_repository/something"}} - repositoriesData, _ := json.Marshal(repositories) buildImageRepository := "quay.io/testorg/testimage" builtImageTag := "a8dce08dbdf290e5d616a83672ad3afcb4b455ef" digest := "sha256:716be32f12f0dd31adbab1f57e9d0b87066e03de51c89dce8ffb397fbac92314" distributionRepositories := []string{"registry.redhat.com/some-product", "registry.redhat.com/other-product"} + matchPackageNames := []string{buildImageRepository, distributionRepositories[0], distributionRepositories[1]} + registryAliases := map[string]string{distributionRepositories[0]: buildImageRepository, distributionRepositories[1]: buildImageRepository} buildResult := BuildResult{ BuiltImageRepository: buildImageRepository, @@ -549,28 +704,22 @@ var _ = Describe("Component nudge controller", func() { ImageRepositoryUsername: imageRepositoryUsername, } - result, err := generateRenovateConfigForNudge(renovateTarget, &buildResult) + resultConfig, err := generateRenovateConfigForNudge(renovateTarget, &buildResult) Expect(err).Should(Succeed()) - - Expect(strings.Contains(result, fmt.Sprintf(`platform: "%s"`, gitProvider))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`username: "%s"`, gitUsername))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`gitAuthor: "%s"`, gitAuthor))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`repositories: %s`, repositoriesData))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"username": "%s"`, imageRepositoryUsername))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"matchHost": "%s"`, imageRepositoryHost))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`endpoint: "%s"`, gitEndpoint))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"fileMatch": %s`, fileMatchesList))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"currentValueTemplate": "%s"`, buildResult.BuiltImageTag))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"depNameTemplate": "%s"`, buildImageRepository))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`groupName: "Component Update %s"`, componentName))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`branchName: "konflux/component-updates/%s"`, componentName))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`commitMessageTopic: "%s"`, componentName))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`followTag: "%s"`, builtImageTag))).Should(BeTrue()) - Expect(strings.Contains(result, fmt.Sprintf(`"matchPackageNames": ["%s","%s","%s"]`, buildImageRepository, distributionRepositories[0], distributionRepositories[1]))).Should(BeTrue()) - registryAlias1 := fmt.Sprintf(`"%s": "%s",`, distributionRepositories[0], buildImageRepository) - registryAlias2 := fmt.Sprintf(`"%s": "%s"`, distributionRepositories[1], buildImageRepository) - Expect(strings.Contains(result, registryAlias1)).Should(BeTrue()) - Expect(strings.Contains(result, registryAlias2)).Should(BeTrue()) + Expect(resultConfig.GitProvider).Should(Equal(gitProvider)) + Expect(resultConfig.Username).Should(Equal(gitUsername)) + Expect(resultConfig.GitAuthor).Should(Equal(gitAuthor)) + Expect(resultConfig.Repositories).Should(Equal(repositories)) + Expect(resultConfig.Endpoint).Should(Equal(gitEndpoint)) + Expect(resultConfig.CustomManagers[0].FileMatch).Should(Equal(fileMatchesList)) + Expect(resultConfig.CustomManagers[0].CurrentValueTemplate).Should(Equal(buildResult.BuiltImageTag)) + Expect(resultConfig.CustomManagers[0].DepNameTemplate).Should(Equal(buildImageRepository)) + Expect(resultConfig.PackageRules[1].GroupName).Should(Equal(fmt.Sprintf("Component Update %s", componentName))) + Expect(resultConfig.PackageRules[1].BranchName).Should(Equal(fmt.Sprintf("konflux/component-updates/%s", componentName))) + Expect(resultConfig.PackageRules[1].CommitMessageTopic).Should(Equal(componentName)) + Expect(resultConfig.PackageRules[1].FollowTag).Should(Equal(builtImageTag)) + Expect(resultConfig.PackageRules[1].MatchPackageNames).Should(Equal(matchPackageNames)) + Expect(resultConfig.RegistryAliases).Should(Equal(registryAliases)) }) }) diff --git a/controllers/renovate_util.go b/controllers/renovate_util.go index 2a863099..fa4af822 100644 --- a/controllers/renovate_util.go +++ b/controllers/renovate_util.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "strings" - "text/template" "time" "github.com/konflux-ci/application-api/api/v1alpha1" @@ -30,14 +29,18 @@ import ( ) const ( - RenovateImageEnvName = "RENOVATE_IMAGE" - DefaultRenovateImageUrl = "quay.io/redhat-appstudio/renovate:v37.74.1" - DefaultRenovateUser = "red-hat-konflux" - CaConfigMapLabel = "config.openshift.io/inject-trusted-cabundle" - CaConfigMapKey = "ca-bundle.crt" - CaFilePath = "tls-ca-bundle.pem" - CaMountPath = "/etc/pki/ca-trust/extracted/pem" - CaVolumeMountName = "trusted-ca" + RenovateImageEnvName = "RENOVATE_IMAGE" + DefaultRenovateImageUrl = "quay.io/redhat-appstudio/renovate:v37.74.1" + DefaultRenovateUser = "red-hat-konflux" + CaConfigMapLabel = "config.openshift.io/inject-trusted-cabundle" + CaConfigMapKey = "ca-bundle.crt" + CaFilePath = "tls-ca-bundle.pem" + CaMountPath = "/etc/pki/ca-trust/extracted/pem" + CaVolumeMountName = "trusted-ca" + GlobalRenovateConfigName = "global-nudging-renovate-config" + ComponentRenovateConfigNamePrefix = "nudging-renovate-config-" + ConfigKeyJson = "config.json" + ConfigKeyJs = "config.js" ) type renovateRepository struct { @@ -66,7 +69,48 @@ type ComponentDependenciesUpdater struct { CredentialProvider *k8s.GitCredentialProvider } -var GenerateRenovateConfigForNudge func(target updateTarget, buildResult *BuildResult) (string, error) = generateRenovateConfigForNudge +type CustomManager struct { + FileMatch []string `json:"fileMatch,omitempty"` + CustomType string `json:"customType"` + DatasourceTemplate string `json:"datasourceTemplate"` + MatchStrings []string `json:"matchStrings"` + CurrentValueTemplate string `json:"currentValueTemplate"` + DepNameTemplate string `json:"depNameTemplate"` +} + +type PackageRule struct { + MatchPackagePatterns []string `json:"matchPackagePatterns,omitempty"` + MatchPackageNames []string `json:"matchPackageNames,omitempty"` + GroupName string `json:"groupName,omitempty"` + BranchName string `json:"branchName,omitempty"` + CommitMessageTopic string `json:"commitMessageTopic,omitempty"` + PRFooter string `json:"prFooter,omitempty"` + RecreateWhen string `json:"recreateWhen,omitempty"` + RebaseWhen string `json:"rebaseWhen,omitempty"` + Enabled bool `json:"enabled"` + FollowTag string `json:"followTag,omitempty"` +} + +type RenovateConfig struct { + GitProvider string `json:"platform"` + Username string `json:"username"` + GitAuthor string `json:"gitAuthor"` + Onboarding bool `json:"onboarding"` + RequireConfig string `json:"requireConfig"` + Repositories []renovateRepository `json:"repositories"` + EnabledManagers []string `json:"enabledManagers"` + Endpoint string `json:"endpoint"` + CustomManagers []CustomManager `json:"customManagers,omitempty"` + RegistryAliases map[string]string `json:"registryAliases,omitempty"` + PackageRules []PackageRule `json:"packageRules,omitempty"` + ForkProcessing string `json:"forkProcessing"` + Extends []string `json:"extends"` + DependencyDashboard bool `json:"dependencyDashboard"` +} + +var DisableAllPackageRules = PackageRule{MatchPackagePatterns: []string{"*"}, Enabled: false} + +var GenerateRenovateConfigForNudge func(target updateTarget, buildResult *BuildResult) (RenovateConfig, error) = generateRenovateConfigForNudge func NewComponentDependenciesUpdater(client client.Client, scheme *runtime.Scheme, eventRecorder record.EventRecorder) *ComponentDependenciesUpdater { return &ComponentDependenciesUpdater{Client: client, Scheme: scheme, EventRecorder: eventRecorder, CredentialProvider: k8s.NewGitCredentialProvider(client)} @@ -246,116 +290,68 @@ func (u ComponentDependenciesUpdater) GetUpdateTargetsGithubApp(ctx context.Cont } // generateRenovateConfigForNudge This method returns renovate config for target -func generateRenovateConfigForNudge(target updateTarget, buildResult *BuildResult) (string, error) { - repositoriesData, _ := json.Marshal(target.Repositories) +func generateRenovateConfigForNudge(target updateTarget, buildResult *BuildResult) (RenovateConfig, error) { fileMatchParts := strings.Split(buildResult.FileMatches, ",") for i := range fileMatchParts { fileMatchParts[i] = strings.TrimSpace(fileMatchParts[i]) } - fileMatch, err := json.Marshal(fileMatchParts) - if err != nil { - return "", err - } - body := ` - {{with $root := .}} - module.exports = { - platform: "{{.GitProvider}}", - username: "{{.Username}}", - gitAuthor: "{{.GitAuthor}}", - onboarding: false, - requireConfig: "ignored", - repositories: {{.Repositories}}, - enabledManagers: "regex", - endpoint: "{{.Endpoint}}", - customManagers: [ - { - "fileMatch": {{.FileMatches}}, - "customType": "regex", - "datasourceTemplate": "docker", - "matchStrings": [ - "{{.BuiltImageRepository}}(:.*)?@(?sha256:[a-f0-9]+)" - {{range .DistributionRepositories}},"{{.}}(:.*)?@(?sha256:[a-f0-9]+)"{{end}} - ], - "currentValueTemplate": "{{.BuiltImageTag}}", - "depNameTemplate": "{{.BuiltImageRepository}}", - } - ], - registryAliases: { - {{range $index, $repo := .DistributionRepositories}}{{if $index}},{{end}} - "{{$repo}}": "{{$.BuiltImageRepository}}"{{end}} - }, - packageRules: [ - { - matchPackagePatterns: ["*"], - enabled: false - }, - { - "matchPackageNames": ["{{.BuiltImageRepository}}"{{range .DistributionRepositories}},"{{.}}"{{end}}], - groupName: "Component Update {{.ComponentName}}", - branchName: "konflux/component-updates/{{.ComponentName}}", - commitMessageTopic: "{{.ComponentName}}", - prFooter: "To execute skipped test pipelines write comment ` + "`/ok-to-test`" + `", - recreateWhen: "always", - rebaseWhen: "behind-base-branch", - enabled: true, - followTag: "{{.BuiltImageTag}}" - } - ], - hostRules: [ - { - "matchHost": "{{.ImageRepositoryHost}}", - "username": "{{.ImageRepositoryUsername}}", - "password": process.env.RENOVATE_REPO_PASS, - } - ], - forkProcessing: "enabled", - extends: [":gitSignOff"], - dependencyDashboard: false - } - {{end}} - ` - - data := struct { - GitProvider string - Username string - GitAuthor string - Endpoint string - ComponentName string - Repositories string - BuiltImageRepository string - BuiltImageTag string - Digest string - DistributionRepositories []string - FileMatches string - ImageRepositoryHost string - ImageRepositoryUsername string - }{ - GitProvider: target.GitProvider, - Username: target.Username, - GitAuthor: target.GitAuthor, - Endpoint: target.Endpoint, - ComponentName: buildResult.Component.Name, - Repositories: string(repositoriesData), - BuiltImageRepository: buildResult.BuiltImageRepository, - BuiltImageTag: buildResult.BuiltImageTag, - Digest: buildResult.Digest, - DistributionRepositories: buildResult.DistributionRepositories, - FileMatches: string(fileMatch), - ImageRepositoryHost: target.ImageRepositoryHost, - ImageRepositoryUsername: target.ImageRepositoryUsername, - } + var matchStrings []string + var registryAliases = make(map[string]string) + var customManagers []CustomManager + var packageRules []PackageRule + var matchPackageNames []string + matchStrings = append(matchStrings, buildResult.BuiltImageRepository+"(:.*)?@(?sha256:[a-f0-9]+)") + matchPackageNames = append(matchPackageNames, buildResult.BuiltImageRepository) + + for _, drepositiry := range buildResult.DistributionRepositories { + matchStrings = append(matchStrings, drepositiry+"(:.*)?@(?sha256:[a-f0-9]+)") + matchPackageNames = append(matchPackageNames, drepositiry) + registryAliases[drepositiry] = buildResult.BuiltImageRepository - configTemplate, err := template.New("renovate").Parse(body) - if err != nil { - return "", err } - build := strings.Builder{} - err = configTemplate.Execute(&build, data) - if err != nil { - return "", err + + customManagers = append(customManagers, CustomManager{ + FileMatch: fileMatchParts, + CustomType: "regex", + DatasourceTemplate: "docker", + MatchStrings: matchStrings, + CurrentValueTemplate: buildResult.BuiltImageTag, + DepNameTemplate: buildResult.BuiltImageRepository, + }) + + packageRules = append(packageRules, DisableAllPackageRules) + packageRules = append(packageRules, PackageRule{ + MatchPackageNames: matchPackageNames, + GroupName: fmt.Sprintf("Component Update %s", buildResult.Component.Name), + BranchName: fmt.Sprintf("konflux/component-updates/%s", buildResult.Component.Name), + CommitMessageTopic: buildResult.Component.Name, + PRFooter: "To execute skipped test pipelines write comment `/ok-to-test`", + RecreateWhen: "always", + RebaseWhen: "behind-base-branch", + Enabled: true, + FollowTag: buildResult.BuiltImageTag, + }) + + renovateConfig := RenovateConfig{ + GitProvider: target.GitProvider, + Username: target.Username, + GitAuthor: target.GitAuthor, + Onboarding: false, + RequireConfig: "ignored", + Repositories: target.Repositories, + // was 'regex' before but because: https://docs.renovatebot.com/configuration-options/#enabledmanagers + EnabledManagers: []string{"custom.regex"}, + Endpoint: target.Endpoint, + CustomManagers: customManagers, + RegistryAliases: registryAliases, + PackageRules: packageRules, + ForkProcessing: "enabled", + Extends: []string{":gitSignOff"}, + DependencyDashboard: false, } - return build.String(), nil + + return renovateConfig, nil } // CreateRenovaterPipeline will create a renovate pipeline in the user namespace, to update component dependencies. @@ -380,21 +376,73 @@ func (u ComponentDependenciesUpdater) CreateRenovaterPipeline(ctx context.Contex secretTokens := map[string]string{} configmaps := map[string]string{} renovateCmds := []string{} + globalConfigString := "" + globalConfigType := "" + + allUserConfigMaps := &corev1.ConfigMapList{} + if err := u.Client.List(ctx, allUserConfigMaps, client.InNamespace(namespace)); err != nil { + return fmt.Errorf("failed to list config maps in %s namespace: %w", namespace, err) + } + for _, userConfigMap := range allUserConfigMaps.Items { + if userConfigMap.Name == GlobalRenovateConfigName { + globalConfigString, globalConfigType = getConfigAndTypeFromConfigMap(userConfigMap) + break + } + } + for _, target := range targets { randomStr1 := RandomString(5) randomStr2 := RandomString(10) randomStr3 := RandomString(10) secretTokens[randomStr2] = target.Token secretTokens[randomStr3] = target.ImageRepositoryPassword - config, err := GenerateRenovateConfigForNudge(target, buildResult) - if err != nil { - return err + componentConfigName := fmt.Sprintf("%s%s", ComponentRenovateConfigNamePrefix, target.ComponentName) + componentConfigString := "" + componentConfigType := "" + configString := "" + configType := "json" + + for _, userConfigMap := range allUserConfigMaps.Items { + if userConfigMap.Name == componentConfigName { + componentConfigString, componentConfigType = getConfigAndTypeFromConfigMap(userConfigMap) + break + } + } + + if globalConfigString != "" || componentConfigString != "" { + if componentConfigString != "" { + configString = componentConfigString + configType = componentConfigType + log.Info("will use custom renovate config for component", "name", componentConfigName, "type", configType) + } else { + configString = globalConfigString + configType = globalConfigType + log.Info("will use custom global renovate config", "name", GlobalRenovateConfigName, "type", configType) + } + + } else { + log.Info("will generate renovate config, no custom ones are present") + renovateConfig, err := GenerateRenovateConfigForNudge(target, buildResult) + if err != nil { + return err + } + + config, err := json.Marshal(renovateConfig) + if err != nil { + return err + } + configString = string(config) } - configmaps[fmt.Sprintf("%s-%s.js", target.ComponentName, randomStr1)] = config - log.Info(fmt.Sprintf("Creating renovate config map entry for %s component with length %d and value %s", target.ComponentName, len(config), config)) + log.Info(fmt.Sprintf("Creating renovate config map entry for %s component with length %d and value %s", target.ComponentName, len(configString), configString)) + + configmaps[fmt.Sprintf("%s-%s.%s", target.ComponentName, randomStr1, configType)] = configString + hostRules := fmt.Sprintf("\"[{'matchHost':'%s','username':'%s','password':'${TOKEN_%s}'}]\"", target.ImageRepositoryHost, target.ImageRepositoryUsername, randomStr3) + + // we are passing host rules via variable, because we can't resolve variable in json config + // also this way we can use custom provided config without any modifications renovateCmds = append(renovateCmds, - fmt.Sprintf("RENOVATE_PR_HOURLY_LIMIT=0 RENOVATE_PR_CONCURRENT_LIMIT=0 RENOVATE_TOKEN=$TOKEN_%s RENOVATE_REPO_PASS=$TOKEN_%s RENOVATE_CONFIG_FILE=/configs/%s-%s.js renovate", randomStr2, randomStr3, target.ComponentName, randomStr1), + fmt.Sprintf("RENOVATE_PR_HOURLY_LIMIT=0 RENOVATE_PR_CONCURRENT_LIMIT=0 RENOVATE_TOKEN=$TOKEN_%s RENOVATE_CONFIG_FILE=/configs/%s-%s.%s RENOVATE_HOST_RULES=%s renovate", randomStr2, target.ComponentName, randomStr1, configType, hostRules), ) } if len(renovateCmds) == 0 { @@ -563,3 +611,17 @@ func (u ComponentDependenciesUpdater) CreateRenovaterPipeline(ctx context.Contex return nil } + +func getConfigAndTypeFromConfigMap(configMap corev1.ConfigMap) (string, string) { + config, exists := configMap.Data[ConfigKeyJson] + if exists && len(config) > 0 { + return config, "json" + } + + config, exists = configMap.Data[ConfigKeyJs] + if exists && len(config) > 0 { + return config, "js" + } + + return "", "" +} diff --git a/controllers/suite_util_test.go b/controllers/suite_util_test.go index ff0f09d2..12399d48 100644 --- a/controllers/suite_util_test.go +++ b/controllers/suite_util_test.go @@ -592,6 +592,17 @@ func createCAConfigMap(configMapKey types.NamespacedName) { } } +func createCustomRenovateConfigMap(configMapKey types.NamespacedName, configMapData map[string]string) { + caConfigMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: configMapKey.Name, Namespace: configMapKey.Namespace}, + Data: configMapData, + } + + if err := k8sClient.Create(ctx, &caConfigMap); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } +} + func waitServiceAccount(serviceAccountKey types.NamespacedName) corev1.ServiceAccount { serviceAccount := corev1.ServiceAccount{} Eventually(func() bool { @@ -635,3 +646,19 @@ func deletePipelineServiceAccount(namespace string) { pipelineServiceAccountkey := types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: namespace} deleteServiceAccount((pipelineServiceAccountkey)) } + +func deleteConfigMap(configMapKey types.NamespacedName) { + configMap := corev1.ConfigMap{} + if err := k8sClient.Get(ctx, configMapKey, &configMap); err != nil { + if k8sErrors.IsNotFound(err) { + return + } + Fail(err.Error()) + } + if err := k8sClient.Delete(ctx, &configMap); err != nil && !k8sErrors.IsNotFound(err) { + Fail(err.Error()) + } + Eventually(func() bool { + return k8sErrors.IsNotFound(k8sClient.Get(ctx, configMapKey, &configMap)) + }, timeout, interval).Should(BeTrue()) +} diff --git a/pkg/k8s/credentials.go b/pkg/k8s/credentials.go index 286b2daa..b323d479 100644 --- a/pkg/k8s/credentials.go +++ b/pkg/k8s/credentials.go @@ -127,7 +127,7 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets for index, secret := range secrets { repositoryAnnotation, exists := secret.Annotations[ScmSecretRepositoryAnnotation] - log.Info("found secret", "secret", secret.Name, "repositoryAnnotation", repositoryAnnotation, "exists", exists) + log.Info("found secret", "repositoryAnnotation", repositoryAnnotation, "exists", exists, "secret", secret.Name) if !exists || repositoryAnnotation == "" { hostOnlySecrets = append(hostOnlySecrets, secret) continue @@ -142,6 +142,7 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets // Direct repository match, return secret log.Info("checking for direct match", "componentRepository", componentRepository, "secretRepositories", secretRepositories) if slices.Contains(secretRepositories, componentRepository) { + log.Info("secret repository is direct match", "secret", secret.Name) return &secret } log.Info("no direct match found", "componentRepository", componentRepository, "secretRepositories", secretRepositories) @@ -176,5 +177,7 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets bestIndex = i } } + + log.Info("Using host only secret based on potential matches", "name", secrets[bestIndex].Name) return &secrets[bestIndex] }