diff --git a/README.md b/README.md index 4ed444c8..c776b38c 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,10 @@ Run the following commands for available flags and subcommands: ### Custom Packages -Idpbuilder supports specifying custom packages using the flag `--package-dir` flag. This flag expects a directory containing ArgoCD application files. +Idpbuilder supports specifying custom packages using the flag `--package-dir` flag. +This flag expects a directory (local or remote) containing ArgoCD application files. +In case of a remote directory, it must be a directory in a git repository, +and the URL format must be a [kustommize remote URL format](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md). Examples of using custom packages are available in the [example](./examples) directory. Let's take a look at [this example](examples/basic). This defines two custom package directories to deploy to the cluster. @@ -153,6 +156,12 @@ To deploy these packages, run the following commands from this repository's root ./idpbuilder create --package-dir examples/basic/package1 --package-dir examples/basic/package2 ``` +Alternatively, you can use the URL format: + +``` +./idpbuilder create --package-dir https://github.com/cnoe-io/idpbuilder//examples/basic/package1 --package-dir https://github.com/cnoe-io/idpbuilder//examples/basic/package2 +``` + Running this command should create three additional ArgoCD applications in your cluster. ```sh diff --git a/pkg/build/build.go b/pkg/build/build.go index 51268e86..47036f02 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -170,6 +170,7 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { return err } defer os.RemoveAll(dir) + setupLog.V(1).Info("Created temp directory for cloning repositories", "dir", dir) setupLog.V(1).Info("Running controllers") if err := b.RunControllers(ctx, mgr, managerExit, dir); err != nil { diff --git a/pkg/cmd/create/root.go b/pkg/cmd/create/root.go index 05468b1c..14485349 100644 --- a/pkg/cmd/create/root.go +++ b/pkg/cmd/create/root.go @@ -56,7 +56,7 @@ func init() { CreateCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "Protocol to use to access web UIs. http or https.") CreateCmd.PersistentFlags().StringVar(&port, "port", "8443", "Port number under which idpBuilder tools are accessible.") CreateCmd.PersistentFlags().BoolVar(&pathRouting, "use-path-routing", false, "When set to true, web UIs are exposed under single domain name.") - CreateCmd.Flags().StringSliceVarP(&extraPackagesDirs, "package-dir", "p", []string{}, "Paths to custom packages") + CreateCmd.Flags().StringSliceVarP(&extraPackagesDirs, "package-dir", "p", []string{}, "Paths to directories containing custom packages") CreateCmd.Flags().StringSliceVarP(&packageCustomizationFiles, "package-custom-file", "c", []string{}, "Name of the package and the path to file to customize the package with. e.g. argocd:/tmp/argocd.yaml") // idpbuilder related flags CreateCmd.Flags().BoolVarP(&noExit, "no-exit", "n", true, "When set, idpbuilder will not exit after all packages are synced. Useful for continuously syncing local directories.") diff --git a/pkg/cmd/get/secrets.go b/pkg/cmd/get/secrets.go index f0449c2b..cd97ce0b 100644 --- a/pkg/cmd/get/secrets.go +++ b/pkg/cmd/get/secrets.go @@ -58,10 +58,11 @@ func getSecretsE(cmd *cobra.Command, args []string) error { defer ctxCancel() kubeConfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config") - opts := build.NewBuildOptions{} - opts.KubeConfigPath = kubeConfigPath - opts.Scheme = k8s.GetScheme() - opts.CancelFunc = ctxCancel + opts := build.NewBuildOptions{ + KubeConfigPath: kubeConfigPath, + Scheme: k8s.GetScheme(), + CancelFunc: ctxCancel, + } b := build.NewBuild(opts) diff --git a/pkg/cmd/helpers/validation.go b/pkg/cmd/helpers/validation.go index 6c9cea51..87a9e42f 100644 --- a/pkg/cmd/helpers/validation.go +++ b/pkg/cmd/helpers/validation.go @@ -36,8 +36,7 @@ func ValidateKubernetesYamlFile(absPath string) error { } func ParsePackageStrings(pkgStrings []string) ([]string, []string, error) { - remote := make([]string, 0, 2) - local := make([]string, 0, 2) + remote, local := make([]string, 0, 2), make([]string, 0, 2) for i := range pkgStrings { loc := pkgStrings[i] _, err := util.NewKustomizeRemote(loc) diff --git a/pkg/controllers/custompackage/controller.go b/pkg/controllers/custompackage/controller.go index 3a701f7e..b53d9f1e 100644 --- a/pkg/controllers/custompackage/controller.go +++ b/pkg/controllers/custompackage/controller.go @@ -281,19 +281,19 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *Reconciler) getArgoCDAppFile(ctx context.Context, resource *v1alpha1.CustomPackage) ([]byte, error) { - if resource.Spec.RemoteRepository.Url != "" { - cloneDir := util.RepoDir(resource.Spec.RemoteRepository.Url, r.TempDir) - st := r.RepoMap.LoadOrStore(resource.Spec.RemoteRepository.Url, cloneDir) - st.MU.Lock() - wt, _, err := util.CloneRemoteRepoToDir(ctx, resource.Spec.RemoteRepository, 1, false, cloneDir, "") - defer st.MU.Unlock() - if err != nil { - return nil, fmt.Errorf("cloning repo, %s: %w", resource.Spec.RemoteRepository.Url, err) - } - return util.ReadWorktreeFile(wt, resource.Spec.ArgoCD.ApplicationFile) + if resource.Spec.RemoteRepository.Url == "" { + return os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) } - return os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) + cloneDir := util.RepoDir(resource.Spec.RemoteRepository.Url, r.TempDir) + st := r.RepoMap.LoadOrStore(resource.Spec.RemoteRepository.Url, cloneDir) + st.MU.Lock() + wt, _, err := util.CloneRemoteRepoToDir(ctx, resource.Spec.RemoteRepository, 1, false, cloneDir, "") + defer st.MU.Unlock() + if err != nil { + return nil, fmt.Errorf("cloning repo, %s: %w", resource.Spec.RemoteRepository.Url, err) + } + return util.ReadWorktreeFile(wt, resource.Spec.ArgoCD.ApplicationFile) } func localRepoName(appName, dir string) string { diff --git a/pkg/controllers/gitrepository/controller.go b/pkg/controllers/gitrepository/controller.go index ba29f875..d08740dd 100644 --- a/pkg/controllers/gitrepository/controller.go +++ b/pkg/controllers/gitrepository/controller.go @@ -239,6 +239,7 @@ func pushToRemote(ctx context.Context, remoteRepo *git.Repository, creds gitProv func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig, tmpDir string, repoMap *util.RepoMap) error { logger := log.FromContext(ctx) tgtCloneDir := util.RepoDir(tgtRepo.cloneUrl, tmpDir) + st := repoMap.LoadOrStore(tgtRepo.cloneUrl, tgtCloneDir) st.MU.Lock() defer st.MU.Unlock() diff --git a/pkg/controllers/gitrepository/gitea.go b/pkg/controllers/gitrepository/gitea.go index 6dcdcb1b..2a9db6c9 100644 --- a/pkg/controllers/gitrepository/gitea.go +++ b/pkg/controllers/gitrepository/gitea.go @@ -98,7 +98,14 @@ func (g *giteaProvider) getRepository(ctx context.Context, repo *v1alpha1.GitRep }, nil } -func (g *giteaProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { +func (g *giteaProvider) updateRepoContent( + ctx context.Context, + repo *v1alpha1.GitRepository, + repoInfo repoInfo, + creds gitProviderCredentials, + tmpDir string, + repoMap *util.RepoMap, +) error { switch repo.Spec.Source.Type { case v1alpha1.SourceTypeLocal, v1alpha1.SourceTypeEmbedded: return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, tmpDir, repoMap) diff --git a/pkg/controllers/gitrepository/github.go b/pkg/controllers/gitrepository/github.go index 58f129c5..2a2ac0c3 100644 --- a/pkg/controllers/gitrepository/github.go +++ b/pkg/controllers/gitrepository/github.go @@ -102,7 +102,14 @@ func (g *gitHubProvider) setProviderCredentials(ctx context.Context, repo *v1alp return g.gitHubClient.setToken(creds.accessToken) } -func (g *gitHubProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { +func (g *gitHubProvider) updateRepoContent( + ctx context.Context, + repo *v1alpha1.GitRepository, + repoInfo repoInfo, + creds gitProviderCredentials, + tmpDir string, + repoMap *util.RepoMap, +) error { return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, tmpDir, repoMap) } diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index 700e2035..aaeb22f8 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -8,15 +8,17 @@ import ( "strings" "time" + argocdapp "github.com/cnoe-io/argocd-api/api/argo/application" "github.com/cnoe-io/idpbuilder/pkg/util" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" argov1alpha1 "github.com/cnoe-io/argocd-api/api/argo/application/v1alpha1" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/globals" "github.com/cnoe-io/idpbuilder/pkg/resources/localbuild" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -142,8 +144,8 @@ func (r *LocalbuildReconciler) ReconcileArgoAppsWithGitea(ctx context.Context, r } } - for i := range resource.Spec.PackageConfigs.CustomPackageDirs { - result, err := r.reconcileCustomPkgDir(ctx, resource, resource.Spec.PackageConfigs.CustomPackageDirs[i]) + for _, s := range resource.Spec.PackageConfigs.CustomPackageDirs { + result, err := r.reconcileCustomPkgDir(ctx, resource, s) if err != nil { return result, err } @@ -187,7 +189,7 @@ func (r *LocalbuildReconciler) reconcileEmbeddedApp(ctx context.Context, appName } err = r.Client.Get(ctx, client.ObjectKeyFromObject(app), app) - if err != nil && errors.IsNotFound(err) { + if err != nil && k8serrors.IsNotFound(err) { localbuild.SetApplicationSpec( app, repo.Status.InternalGitRepositoryUrl, @@ -290,6 +292,72 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a return true, nil } +func (r *LocalbuildReconciler) reconcileCustomPkg( + ctx context.Context, + resource *v1alpha1.Localbuild, + b []byte, + filePath string, + remote *util.KustomizeRemote, +) error { + o := &unstructured.Unstructured{} + _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) + if fErr != nil { + return fErr + } + + if isSupportedArgoCDTypes(gvk) { + appName := o.GetName() + appNS := o.GetNamespace() + customPkg := &v1alpha1.CustomPackage{ + ObjectMeta: metav1.ObjectMeta{ + Name: getCustomPackageName(filepath.Base(filePath), appName), + Namespace: globals.GetProjectNamespace(resource.Name), + }, + } + + cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) + + _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { + if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { + return err + } + if customPkg.ObjectMeta.Annotations == nil { + customPkg.ObjectMeta.Annotations = make(map[string]string) + } + + util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) + + customPkg.Spec = v1alpha1.CustomPackageSpec{ + Replicate: true, + GitServerURL: resource.Status.Gitea.ExternalURL, + InternalGitServeURL: resource.Status.Gitea.InternalURL, + GitServerAuthSecretRef: v1alpha1.SecretReference{ + Name: resource.Status.Gitea.AdminUserSecretName, + Namespace: resource.Status.Gitea.AdminUserSecretNamespace, + }, + ArgoCD: v1alpha1.ArgoCDPackageSpec{ + ApplicationFile: filePath, + Name: appName, + Namespace: appNS, + }, + } + + if remote != nil { + customPkg.Spec.RemoteRepository = v1alpha1.RemoteRepositorySpec{ + Url: remote.CloneUrl(), + Ref: remote.Ref, + CloneSubmodules: remote.Submodules, + Path: remote.Path(), + } + } + + return nil + }) + return fErr + } + return nil +} + func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resource *v1alpha1.Localbuild, pkgUrl string) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -318,71 +386,16 @@ func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resour return ctrl.Result{}, fmt.Errorf("getting yaml files from repo, %s: %w", pkgUrl, err) } - for i := range yamlFiles { - n := yamlFiles[i] - b, fErr := util.ReadWorktreeFile(wt, n) + for _, yamlFile := range yamlFiles { + b, fErr := util.ReadWorktreeFile(wt, yamlFile) if fErr != nil { - logger.V(1).Info("processing", "file", n, "err", fErr) + logger.V(1).Info("processing", "file", yamlFile, "err", fErr) continue } - o := &unstructured.Unstructured{} - _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) - if fErr != nil { - continue - } - - if gvk.Kind == "Application" && gvk.Group == "argoproj.io" { - appName := o.GetName() - appNS := o.GetNamespace() - customPkg := &v1alpha1.CustomPackage{ - ObjectMeta: metav1.ObjectMeta{ - Name: getCustomPackageName(filepath.Base(n), appName), - Namespace: globals.GetProjectNamespace(resource.Name), - }, - } - - cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) - if err != nil { - logger.Error(err, "this resource may not sync correctly") - } - - _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { - if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { - return err - } - if customPkg.ObjectMeta.Annotations == nil { - customPkg.ObjectMeta.Annotations = make(map[string]string) - } - - util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) - - customPkg.Spec = v1alpha1.CustomPackageSpec{ - Replicate: true, - GitServerURL: resource.Status.Gitea.ExternalURL, - InternalGitServeURL: resource.Status.Gitea.InternalURL, - GitServerAuthSecretRef: v1alpha1.SecretReference{ - Name: resource.Status.Gitea.AdminUserSecretName, - Namespace: resource.Status.Gitea.AdminUserSecretNamespace, - }, - ArgoCD: v1alpha1.ArgoCDPackageSpec{ - ApplicationFile: n, - Name: appName, - Namespace: appNS, - }, - RemoteRepository: v1alpha1.RemoteRepositorySpec{ - Url: remote.CloneUrl(), - Ref: remote.Ref, - CloneSubmodules: remote.Submodules, - Path: remote.Path(), - }, - } - return nil - }) - if fErr != nil { - logger.Error(fErr, "failed creating custom package object", "name", appName, "namespace", appNS) - continue - } + rErr := r.reconcileCustomPkg(ctx, resource, b, yamlFile, remote) + if rErr != nil { + logger.Error(rErr, "reconciling custom pkg", "file", yamlFile, "pkgUrl", pkgUrl) } } return ctrl.Result{}, nil @@ -409,56 +422,9 @@ func (r *LocalbuildReconciler) reconcileCustomPkgDir(ctx context.Context, resour continue } - o := &unstructured.Unstructured{} - _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) - if fErr != nil { - continue - } - if gvk.Kind == "Application" && gvk.Group == "argoproj.io" { - appName := o.GetName() - appNS := o.GetNamespace() - customPkg := &v1alpha1.CustomPackage{ - ObjectMeta: metav1.ObjectMeta{ - Name: getCustomPackageName(file.Name(), appName), - Namespace: globals.GetProjectNamespace(resource.Name), - }, - } - - cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) - if err != nil { - logger.Error(err, "this resource may not sync correctly") - } - - _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { - if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { - return err - } - if customPkg.ObjectMeta.Annotations == nil { - customPkg.ObjectMeta.Annotations = make(map[string]string) - } - - util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) - - customPkg.Spec = v1alpha1.CustomPackageSpec{ - Replicate: true, - GitServerURL: resource.Status.Gitea.ExternalURL, - InternalGitServeURL: resource.Status.Gitea.InternalURL, - GitServerAuthSecretRef: v1alpha1.SecretReference{ - Name: resource.Status.Gitea.AdminUserSecretName, - Namespace: resource.Status.Gitea.AdminUserSecretNamespace, - }, - ArgoCD: v1alpha1.ArgoCDPackageSpec{ - ApplicationFile: filePath, - Name: appName, - Namespace: appNS, - }, - } - return nil - }) - if fErr != nil { - logger.Error(fErr, "failed creating custom package object", "name", appName, "namespace", appNS) - continue - } + rErr := r.reconcileCustomPkg(ctx, resource, b, filePath, nil) + if rErr != nil { + logger.Error(rErr, "reconciling custom pkg", "file", filePath, "pkgDir", pkgDir) } } @@ -527,6 +493,13 @@ func getCustomPackageName(fileName, appName string) string { return fmt.Sprintf("%s-%s", strings.ToLower(s[0]), appName) } +func isSupportedArgoCDTypes(gvk *schema.GroupVersionKind) bool { + if gvk == nil { + return false + } + return gvk.Kind == argocdapp.ApplicationKind && gvk.Group == argocdapp.Group +} + func GetEmbeddedRawInstallResources(name string, templateData any, config v1alpha1.PackageCustomization, scheme *runtime.Scheme) ([][]byte, error) { switch name { case "argocd": diff --git a/pkg/controllers/run.go b/pkg/controllers/run.go index 2df33c3b..19eae91f 100644 --- a/pkg/controllers/run.go +++ b/pkg/controllers/run.go @@ -12,7 +12,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ctxCancel context.CancelFunc, exitOnSync bool, cfg util.CorePackageTemplateConfig, tmpDir string) error { +func RunControllers( + ctx context.Context, + mgr manager.Manager, + exitCh chan error, + ctxCancel context.CancelFunc, + exitOnSync bool, + cfg util.CorePackageTemplateConfig, + tmpDir string, +) error { logger := log.FromContext(ctx) repoMap := util.NewRepoLock()