From b64b6277a3889b51297a224ccbaba475d31254db Mon Sep 17 00:00:00 2001 From: Manabu McCloskey Date: Tue, 14 May 2024 00:17:31 +0000 Subject: [PATCH] support remote repository Signed-off-by: Manabu McCloskey --- api/v1alpha1/custom_package_types.go | 17 +- api/v1alpha1/gitrepository_types.go | 8 +- api/v1alpha1/localbuild_types.go | 1 + api/v1alpha1/zz_generated.deepcopy.go | 22 +++ pkg/build/build.go | 43 ++-- pkg/cmd/create/root.go | 39 +++- pkg/cmd/get/secrets.go | 9 +- pkg/cmd/helpers/validation.go | 58 ++++-- pkg/cmd/helpers/validation_test.go | 39 ++++ pkg/controllers/custompackage/controller.go | 147 ++++++++++---- .../custompackage/controller_test.go | 10 +- pkg/controllers/gitrepository/controller.go | 136 +++++++++---- pkg/controllers/gitrepository/gitea.go | 10 +- pkg/controllers/gitrepository/github.go | 2 +- pkg/controllers/localbuild/controller.go | 124 ++++++++++-- .../idpbuilder.cnoe.io_custompackages.yaml | 22 +++ .../idpbuilder.cnoe.io_gitrepositories.yaml | 24 +++ .../idpbuilder.cnoe.io_localbuilds.yaml | 4 + pkg/util/files.go | 130 ++++++++++++ pkg/util/fs_test.go | 33 ++++ pkg/util/url.go | 187 ++++++++++++++++++ pkg/util/url_test.go | 82 ++++++++ pkg/util/util_test.go | 45 ++++- 23 files changed, 1046 insertions(+), 146 deletions(-) create mode 100644 pkg/util/url.go create mode 100644 pkg/util/url_test.go diff --git a/api/v1alpha1/custom_package_types.go b/api/v1alpha1/custom_package_types.go index 04620c6c..71c39c57 100644 --- a/api/v1alpha1/custom_package_types.go +++ b/api/v1alpha1/custom_package_types.go @@ -4,6 +4,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + CNOEURIScheme = "cnoe://" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status type CustomPackage struct { @@ -30,12 +34,23 @@ type CustomPackageSpec struct { GitServerAuthSecretRef SecretReference `json:"gitServerAuthSecretRef"` // InternalGitServeURL specifies the base URL for the git server accessible within the cluster. // for example, http://my-gitea-http.gitea.svc.cluster.local:3000 - InternalGitServeURL string `json:"internalGitServeURL"` + InternalGitServeURL string `json:"internalGitServeURL"` + RemoteRepository RemoteRepositorySpec `json:"remoteRepository"` // Replicate specifies whether to replicate remote or local contents to the local gitea server. // +kubebuilder:default:=false Replicate bool `json:"replicate"` } +// RemoteRepositorySpec specifies information about remote repositories. +type RemoteRepositorySpec struct { + CloneSubmodules bool `json:"cloneSubmodules"` + Path string `json:"path"` + // Url specifies the url to the repository containing the ArgoCD application file + Url string `json:"url"` + // Ref specifies the specific ref supported by git fetch + Ref string `json:"ref"` +} + type ArgoCDPackageSpec struct { // ApplicationFile specifies the absolute path to the ArgoCD application file ApplicationFile string `json:"applicationFile"` diff --git a/api/v1alpha1/gitrepository_types.go b/api/v1alpha1/gitrepository_types.go index 457c110c..d3ac32af 100644 --- a/api/v1alpha1/gitrepository_types.go +++ b/api/v1alpha1/gitrepository_types.go @@ -8,6 +8,9 @@ const ( GitProviderGitea = "gitea" GitProviderGitHub = "github" GiteaAdminUserName = "giteaAdmin" + SourceTypeLocal = "local" + SourceTypeRemote = "remote" + SourceTypeEmbedded = "embedded" ) type GitRepositorySpec struct { @@ -27,9 +30,10 @@ type GitRepositorySource struct { // Path is the absolute path to directory that contains Kustomize structure or raw manifests. // This is required when Type is set to local. // +kubebuilder:validation:Optional - Path string `json:"path"` + Path string `json:"path"` + RemoteRepository RemoteRepositorySpec `json:"remoteRepository"` // Type is the source type. - // +kubebuilder:validation:Enum:=local;embedded + // +kubebuilder:validation:Enum:=local;embedded;remote // +kubebuilder:default:=embedded Type string `json:"type"` } diff --git a/api/v1alpha1/localbuild_types.go b/api/v1alpha1/localbuild_types.go index 99f9ce22..d8872ee2 100644 --- a/api/v1alpha1/localbuild_types.go +++ b/api/v1alpha1/localbuild_types.go @@ -38,6 +38,7 @@ type PackageConfigsSpec struct { Argo ArgoPackageConfigSpec `json:"argoPackageConfigs,omitempty"` EmbeddedArgoApplications EmbeddedArgoApplicationsPackageConfigSpec `json:"embeddedArgoApplicationsPackageConfigs,omitempty"` CustomPackageDirs []string `json:"customPackageDirs,omitempty"` + CustomPackageUrls []string `json:"customPackageUrls,omitempty"` } type LocalbuildSpec struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c3cf1482..1061d56d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -148,6 +148,7 @@ func (in *CustomPackageSpec) DeepCopyInto(out *CustomPackageSpec) { *out = *in out.ArgoCD = in.ArgoCD out.GitServerAuthSecretRef = in.GitServerAuthSecretRef + out.RemoteRepository = in.RemoteRepository } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomPackageSpec. @@ -264,6 +265,7 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitRepositorySource) DeepCopyInto(out *GitRepositorySource) { *out = *in + out.RemoteRepository = in.RemoteRepository } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositorySource. @@ -459,6 +461,11 @@ func (in *PackageConfigsSpec) DeepCopyInto(out *PackageConfigsSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CustomPackageUrls != nil { + in, out := &in.CustomPackageUrls, &out.CustomPackageUrls + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackageConfigsSpec. @@ -501,6 +508,21 @@ func (in *Provider) DeepCopy() *Provider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteRepositorySpec) DeepCopyInto(out *RemoteRepositorySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteRepositorySpec. +func (in *RemoteRepositorySpec) DeepCopy() *RemoteRepositorySpec { + if in == nil { + return nil + } + out := new(RemoteRepositorySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretReference) DeepCopyInto(out *SecretReference) { *out = *in diff --git a/pkg/build/build.go b/pkg/build/build.go index d6b78ecd..73052bfd 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -32,28 +32,42 @@ type Build struct { kubeVersion string extraPortsMapping string customPackageDirs []string + customPackageUrls []string packageCustomization map[string]v1alpha1.PackageCustomization exitOnSync bool scheme *runtime.Scheme CancelFunc context.CancelFunc } -func NewBuild(name, kubeVersion, kubeConfigPath, kindConfigPath, extraPortsMapping string, cfg util.CorePackageTemplateConfig, - customPackageDirs []string, exitOnSync bool, scheme *runtime.Scheme, ctxCancel context.CancelFunc, - packageCustomization map[string]v1alpha1.PackageCustomization) *Build { +type NewBuildOptions struct { + Name string + TemplateData util.CorePackageTemplateConfig + KindConfigPath string + KubeConfigPath string + KubeVersion string + ExtraPortsMapping string + CustomPackageDirs []string + CustomPackageUrls []string + PackageCustomization map[string]v1alpha1.PackageCustomization + ExitOnSync bool + Scheme *runtime.Scheme + CancelFunc context.CancelFunc +} +func NewBuild(opts NewBuildOptions) *Build { return &Build{ - name: name, - kindConfigPath: kindConfigPath, - kubeConfigPath: kubeConfigPath, - kubeVersion: kubeVersion, - extraPortsMapping: extraPortsMapping, - customPackageDirs: customPackageDirs, - packageCustomization: packageCustomization, - exitOnSync: exitOnSync, - scheme: scheme, - cfg: cfg, - CancelFunc: ctxCancel, + name: opts.Name, + kindConfigPath: opts.KindConfigPath, + kubeConfigPath: opts.KubeConfigPath, + kubeVersion: opts.KubeVersion, + extraPortsMapping: opts.ExtraPortsMapping, + customPackageDirs: opts.CustomPackageDirs, + customPackageUrls: opts.CustomPackageUrls, + packageCustomization: opts.PackageCustomization, + exitOnSync: opts.ExitOnSync, + scheme: opts.Scheme, + cfg: opts.TemplateData, + CancelFunc: opts.CancelFunc, } } @@ -178,6 +192,7 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { PackageCustomization: b.packageCustomization, }, CustomPackageDirs: b.customPackageDirs, + CustomPackageUrls: b.customPackageUrls, }, } diff --git a/pkg/cmd/create/root.go b/pkg/cmd/create/root.go index 2fcd9641..05468b1c 100644 --- a/pkg/cmd/create/root.go +++ b/pkg/cmd/create/root.go @@ -84,12 +84,15 @@ func create(cmd *cobra.Command, args []string) error { } var absDirPaths []string + var remotePaths []string + if len(extraPackagesDirs) > 0 { - p, err := helpers.GetAbsFilePaths(extraPackagesDirs, true) - if err != nil { - return err + r, l, pErr := helpers.ParsePackageStrings(extraPackagesDirs) + if pErr != nil { + return pErr } - absDirPaths = p + absDirPaths = l + remotePaths = r } o := make(map[string]v1alpha1.PackageCustomization) @@ -106,17 +109,31 @@ func create(cmd *cobra.Command, args []string) error { exitOnSync = !noExit } - b := build.NewBuild( - buildName, kubeVersion, kubeConfigPath, kindConfigPath, extraPortsMapping, - util.CorePackageTemplateConfig{ + opts := build.NewBuildOptions{ + Name: buildName, + KubeVersion: kubeVersion, + KubeConfigPath: kubeConfigPath, + KindConfigPath: kindConfigPath, + ExtraPortsMapping: extraPortsMapping, + + TemplateData: util.CorePackageTemplateConfig{ Protocol: protocol, Host: host, IngressHost: ingressHost, Port: port, UsePathRouting: pathRouting, }, - absDirPaths, exitOnSync, k8s.GetScheme(), ctxCancel, o, - ) + + CustomPackageDirs: absDirPaths, + CustomPackageUrls: remotePaths, + ExitOnSync: exitOnSync, + PackageCustomization: o, + + Scheme: k8s.GetScheme(), + CancelFunc: ctxCancel, + } + + b := build.NewBuild(opts) if err := b.Run(ctx, recreateCluster); err != nil { return err @@ -153,7 +170,9 @@ func validate() error { return pErr } } - return nil + + _, _, err = helpers.ParsePackageStrings(extraPackagesDirs) + return err } func getPackageCustomFile(input string) (v1alpha1.PackageCustomization, error) { diff --git a/pkg/cmd/get/secrets.go b/pkg/cmd/get/secrets.go index b5c4ec70..f0449c2b 100644 --- a/pkg/cmd/get/secrets.go +++ b/pkg/cmd/get/secrets.go @@ -13,7 +13,6 @@ import ( "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/build" "github.com/cnoe-io/idpbuilder/pkg/k8s" - "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -59,8 +58,12 @@ func getSecretsE(cmd *cobra.Command, args []string) error { defer ctxCancel() kubeConfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config") - b := build.NewBuild("", "", kubeConfigPath, "", "", - util.CorePackageTemplateConfig{}, []string{}, false, k8s.GetScheme(), ctxCancel, nil) + opts := build.NewBuildOptions{} + opts.KubeConfigPath = kubeConfigPath + opts.Scheme = k8s.GetScheme() + opts.CancelFunc = ctxCancel + + b := build.NewBuild(opts) kubeConfig, err := b.GetKubeConfig() if err != nil { diff --git a/pkg/cmd/helpers/validation.go b/pkg/cmd/helpers/validation.go index ef77e7d5..6c9cea51 100644 --- a/pkg/cmd/helpers/validation.go +++ b/pkg/cmd/helpers/validation.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/cnoe-io/idpbuilder/pkg/util" "sigs.k8s.io/kustomize/kyaml/kio" ) @@ -34,27 +35,54 @@ func ValidateKubernetesYamlFile(absPath string) error { return nil } +func ParsePackageStrings(pkgStrings []string) ([]string, []string, error) { + remote := make([]string, 0, 2) + local := make([]string, 0, 2) + for i := range pkgStrings { + loc := pkgStrings[i] + _, err := util.NewKustomizeRemote(loc) + if err == nil { + remote = append(remote, loc) + continue + } + + absPath, err := getAbsPath(loc, true) + if err == nil { + local = append(local, absPath) + continue + } + return nil, nil, err + } + + return remote, local, nil +} + +func getAbsPath(path string, isDir bool) (string, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to validate path %s : %w", path, err) + } + f, err := os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("failed to validate path %s : %w", absPath, err) + } + if isDir && !f.IsDir() { + return "", fmt.Errorf("given path is not a directory. %s", absPath) + } + if !isDir && !f.Mode().IsRegular() { + return "", fmt.Errorf("give path is not a file. %s", absPath) + } + return absPath, nil +} + func GetAbsFilePaths(paths []string, isDir bool) ([]string, error) { out := make([]string, len(paths)) for i := range paths { - path := paths[i] - absPath, err := filepath.Abs(path) - if err != nil { - return nil, fmt.Errorf("failed to validate path %s : %w", path, err) - } - f, err := os.Stat(absPath) + absPath, err := getAbsPath(paths[i], isDir) if err != nil { - return nil, fmt.Errorf("failed to validate path %s : %w", absPath, err) - } - if isDir && !f.IsDir() { - return nil, fmt.Errorf("given path is not a directory. %s", absPath) - } - if !isDir && !f.Mode().IsRegular() { - return nil, fmt.Errorf("give path is not a file. %s", absPath) + return nil, err } - out[i] = absPath } - return out, nil } diff --git a/pkg/cmd/helpers/validation_test.go b/pkg/cmd/helpers/validation_test.go index 1134ce7a..e4d454be 100644 --- a/pkg/cmd/helpers/validation_test.go +++ b/pkg/cmd/helpers/validation_test.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/stretchr/testify/assert" ) func TestValidateKubernetesYaml(t *testing.T) { @@ -33,3 +35,40 @@ func TestValidateKubernetesYaml(t *testing.T) { } } } + +func TestParsePackageStrings(t *testing.T) { + cases := map[string]struct { + expectErr bool + inputPaths []string + remote int + local int + }{ + "allLocal": {expectErr: false, inputPaths: []string{"test-data", "."}, remote: 0, local: 2}, + "allRemote": {expectErr: false, inputPaths: []string{ + "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + "git@github.com:owner/repo//examples", + }, remote: 2, local: 0}, + "mix": {expectErr: false, inputPaths: []string{ + "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + "test-data", + }, remote: 1, local: 1}, + "invalidLocalPath": {expectErr: true, inputPaths: []string{ + "does-not-exist", + }, remote: 0, local: 0}, + "invalidRemotePath": {expectErr: true, inputPaths: []string{ + "https:// github.com/kubernetes-sigs/kustomize//examples", + }, remote: 0, local: 0}, + } + + for k := range cases { + c := cases[k] + remote, local, err := ParsePackageStrings(c.inputPaths) + if cases[k].expectErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, c.remote, len(remote)) + assert.Equal(t, c.local, len(local)) + } +} diff --git a/pkg/controllers/custompackage/controller.go b/pkg/controllers/custompackage/controller.go index a0c65685..ae53f95d 100644 --- a/pkg/controllers/custompackage/controller.go +++ b/pkg/controllers/custompackage/controller.go @@ -8,11 +8,10 @@ import ( "strings" "time" - "github.com/cnoe-io/idpbuilder/api/v1alpha1" - "github.com/cnoe-io/idpbuilder/pkg/util" - argov1alpha1 "github.com/cnoe-io/argocd-api/api/argo/application/v1alpha1" + "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/k8s" + "github.com/cnoe-io/idpbuilder/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -71,7 +70,7 @@ func (r *Reconciler) postProcessReconcile(ctx context.Context, req ctrl.Request, // create an in-cluster repository CR, update the application spec, then apply func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alpha1.CustomPackage) (ctrl.Result, error) { - b, err := os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) + b, err := getArgoCDAppFile(ctx, resource) if err != nil { return ctrl.Result{}, fmt.Errorf("reading file %s: %w", resource.Spec.ArgoCD.ApplicationFile, err) } @@ -91,17 +90,17 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp app, ok := objs[0].(*argov1alpha1.Application) if !ok { - return ctrl.Result{}, fmt.Errorf("object is not an PackageSpec application %s", resource.Spec.ArgoCD.ApplicationFile) + return ctrl.Result{}, fmt.Errorf("object is not an ArgoCD application %s", resource.Spec.ArgoCD.ApplicationFile) } appName := app.GetName() if resource.Spec.Replicate { - repoRefs := make([]v1alpha1.ObjectRef, 0, 1) synced := true + repoRefs := make([]v1alpha1.ObjectRef, 0, 1) if app.Spec.HasMultipleSources() { for j := range app.Spec.Sources { s := &app.Spec.Sources[j] - res, repo, sErr := r.reconcileArgocdSource(ctx, resource, appName, resource.Spec.ArgoCD.ApplicationFile, s.RepoURL) + res, repo, sErr := r.reconcileArgoCDSource(ctx, resource, s, appName) if sErr != nil { return res, sErr } @@ -119,7 +118,7 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp } } else { s := app.Spec.Source - res, repo, sErr := r.reconcileArgocdSource(ctx, resource, appName, resource.Spec.ArgoCD.ApplicationFile, s.RepoURL) + res, repo, sErr := r.reconcileArgoCDSource(ctx, resource, s, appName) if sErr != nil { return res, sErr } @@ -161,37 +160,84 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp return ctrl.Result{RequeueAfter: requeueTime}, nil } -func (r *Reconciler) reconcileArgocdSource(ctx context.Context, resource *v1alpha1.CustomPackage, appName, pkgDir, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { - logger := log.FromContext(ctx) - - process, absPath, err := isCNOEDirectory(pkgDir, repoURL) - if err != nil { - logger.Error(err, "processing argocd app source", "dir", pkgDir, "repoURL", repoURL) - return ctrl.Result{}, nil, err +func (r *Reconciler) reconcileArgoCDSource(ctx context.Context, resource *v1alpha1.CustomPackage, appSource *argov1alpha1.ApplicationSource, appName string) (ctrl.Result, *v1alpha1.GitRepository, error) { + if isCNOEScheme(appSource.RepoURL) { + if resource.Spec.RemoteRepository.Url == "" { + return r.reconcileArgoCDLocalSource(ctx, resource, appName, appSource.RepoURL) + } + return r.reconcileArgoCDRemoteSource(ctx, resource, appName, appSource.RepoURL) } - if !process { - return ctrl.Result{}, nil, nil + return ctrl.Result{}, nil, nil +} + +func (r *Reconciler) reconcileArgoCDRemoteSource(ctx context.Context, resource *v1alpha1.CustomPackage, appName, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { + relativePath := strings.TrimPrefix(repoURL, v1alpha1.CNOEURIScheme) + // no guarantee that this path exists + dirPath := filepath.Join(resource.Spec.RemoteRepository.Path, relativePath) + + repo := &v1alpha1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: remoteRepoName(appName, dirPath, resource.Spec.RemoteRepository), + Namespace: resource.Namespace, + }, } - repo, err := r.reconcileGitRepo(ctx, resource, repoName(appName, absPath), absPath) - if err != nil { + cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { + if err := controllerutil.SetControllerReference(resource, repo, r.Scheme); err != nil { + return err + } + + if repo.ObjectMeta.Annotations == nil { + repo.ObjectMeta.Annotations = make(map[string]string) + } + util.SetCLIStartTimeAnnotationValue(repo.ObjectMeta.Annotations, cliStartTime) + + repo.Spec = v1alpha1.GitRepositorySpec{ + Source: v1alpha1.GitRepositorySource{ + Type: v1alpha1.SourceTypeRemote, + RemoteRepository: resource.Spec.RemoteRepository, + Path: dirPath, + }, + Provider: v1alpha1.Provider{ + Name: v1alpha1.GitProviderGitea, + GitURL: resource.Spec.GitServerURL, + InternalGitURL: resource.Spec.InternalGitServeURL, + OrganizationName: v1alpha1.GiteaAdminUserName, + }, + SecretRef: resource.Spec.GitServerAuthSecretRef, + } + + return nil + }) + + if err != nil && !errors.IsAlreadyExists(err) { return ctrl.Result{}, nil, err } return ctrl.Result{}, repo, nil } -func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.CustomPackage, repoName, absPath string) (*v1alpha1.GitRepository, error) { +func (r *Reconciler) reconcileArgoCDLocalSource(ctx context.Context, resource *v1alpha1.CustomPackage, appName, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { + logger := log.FromContext(ctx) + + absPath, err := getCNOEAbsPath(resource.Spec.ArgoCD.ApplicationFile, repoURL) + if err != nil { + logger.Error(err, "processing argocd app source", "dir", resource.Spec.ArgoCD.ApplicationFile, "repoURL", repoURL) + return ctrl.Result{}, nil, err + } + repo := &v1alpha1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: repoName, + Name: localRepoName(appName, absPath), Namespace: resource.Namespace, }, } cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { if err := controllerutil.SetControllerReference(resource, repo, r.Scheme); err != nil { return err } @@ -203,7 +249,7 @@ func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.Cu repo.Spec = v1alpha1.GitRepositorySpec{ Source: v1alpha1.GitRepositorySource{ - Type: "local", + Type: v1alpha1.SourceTypeLocal, Path: absPath, }, Provider: v1alpha1.Provider{ @@ -220,10 +266,10 @@ func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.Cu // it's possible for an application to specify the same directory multiple times in the spec. // if there is a repository already created for this package, no further action is necessary. if !errors.IsAlreadyExists(err) { - return repo, err + return ctrl.Result{}, repo, err } - return repo, nil + return ctrl.Result{}, repo, nil } func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -232,27 +278,44 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func isCNOEDirectory(fPath, repoURL string) (bool, string, error) { - if strings.HasPrefix(repoURL, "cnoe://") { - parentDir := filepath.Dir(fPath) - relativePath := strings.TrimPrefix(repoURL, "cnoe://") - absPath, err := filepath.Abs(filepath.Join(parentDir, relativePath)) +func getArgoCDAppFile(ctx context.Context, resource *v1alpha1.CustomPackage) ([]byte, error) { + if resource.Spec.RemoteRepository.Url != "" { + wt, _, err := util.CloneRemoteRepoToMemory(ctx, resource.Spec.RemoteRepository, 1, false) if err != nil { - return false, "", err - } - - f, err := os.Stat(absPath) - if err != nil { - return false, "", err - } - if !f.IsDir() { - return false, "", fmt.Errorf("path not a directory: %s", absPath) + return nil, fmt.Errorf("cloning repo, %s: %w", resource.Spec.RemoteRepository.Url, err) } - return true, absPath, err + return util.ReadWorktreeFile(wt, resource.Spec.ArgoCD.ApplicationFile) } - return false, "", nil + + return os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) } -func repoName(appName, dir string) string { +func localRepoName(appName, dir string) string { return fmt.Sprintf("%s-%s", appName, filepath.Base(dir)) } + +func remoteRepoName(appName, pathToPkg string, repo v1alpha1.RemoteRepositorySpec) string { + return fmt.Sprintf("%s-%s", appName, filepath.Base(pathToPkg)) +} + +func isCNOEScheme(repoURL string) bool { + return strings.HasPrefix(repoURL, v1alpha1.CNOEURIScheme) +} + +func getCNOEAbsPath(fPath, repoURL string) (string, error) { + parentDir := filepath.Dir(fPath) + relativePath := strings.TrimPrefix(repoURL, v1alpha1.CNOEURIScheme) + absPath, err := filepath.Abs(filepath.Join(parentDir, relativePath)) + if err != nil { + return "", err + } + + f, err := os.Stat(absPath) + if err != nil { + return "", err + } + if !f.IsDir() { + return "", fmt.Errorf("path not a directory: %s", absPath) + } + return absPath, err +} diff --git a/pkg/controllers/custompackage/controller_test.go b/pkg/controllers/custompackage/controller_test.go index 6e9afa88..53baecef 100644 --- a/pkg/controllers/custompackage/controller_test.go +++ b/pkg/controllers/custompackage/controller_test.go @@ -156,7 +156,7 @@ func TestReconcileCustomPkg(t *testing.T) { c := mgr.GetClient() repo := v1alpha1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: repoName("my-app", "test/resources/customPackages/testDir/busybox"), + Name: localRepoName("my-app", "test/resources/customPackages/testDir/busybox"), Namespace: "test", }, } @@ -197,8 +197,8 @@ func TestReconcileCustomPkg(t *testing.T) { if err != nil { t.Fatalf("failed getting my-app %v", err) } - if strings.HasPrefix(localApp.Spec.Source.RepoURL, "cnoe://") { - t.Fatalf("cnoe:// prefix should be removed") + if strings.HasPrefix(localApp.Spec.Source.RepoURL, v1alpha1.CNOEURIScheme) { + t.Fatalf("%s prefix should be removed", v1alpha1.CNOEURIScheme) } for _, n := range []string{"guestbook", "guestbook2"} { @@ -219,7 +219,7 @@ func TestReconcileCustomPkg(t *testing.T) { t.Fatalf("failed getting my-app2 %v", err) } - if strings.HasPrefix(localApp2.Spec.Sources[0].RepoURL, "cnoe://") { - t.Fatalf("cnoe:// prefix should be removed") + if strings.HasPrefix(localApp2.Spec.Sources[0].RepoURL, v1alpha1.CNOEURIScheme) { + t.Fatalf("%s prefix should be removed", v1alpha1.CNOEURIScheme) } } diff --git a/pkg/controllers/gitrepository/controller.go b/pkg/controllers/gitrepository/controller.go index 78062776..0baa2bdf 100644 --- a/pkg/controllers/gitrepository/controller.go +++ b/pkg/controllers/gitrepository/controller.go @@ -8,13 +8,13 @@ import ( "net" "net/http" "os" - "path/filepath" "time" "code.gitea.io/sdk/gitea" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" @@ -65,6 +65,10 @@ func getOrganizationName(repo v1alpha1.GitRepository) string { return repo.Spec.Provider.OrganizationName } +func getFallbackRepositoryURL(repo *v1alpha1.GitRepository, info repoInfo) string { + return fmt.Sprintf("%s/%s.git", repo.Spec.Provider.GitURL, info.fullName) +} + func GetGitProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) (gitProvider, error) { switch repo.Spec.Provider.Name { case v1alpha1.GitProviderGitea: @@ -103,9 +107,6 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) } defer r.postProcessReconcile(ctx, req, &gitRepo) - if !r.shouldProcess(gitRepo) { - return ctrl.Result{Requeue: false}, nil - } logger.V(1).Info("reconciling GitRepository", "name", req.Name, "namespace", req.Namespace) result, err := r.reconcileGitRepo(ctx, &gitRepo) @@ -120,7 +121,6 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *RepositoryReconciler) postProcessReconcile(ctx context.Context, req ctrl.Request, repo *v1alpha1.GitRepository) { logger := log.FromContext(ctx) - err := r.Status().Update(ctx, repo) if err != nil { logger.Error(err, "failed updating repo status") @@ -185,14 +185,51 @@ func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager, notifyChan cha Complete(r) } -func (r *RepositoryReconciler) shouldProcess(repo v1alpha1.GitRepository) bool { - if repo.Spec.Source.Type == "local" && !filepath.IsAbs(repo.Spec.Source.Path) { - return false +func addAllAndCommit(path string, gitRepo *git.Repository) (plumbing.Hash, bool, error) { + tree, err := gitRepo.Worktree() + if err != nil { + return plumbing.Hash{}, false, fmt.Errorf("getting git worktree: %w", err) + } + + err = tree.AddGlob("*") + if err != nil { + return plumbing.Hash{}, false, fmt.Errorf("adding git files: %w", err) + } + + status, err := tree.Status() + if err != nil { + return plumbing.Hash{}, false, fmt.Errorf("getting git status: %w", err) } - return true + + if status.IsClean() { + h, _ := gitRepo.Head() + return h.Hash(), false, nil + } + + h, err := tree.Commit(fmt.Sprintf("updated from %s", path), &git.CommitOptions{ + All: true, + AllowEmptyCommits: false, + Author: &object.Signature{ + Name: gitCommitAuthorName, + Email: gitCommitAuthorEmail, + When: time.Now(), + }, + }) + return h, true, nil } -func updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) error { +func pushToRemote(ctx context.Context, remoteRepo *git.Repository, creds gitProviderCredentials) error { + auth, err := getBasicAuth(creds) + if err != nil { + return fmt.Errorf("getting basic auth: %w", err) + } + return remoteRepo.PushContext(ctx, &git.PushOptions{ + Auth: &auth, + InsecureSkipTLS: true, + }) +} + +func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) error { logger := log.FromContext(ctx) tempDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s", repo.Name, repo.Namespace)) @@ -217,7 +254,7 @@ func updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoIn // if we cannot clone with gitea's configured url, then we fallback to using the url provided in spec. logger.V(1).Info("failed cloning with returned clone URL. Falling back to default url.", "err", err) - cloneOptions.URL = fmt.Sprintf("%s/%s.git", repo.Spec.Provider.GitURL, repoInfo.fullName) + cloneOptions.URL = getFallbackRepositoryURL(repo, repoInfo) c, retErr := git.PlainCloneContext(ctx, tempDir, false, cloneOptions) if retErr != nil { return fmt.Errorf("cloning repo with fall back url: %w", retErr) @@ -230,49 +267,72 @@ func updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoIn return fmt.Errorf("writing repo contents: %w", err) } - tree, err := clonedRepo.Worktree() + hash, push, err := addAllAndCommit(repo.Spec.Source.Path, clonedRepo) if err != nil { - return fmt.Errorf("getting git worktree: %w", err) + return fmt.Errorf("add and commit %w", err) } - err = tree.AddGlob("*") - if err != nil { - return fmt.Errorf("adding git files: %w", err) + if push { + err := pushToRemote(ctx, clonedRepo, creds) + if err != nil { + return fmt.Errorf("pushing to git: %w", err) + } + repo.Status.LatestCommit.Hash = hash.String() + return nil } - status, err := tree.Status() + repo.Status.LatestCommit.Hash = hash.String() + return nil +} + +func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { + logger := log.FromContext(ctx) + + remoteWT, _, err := util.CloneRemoteRepoToMemory(ctx, repo.Spec.Source.RemoteRepository, 1, false) if err != nil { - return fmt.Errorf("getting git status: %w", err) + return fmt.Errorf("cloning repo, %s: %w", repo.Spec.Source.RemoteRepository.Url, err) } - if status.IsClean() { - h, _ := clonedRepo.Head() - repo.Status.LatestCommit.Hash = h.Hash().String() - return nil + localRepoSpec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: ".", + Url: repoInfo.cloneUrl, + Ref: "", } - commit, err := tree.Commit(fmt.Sprintf("updated from %s", repo.Spec.Source.Path), &git.CommitOptions{ - All: true, - AllowEmptyCommits: false, - Author: &object.Signature{ - Name: gitCommitAuthorName, - Email: gitCommitAuthorEmail, - When: time.Now(), - }, - }) + var localRepo *git.Repository + + localWT, lr, err := util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) if err != nil { - return fmt.Errorf("committing git files: %w", err) + logger.V(1).Info("failed cloning repo. trying fallback url", "err", err, "url", repoInfo.cloneUrl) + localRepoSpec.Url = getFallbackRepositoryURL(repo, repoInfo) + _, lr, err = util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) + if err != nil { + return fmt.Errorf("cloning repo, %s: %w", repoInfo.cloneUrl, err) + } } - err = clonedRepo.Push(&git.PushOptions{ - Auth: &auth, - InsecureSkipTLS: true, - }) + localRepo = lr + err = util.CopyTreeToTree(remoteWT, localWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".") if err != nil { - return fmt.Errorf("pushing to git: %w", err) + return fmt.Errorf("copying contents, %s: %w", repoInfo.cloneUrl, err) + } + + hash, push, err := addAllAndCommit(repo.Spec.Source.Path, localRepo) + if err != nil { + return fmt.Errorf("add and commit %w", err) + } + + if push { + err := pushToRemote(ctx, localRepo, creds) + if err != nil { + return fmt.Errorf("pushing to git: %w", err) + } + repo.Status.LatestCommit.Hash = hash.String() + return nil } - repo.Status.LatestCommit.Hash = commit.String() + repo.Status.LatestCommit.Hash = hash.String() return nil } diff --git a/pkg/controllers/gitrepository/gitea.go b/pkg/controllers/gitrepository/gitea.go index 8a5b3b65..35d7a381 100644 --- a/pkg/controllers/gitrepository/gitea.go +++ b/pkg/controllers/gitrepository/gitea.go @@ -40,6 +40,7 @@ func (g *giteaProvider) createRepository(ctx context.Context, repo *v1alpha1.Git DefaultBranch: DefaultBranchName, AutoInit: true, }) + if err != nil { return repoInfo{}, fmt.Errorf("failed to create git repository: %w", err) } @@ -98,7 +99,14 @@ func (g *giteaProvider) getRepository(ctx context.Context, repo *v1alpha1.GitRep } func (g *giteaProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { - return updateRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) + switch repo.Spec.Source.Type { + case v1alpha1.SourceTypeLocal, v1alpha1.SourceTypeEmbedded: + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) + case v1alpha1.SourceTypeRemote: + return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds) + default: + return nil + } } func writeRepoContents(repo *v1alpha1.GitRepository, dstPath string, config util.CorePackageTemplateConfig, scheme *runtime.Scheme) error { diff --git a/pkg/controllers/gitrepository/github.go b/pkg/controllers/gitrepository/github.go index ba21a27b..1f3613e2 100644 --- a/pkg/controllers/gitrepository/github.go +++ b/pkg/controllers/gitrepository/github.go @@ -103,7 +103,7 @@ func (g *gitHubProvider) setProviderCredentials(ctx context.Context, repo *v1alp } func (g *gitHubProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { - return updateRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) } func newGitHubClient(httpClient *http.Client) gitHubClient { diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index 47b67182..77186e7d 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -9,7 +9,6 @@ import ( "time" "github.com/cnoe-io/idpbuilder/pkg/util" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" argov1alpha1 "github.com/cnoe-io/argocd-api/api/argo/application/v1alpha1" @@ -140,12 +139,18 @@ func (r *LocalbuildReconciler) ReconcileArgoAppsWithGitea(ctx context.Context, r return result, fmt.Errorf("reconciling bootstrap apps %w", err) } } - if resource.Spec.PackageConfigs.CustomPackageDirs != nil { - for i := range resource.Spec.PackageConfigs.CustomPackageDirs { - result, err := r.reconcileCustomPkg(ctx, resource, resource.Spec.PackageConfigs.CustomPackageDirs[i]) - if err != nil { - return result, err - } + + for i := range resource.Spec.PackageConfigs.CustomPackageDirs { + result, err := r.reconcileCustomPkgDir(ctx, resource, resource.Spec.PackageConfigs.CustomPackageDirs[i]) + if err != nil { + return result, err + } + } + + for _, s := range resource.Spec.PackageConfigs.CustomPackageUrls { + result, err := r.reconcileCustomPkgUrl(ctx, resource, s) + if err != nil { + return result, err } } @@ -220,7 +225,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.Annotations) if err != nil { - return true, err + return false, err } repos := &v1alpha1.GitRepositoryList{} @@ -228,6 +233,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a if err != nil { return false, fmt.Errorf("listing repositories %w", err) } + for i := range repos.Items { repo := repos.Items[i] @@ -258,7 +264,6 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a if err != nil { return false, fmt.Errorf("listing custom packages %w", err) } - for i := range pkgs.Items { pkg := pkgs.Items[i] startTimeAnnotation, gErr := util.GetCLIStartTimeAnnotationValue(pkg.ObjectMeta.Annotations) @@ -267,7 +272,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a } if startTimeAnnotation != cliStartTime { - continue + return false, nil } observedTime, gErr := util.GetLastObservedSyncTimeAnnotationValue(pkg.ObjectMeta.Annotations) @@ -275,7 +280,6 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a logger.Info(gErr.Error()) return false, nil } - if !pkg.Status.Synced || cliStartTime != observedTime { return false, nil } @@ -284,7 +288,101 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a return true, nil } -func (r *LocalbuildReconciler) reconcileCustomPkg(ctx context.Context, resource *v1alpha1.Localbuild, pkgDir string) (ctrl.Result, error) { +func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resource *v1alpha1.Localbuild, pkgUrl string) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + remote, err := util.NewKustomizeRemote(pkgUrl) + if err != nil { + return ctrl.Result{}, fmt.Errorf("parsing url, %s: %w", pkgUrl, err) + } + rs := v1alpha1.RemoteRepositorySpec{ + Url: remote.CloneUrl(), + Ref: remote.Ref, + CloneSubmodules: remote.Submodules, + Path: remote.Path(), + } + + wt, _, err := util.CloneRemoteRepoToMemory(ctx, rs, 1, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("cloning repo, %s: %w", pkgUrl, err) + } + + yamlFiles, err := util.GetWorktreeYamlFiles(remote.Path(), wt, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("getting yaml files from repo, %s: %w", pkgUrl, err) + } + + for i := range yamlFiles { + n := yamlFiles[i] + b, fErr := util.ReadWorktreeFile(wt, n) + if fErr != nil { + logger.V(1).Info("processing", "file", n, "err", fErr) + continue + } + + o := &unstructured.Unstructured{} + _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) + if fErr != nil { + continue + } + + if gvk.Kind == "Application" && gvk.Group == "argoproj.io" { + appName := o.GetName() + appNS := o.GetNamespace() + customPkg := &v1alpha1.CustomPackage{ + ObjectMeta: metav1.ObjectMeta{ + Name: getCustomPackageName(filepath.Base(n), appName), + Namespace: globals.GetProjectNamespace(resource.Name), + }, + } + + cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) + if err != nil { + logger.Error(err, "this resource may not sync correctly") + } + + _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { + if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { + return err + } + if customPkg.ObjectMeta.Annotations == nil { + customPkg.ObjectMeta.Annotations = make(map[string]string) + } + + util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) + + customPkg.Spec = v1alpha1.CustomPackageSpec{ + Replicate: true, + GitServerURL: resource.Status.Gitea.ExternalURL, + InternalGitServeURL: resource.Status.Gitea.InternalURL, + GitServerAuthSecretRef: v1alpha1.SecretReference{ + Name: resource.Status.Gitea.AdminUserSecretName, + Namespace: resource.Status.Gitea.AdminUserSecretNamespace, + }, + ArgoCD: v1alpha1.ArgoCDPackageSpec{ + ApplicationFile: n, + Name: appName, + Namespace: appNS, + }, + RemoteRepository: v1alpha1.RemoteRepositorySpec{ + Url: remote.CloneUrl(), + Ref: remote.Ref, + CloneSubmodules: remote.Submodules, + Path: remote.Path(), + }, + } + return nil + }) + if fErr != nil { + logger.Error(fErr, "failed creating custom package object", "name", appName, "namespace", appNS) + continue + } + } + } + return ctrl.Result{}, nil +} + +func (r *LocalbuildReconciler) reconcileCustomPkgDir(ctx context.Context, resource *v1alpha1.Localbuild, pkgDir string) (ctrl.Result, error) { logger := log.FromContext(ctx) files, err := os.ReadDir(pkgDir) @@ -400,7 +498,7 @@ func (r *LocalbuildReconciler) reconcileGitRepo(ctx context.Context, resource *v }, } - if repoType == "embedded" { + if repoType == v1alpha1.SourceTypeEmbedded { repo.Spec.Source.EmbeddedAppName = embeddedName } else { repo.Spec.Source.Path = absPath diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml index 953cc204..507dd2f8 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml @@ -74,6 +74,27 @@ spec: InternalGitServeURL specifies the base URL for the git server accessible within the cluster. for example, http://my-gitea-http.gitea.svc.cluster.local:3000 type: string + remoteRepository: + description: RemoteRepositorySpec specifies information about remote + repositories. + properties: + cloneSubmodules: + type: boolean + path: + type: string + ref: + description: Ref specifies the specific ref supported by git fetch + type: string + url: + description: Url specifies the url to the repository containing + the ArgoCD application file + type: string + required: + - cloneSubmodules + - path + - ref + - url + type: object replicate: default: false description: Replicate specifies whether to replicate remote or local @@ -83,6 +104,7 @@ spec: - gitServerAuthSecretRef - gitServerURL - internalGitServeURL + - remoteRepository - replicate type: object status: diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml index b334960a..498b932c 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml @@ -100,14 +100,38 @@ spec: Path is the absolute path to directory that contains Kustomize structure or raw manifests. This is required when Type is set to local. type: string + remoteRepository: + description: RemoteRepositorySpec specifies information about + remote repositories. + properties: + cloneSubmodules: + type: boolean + path: + type: string + ref: + description: Ref specifies the specific ref supported by git + fetch + type: string + url: + description: Url specifies the url to the repository containing + the ArgoCD application file + type: string + required: + - cloneSubmodules + - path + - ref + - url + type: object type: default: embedded description: Type is the source type. enum: - local - embedded + - remote type: string required: + - remoteRepository - type type: object required: diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml index d19ea371..d25483e9 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml @@ -52,6 +52,10 @@ spec: items: type: string type: array + customPackageUrls: + items: + type: string + type: array embeddedArgoApplicationsPackageConfigs: description: EmbeddedArgoApplicationsPackageConfigSpec Controls the installation of the embedded argo applications. diff --git a/pkg/util/files.go b/pkg/util/files.go index 5a5cdf5c..616c4807 100644 --- a/pkg/util/files.go +++ b/pkg/util/files.go @@ -2,11 +2,20 @@ package util import ( "bytes" + "context" "fmt" "io" "os" "path/filepath" + "strings" "text/template" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" ) func CopyDirectory(scrDir, dest string) error { @@ -108,3 +117,124 @@ func ApplyTemplate(in []byte, templateData any) ([]byte, error) { return ret.Bytes(), nil } + +// returns all files with yaml or yml suffix from a worktree +func GetWorktreeYamlFiles(parent string, wt billy.Filesystem, recurse bool) ([]string, error) { + if strings.HasSuffix(parent, "/") { + parent = strings.TrimSuffix(parent, "/") + } + paths := make([]string, 0, 10) + ents, err := wt.ReadDir(parent) + if err != nil { + return nil, err + } + for i := range ents { + ent := ents[i] + if ent.IsDir() && recurse { + dir := fmt.Sprintf("%s/%s", parent, ent.Name()) + rPaths, dErr := GetWorktreeYamlFiles(dir, wt, recurse) + if dErr != nil { + return nil, fmt.Errorf("reading %s : %w", ent.Name(), dErr) + } + paths = append(paths, rPaths...) + } + if ent.Mode().IsRegular() && (strings.HasSuffix(ent.Name(), "yaml") || strings.HasSuffix(ent.Name(), "yml")) { + paths = append(paths, fmt.Sprintf("%s/%s", parent, ent.Name())) + } + } + return paths, nil +} + +func ReadWorktreeFile(wt billy.Filesystem, path string) ([]byte, error) { + f, fErr := wt.Open(path) + if fErr != nil { + return nil, fmt.Errorf("opening %s", path) + } + defer f.Close() + + b := new(bytes.Buffer) + _, fErr = b.ReadFrom(f) + if fErr != nil { + return nil, fmt.Errorf("reading %s", path) + } + + return b.Bytes(), nil +} + +func CloneRemoteRepoToMemory(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool) (billy.Filesystem, *git.Repository, error) { + cloneOptions := &git.CloneOptions{ + URL: remote.Url, + Depth: depth, + ShallowSubmodules: true, + SingleBranch: true, + Tags: git.AllTags, + InsecureSkipTLS: insecureSkipTLS, + } + if remote.CloneSubmodules { + cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth + } + + if remote.Ref != "" { + cloneOptions.ReferenceName = plumbing.NewTagReferenceName(remote.Ref) + } + + wt := memfs.New() + var cloned *git.Repository + cloned, err := git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(remote.Ref) + cloned, err = git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + return nil, nil, err + } + } + return wt, cloned, nil +} + +func CopyTreeToTree(srcWT, dstWT billy.Filesystem, srcPath, dstPath string) error { + files, err := srcWT.ReadDir(srcPath) + if err != nil { + return err + } + + for i := range files { + srcFile := files[i] + fullSrcPath := filepath.Join(srcPath, srcFile.Name()) + fullDstPath := filepath.Join(dstPath, srcFile.Name()) + if srcFile.Mode().IsRegular() { + cErr := CopyWTFile(srcWT, dstWT, fullSrcPath, fullDstPath) + if cErr != nil { + return cErr + } + continue + } + + if srcFile.IsDir() { + dErr := CopyTreeToTree(srcWT, dstWT, fullSrcPath, fullDstPath) + if dErr != nil { + return dErr + } + } + } + return nil +} + +func CopyWTFile(srcWT, dstWT billy.Filesystem, srcFile, dstFile string) error { + newFile, err := dstWT.Create(dstFile) + if err != nil { + return fmt.Errorf("creating file %s: %w", dstFile, err) + } + defer newFile.Close() + + srcF, err := srcWT.Open(srcFile) + if err != nil { + return fmt.Errorf("reading file %s: %w", srcFile, err) + } + defer srcF.Close() + + _, err = io.Copy(newFile, srcF) + if err != nil { + return fmt.Errorf("copying file %s: %w", srcFile, err) + } + return nil +} diff --git a/pkg/util/fs_test.go b/pkg/util/fs_test.go index da0a692e..1a6d1b57 100644 --- a/pkg/util/fs_test.go +++ b/pkg/util/fs_test.go @@ -1,15 +1,21 @@ package util import ( + "context" "fmt" "io/fs" "os" "path/filepath" + "strings" "testing" "testing/fstest" "github.com/cnoe-io/idpbuilder/globals" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/storage/memory" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" ) func TestWriteFS(t *testing.T) { @@ -72,3 +78,30 @@ func TestWriteFS(t *testing.T) { }) } } + +func TestGetWorktreeYamlFiles(t *testing.T) { + filepath.Join() + cloneOptions := &git.CloneOptions{ + URL: "https://github.com/cnoe-io/idpbuilder", + Depth: 1, + ShallowSubmodules: true, + } + + wt := memfs.New() + _, err := git.CloneContext(context.Background(), memory.NewStorage(), wt, cloneOptions) + if err != nil { + t.Fatalf(err.Error()) + } + + paths, err := GetWorktreeYamlFiles("./pkg", wt, true) + + assert.Equal(t, nil, err) + assert.NotEqual(t, 0, len(paths)) + for _, s := range paths { + assert.Equal(t, true, strings.HasSuffix(s, "yaml") || strings.HasSuffix(s, "yml")) + } + + paths, err = GetWorktreeYamlFiles("./pkg", wt, false) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(paths)) +} diff --git a/pkg/util/url.go b/pkg/util/url.go new file mode 100644 index 00000000..0fb20fb3 --- /dev/null +++ b/pkg/util/url.go @@ -0,0 +1,187 @@ +package util + +import ( + "fmt" + "net/url" + "strconv" + "strings" + "time" +) + +// constants from remote target parameters supported by Kustomize +// https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md +const ( + QueryStringRef = "ref" + QueryStringVersion = "version" + QueryStringTimeout = "timeout" + QueryStringSubmodules = "submodules" + + RepoUrlDelimiter = "//" + SCPDelimiter = ":" + UserDelimiter = "@" + + defaultTimeout = time.Second * 27 + defaultCloneSubmodule = true + + errMsgUrlUnsupported = "url must have // after the repository url. example: https://github.com/kubernetes-sigs/kustomize//examples" + errMsgUrlColon = "first path segment in URL cannot contain colon" +) + +type KustomizeRemote struct { + raw string + + Scheme string + User string + Password string + Host string + Port string + RepoPath string + + FilePath string + + Ref string + Submodules bool + Timeout time.Duration +} + +func (g *KustomizeRemote) CloneUrl() string { + sb := strings.Builder{} + if g.Scheme != "" { + sb.WriteString(fmt.Sprintf("%s://", g.Scheme)) + } + if g.User != "" { + sb.WriteString(g.User) + if g.Password != "" { + sb.WriteString(fmt.Sprintf(":%s", g.Password)) + } + sb.Write([]byte(UserDelimiter)) + } + + sb.WriteString(g.Host) + if g.Port != "" { + sb.WriteString(fmt.Sprintf(":%s", g.Port)) + } + if g.Scheme == "" { + sb.WriteString(":") + } else { + sb.WriteString("/") + } + + sb.WriteString(g.RepoPath) + return sb.String() +} + +func (g *KustomizeRemote) Path() string { + return g.FilePath +} + +func (g *KustomizeRemote) parseQuery() error { + _, query, _ := strings.Cut(g.raw, "?") + values, err := url.ParseQuery(query) + + if err != nil { + return err + } + + // if empty, it means we checkout the default branch + version := values.Get(QueryStringVersion) + ref := values.Get(QueryStringRef) + // ref has higher priority per kustomize doc + if ref != "" { + version = ref + } + + duration := defaultTimeout + timeoutString := values.Get(QueryStringTimeout) + if timeoutString != "" { + timeoutInt, sErr := strconv.Atoi(timeoutString) + if sErr == nil { + duration = time.Duration(timeoutInt) * time.Second + } else { + t, sErr := time.ParseDuration(timeoutString) + if sErr == nil { + duration = t + } + } + } + + cloneSubmodules := defaultCloneSubmodule + submodule := values.Get(QueryStringSubmodules) + if submodule != "" { + v, pErr := strconv.ParseBool(submodule) + if pErr == nil { + cloneSubmodules = v + } + } + + g.Ref = version + g.Submodules = cloneSubmodules + g.Timeout = duration + + return nil +} + +func (g *KustomizeRemote) parse() error { + parsed, err := url.Parse(g.raw) + if err != nil { + if strings.Contains(err.Error(), errMsgUrlColon) { + return g.parseSCPStyle() + } + return err + } + + g.Scheme, g.User, g.Host = parsed.Scheme, parsed.User.Username(), parsed.Host + p, ok := parsed.User.Password() + if ok { + g.Password = p + } + + err = g.parseQuery() + if err != nil { + return fmt.Errorf("parsing query parameters in package url: %s: %w", g.raw, err) + } + + return g.parsePath(parsed.Path) +} + +func (g *KustomizeRemote) parseSCPStyle() error { + // example git@github.com:owner/repo + cIndex := strings.Index(g.raw, SCPDelimiter) + if cIndex == -1 { + return fmt.Errorf("not a valid SCP style URL") + } + + uIndex := strings.Index(g.raw[:cIndex], UserDelimiter) + if uIndex != -1 { + g.User = g.raw[:uIndex] + } + g.Host = g.raw[uIndex+1 : cIndex] + err := g.parseQuery() + if err != nil { + return fmt.Errorf("parsing query parameters in package url: %s: %w", g.raw, err) + } + + pathEnd := len(g.raw) + qIndex := strings.Index(g.raw, "?") + if qIndex != -1 { + pathEnd = qIndex + } + return g.parsePath(g.raw[cIndex+1 : pathEnd]) +} + +func (g *KustomizeRemote) parsePath(path string) error { + // example kubernetes-sigs/kustomize//examples/multibases/dev/ + index := strings.Index(path, RepoUrlDelimiter) + if index == -1 { + return fmt.Errorf(errMsgUrlUnsupported) + } + + g.RepoPath = strings.TrimPrefix(path[:index], "/") + g.FilePath = strings.TrimSuffix(path[index+2:], "/") + return nil +} + +func NewKustomizeRemote(uri string) (*KustomizeRemote, error) { + r := &KustomizeRemote{raw: uri} + return r, r.parse() +} diff --git a/pkg/util/url_test.go b/pkg/util/url_test.go new file mode 100644 index 00000000..d4204be2 --- /dev/null +++ b/pkg/util/url_test.go @@ -0,0 +1,82 @@ +package util + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestURLParse(t *testing.T) { + + type expect struct { + cloneUrl string + path string + ref string + submodule bool + timeout time.Duration + err bool + } + + type testCase struct { + expect expect + input string + } + + cases := []testCase{ + { + input: "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + expect: expect{ + cloneUrl: "https://github.com/kubernetes-sigs/kustomize", + path: "examples/multibases/dev", + ref: "v3.3.1", + submodule: true, + timeout: 120 * time.Second, + }, + }, + { + input: "git@github.com:owner/repo//examples?timeout=120&version=v3.3.1", + expect: expect{ + cloneUrl: "git@github.com:owner/repo", + path: "examples", + ref: "v3.3.1", + submodule: true, + timeout: 120 * time.Second, + }, + }, + { + input: "https:// /(@kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + expect: expect{ + err: true, + }, + }, + { + input: "https://my.github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?version=v3.3.1&submodules=false&timeout=1s", + expect: expect{ + cloneUrl: "https://my.github.com/kubernetes-sigs/kustomize", + path: "examples/multibases/dev", + ref: "v3.3.1", + submodule: false, + timeout: 1 * time.Second, + }, + }, + } + + for i := range cases { + c := cases[i] + + r, err := NewKustomizeRemote(c.input) + if err != nil { + if !c.expect.err { + assert.Fail(t, err.Error()) + } else { + continue + } + } + assert.Equal(t, c.expect.path, r.Path()) + assert.Equal(t, c.expect.cloneUrl, r.CloneUrl()) + assert.Equal(t, c.expect.timeout, r.Timeout) + assert.Equal(t, c.expect.ref, r.Ref) + assert.Equal(t, c.expect.submodule, r.Submodules) + } +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index bfa76ab5..ac154e65 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,14 +1,20 @@ package util import ( + "context" + "path/filepath" "strconv" "testing" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/stretchr/testify/assert" ) var specialCharMap = make(map[string]struct{}) func TestGeneratePassword(t *testing.T) { - for i := range specialChars { specialCharMap[string(specialChars[i])] = struct{}{} } @@ -43,3 +49,40 @@ func TestGeneratePassword(t *testing.T) { } } } + +func TestCopyTreeToTree(t *testing.T) { + spec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: "examples/basic", + Url: "https://github.com/cnoe-io/idpbuilder", + Ref: "", + } + + dst := memfs.New() + src, _, err := CloneRemoteRepoToMemory(context.Background(), spec, 1, false) + assert.Nil(t, err) + + err = CopyTreeToTree(src, dst, spec.Path, ".") + assert.Nil(t, err) + testCopiedFiles(t, src, dst, spec.Path, ".") +} + +func testCopiedFiles(t *testing.T, src, dst billy.Filesystem, srcStartPath, dstStartPath string) { + files, err := src.ReadDir(srcStartPath) + assert.Nil(t, err) + + for i := range files { + file := files[i] + if file.Mode().IsRegular() { + srcB, err := ReadWorktreeFile(src, filepath.Join(srcStartPath, file.Name())) + assert.Nil(t, err) + + dstB, err := ReadWorktreeFile(dst, filepath.Join(dstStartPath, file.Name())) + assert.Nil(t, err) + assert.Equal(t, srcB, dstB) + } + if file.IsDir() { + testCopiedFiles(t, src, dst, filepath.Join(srcStartPath, file.Name()), filepath.Join(dstStartPath, file.Name())) + } + } +}