From 275c99aa9496e98807e67a5bb6725904e641a871 Mon Sep 17 00:00:00 2001 From: Manabu McCloskey Date: Thu, 16 May 2024 00:43:45 +0000 Subject: [PATCH] wip --- go.mod | 15 +- go.sum | 27 +- pkg/build/build.go | 15 +- pkg/controllers/custompackage/controller.go | 12 +- pkg/controllers/gitrepository/controller.go | 47 ++-- .../gitrepository/controller_test.go | 14 +- .../gitrepository/git_repository.go | 3 +- pkg/controllers/gitrepository/gitea.go | 8 +- pkg/controllers/gitrepository/github.go | 4 +- pkg/controllers/localbuild/controller.go | 8 +- pkg/controllers/run.go | 10 +- pkg/util/files.go | 130 --------- pkg/util/fs_test.go | 33 --- pkg/util/git_repository.go | 260 ++++++++++++++++++ pkg/util/git_repository_test.go | 97 +++++++ pkg/util/util_test.go | 44 --- 16 files changed, 459 insertions(+), 268 deletions(-) create mode 100644 pkg/util/git_repository.go create mode 100644 pkg/util/git_repository_test.go diff --git a/go.mod b/go.mod index c1465bd3..a8699204 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,12 @@ module github.com/cnoe-io/idpbuilder go 1.21.3 -toolchain go1.21.5 - require ( code.gitea.io/sdk/gitea v0.16.0 github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d github.com/docker/docker v24.0.7+incompatible - github.com/go-git/go-git/v5 v5.10.0 + github.com/go-git/go-billy/v5 v5.5.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/go-logr/logr v1.4.1 github.com/google/go-cmp v0.6.0 github.com/google/go-github/v61 v61.0.0 @@ -29,12 +28,11 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect @@ -49,7 +47,6 @@ require ( github.com/go-errors/errors v1.4.2 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect @@ -90,9 +87,9 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/sergi/go-diff v1.1.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skeema/knownhosts v1.2.0 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect diff --git a/go.sum b/go.sum index dfa88a66..17caef73 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,8 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -25,8 +23,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d h1:cmcSrS0OYTLlGsBFshaAG29qC+PC5LBtKRhnEknlzgU= github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d/go.mod h1:IXG3LiEAeckMfjdwJnt6qC0ee4J4U5bleMuk1HN82ZA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -59,8 +58,8 @@ github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1 github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= @@ -71,8 +70,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ= -github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -140,8 +139,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -190,13 +187,13 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= diff --git a/pkg/build/build.go b/pkg/build/build.go index 73052bfd..51268e86 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -3,9 +3,11 @@ package build import ( "context" "fmt" + "os" "time" "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/globals" "github.com/cnoe-io/idpbuilder/pkg/controllers" "github.com/cnoe-io/idpbuilder/pkg/kind" "github.com/cnoe-io/idpbuilder/pkg/util" @@ -120,8 +122,8 @@ func (b *Build) ReconcileCRDs(ctx context.Context, kubeClient client.Client) err return nil } -func (b *Build) RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error) error { - return controllers.RunControllers(ctx, mgr, exitCh, b.CancelFunc, b.exitOnSync, b.cfg) +func (b *Build) RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, tmpDir string) error { + return controllers.RunControllers(ctx, mgr, exitCh, b.CancelFunc, b.exitOnSync, b.cfg, tmpDir) } func (b *Build) Run(ctx context.Context, recreateCluster bool) error { @@ -162,8 +164,15 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { return err } + dir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", globals.ProjectName, b.name)) + if err != nil { + setupLog.Error(err, "creating temp dir") + return err + } + defer os.RemoveAll(dir) + setupLog.V(1).Info("Running controllers") - if err := b.RunControllers(ctx, mgr, managerExit); err != nil { + if err := b.RunControllers(ctx, mgr, managerExit, dir); err != nil { setupLog.Error(err, "Error running controllers") return err } diff --git a/pkg/controllers/custompackage/controller.go b/pkg/controllers/custompackage/controller.go index ae53f95d..7daba930 100644 --- a/pkg/controllers/custompackage/controller.go +++ b/pkg/controllers/custompackage/controller.go @@ -31,6 +31,8 @@ type Reconciler struct { Recorder record.EventRecorder Scheme *runtime.Scheme Config util.CorePackageTemplateConfig + TempDir string + RepoMap *util.RepoMap } func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -70,7 +72,7 @@ func (r *Reconciler) postProcessReconcile(ctx context.Context, req ctrl.Request, // create an in-cluster repository CR, update the application spec, then apply func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alpha1.CustomPackage) (ctrl.Result, error) { - b, err := getArgoCDAppFile(ctx, resource) + b, err := r.getArgoCDAppFile(ctx, resource) if err != nil { return ctrl.Result{}, fmt.Errorf("reading file %s: %w", resource.Spec.ArgoCD.ApplicationFile, err) } @@ -278,9 +280,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func getArgoCDAppFile(ctx context.Context, resource *v1alpha1.CustomPackage) ([]byte, error) { +func (r *Reconciler) getArgoCDAppFile(ctx context.Context, resource *v1alpha1.CustomPackage) ([]byte, error) { if resource.Spec.RemoteRepository.Url != "" { - wt, _, err := util.CloneRemoteRepoToMemory(ctx, resource.Spec.RemoteRepository, 1, false) + cloneDir := util.RepoDir(resource.Spec.RemoteRepository.Url, r.TempDir) + st := r.RepoMap.Get(resource.Spec.RemoteRepository.Url, cloneDir) + st.MU.Lock() + wt, _, err := util.CloneRemoteRepoToDir(ctx, resource.Spec.RemoteRepository, 1, false, cloneDir, "") + defer st.MU.Unlock() if err != nil { return nil, fmt.Errorf("cloning repo, %s: %w", resource.Spec.RemoteRepository.Url, err) } diff --git a/pkg/controllers/gitrepository/controller.go b/pkg/controllers/gitrepository/controller.go index 0baa2bdf..a8416e2f 100644 --- a/pkg/controllers/gitrepository/controller.go +++ b/pkg/controllers/gitrepository/controller.go @@ -47,6 +47,8 @@ type RepositoryReconciler struct { Scheme *runtime.Scheme Config util.CorePackageTemplateConfig GitProviderFunc gitProviderFunc + TempDir string + RepoMap *util.RepoMap } type gitProviderFunc func(context.Context, *v1alpha1.GitRepository, client.Client, *runtime.Scheme, util.CorePackageTemplateConfig) (gitProvider, error) @@ -167,7 +169,7 @@ func (r *RepositoryReconciler) reconcileGitRepo(ctx context.Context, repo *v1alp providerRepo = p } - err = provider.updateRepoContent(ctx, repo, providerRepo, creds) + err = provider.updateRepoContent(ctx, repo, providerRepo, creds, r.TempDir, r.RepoMap) if err != nil { return ctrl.Result{}, fmt.Errorf("updating repository contents: %w", err) } @@ -229,7 +231,7 @@ func pushToRemote(ctx context.Context, remoteRepo *git.Repository, creds gitProv }) } -func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) error { +func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig, repoMap *util.RepoMap) error { logger := log.FromContext(ctx) tempDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s", repo.Name, repo.Namespace)) @@ -285,37 +287,50 @@ func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository return nil } -func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { +func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { logger := log.FromContext(ctx) - remoteWT, _, err := util.CloneRemoteRepoToMemory(ctx, repo.Spec.Source.RemoteRepository, 1, false) + cloneDir := util.RepoDir(repo.Spec.Source.RemoteRepository.Url, tmpDir) + st := repoMap.Get(repo.Spec.Source.RemoteRepository.Url, cloneDir) + st.MU.Lock() + defer st.MU.Unlock() + remoteWT, _, err := util.CloneRemoteRepoToDir(ctx, repo.Spec.Source.RemoteRepository, 1, false, cloneDir, "") if err != nil { + logger.V(1).Info("cloning repo.", "err", err, "url", repo.Spec.Source.RemoteRepository, "dir", cloneDir) return fmt.Errorf("cloning repo, %s: %w", repo.Spec.Source.RemoteRepository.Url, err) } localRepoSpec := v1alpha1.RemoteRepositorySpec{ CloneSubmodules: false, Path: ".", - Url: repoInfo.cloneUrl, + Url: tgtRepo.cloneUrl, Ref: "", } - var localRepo *git.Repository + tgtCloneDir := util.RepoDir(tgtRepo.cloneUrl, tmpDir) + lst := repoMap.Get(localRepoSpec.Url, tgtCloneDir) - localWT, lr, err := util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) + lst.MU.Lock() + defer lst.MU.Unlock() + tgtRepoWT, localRepo, err := util.CloneRemoteRepoToDir(ctx, localRepoSpec, 1, true, tgtCloneDir, getFallbackRepositoryURL(repo, tgtRepo)) if err != nil { - logger.V(1).Info("failed cloning repo. trying fallback url", "err", err, "url", repoInfo.cloneUrl) - localRepoSpec.Url = getFallbackRepositoryURL(repo, repoInfo) - _, lr, err = util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) - if err != nil { - return fmt.Errorf("cloning repo, %s: %w", repoInfo.cloneUrl, err) - } + return fmt.Errorf("cloning repo %s: %w", repo.Spec.Source.RemoteRepository.Url, err) } - localRepo = lr - err = util.CopyTreeToTree(remoteWT, localWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".") + //localWT, lr, err := util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) + //if err != nil { + // logger.V(1).Info("failed cloning repo. trying fallback url", "err", err, "url", repoInfo.cloneUrl) + // localRepoSpec.Url = getFallbackRepositoryURL(repo, repoInfo) + // _, lr, err = util.CloneRemoteRepoToMemory(ctx, localRepoSpec, 1, true) + // if err != nil { + // return fmt.Errorf("cloning repo, %s: %w", repoInfo.cloneUrl, err) + // } + //} + + //localRepo = lr + err = util.CopyTreeToTree(remoteWT, tgtRepoWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".") if err != nil { - return fmt.Errorf("copying contents, %s: %w", repoInfo.cloneUrl, err) + return fmt.Errorf("copying contents, %s: %w", tgtRepo.cloneUrl, err) } hash, push, err := addAllAndCommit(repo.Spec.Source.Path, localRepo) diff --git a/pkg/controllers/gitrepository/controller_test.go b/pkg/controllers/gitrepository/controller_test.go index f8e8eabc..f8e0db9f 100644 --- a/pkg/controllers/gitrepository/controller_test.go +++ b/pkg/controllers/gitrepository/controller_test.go @@ -228,7 +228,7 @@ func TestGitRepositoryContentReconcile(t *testing.T) { giteaClient: mockGitea{}, } // add file to source directory, reconcile, clone the repo and check if the added file exists - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}, "", util.NewRepoLock()) if err != nil { t.Fatalf("failed adding %v", err) } @@ -250,7 +250,7 @@ func TestGitRepositoryContentReconcile(t *testing.T) { if err != nil { t.Fatalf("failed to remove added file %v", err) } - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}, "", util.NewRepoLock()) if err != nil { t.Fatalf("failed removing %v", err) } @@ -299,7 +299,7 @@ func TestGitRepositoryContentReconcileEmbedded(t *testing.T) { Client: &fakeClient{}, giteaClient: mockGitea{}, } - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}, "", util.NewRepoLock()) if err != nil { t.Fatalf("failed adding %v", err) } @@ -316,8 +316,11 @@ func TestGitRepositoryReconcile(t *testing.T) { if err != nil { t.Fatalf("failed to get absolute path: %v", err) } + updateDir, _, _ := setUpLocalRepo() + defer os.RemoveAll(updateDir) addDir, err := setupDir() + fmt.Println(addDir) defer os.RemoveAll(addDir) if err != nil { t.Fatalf("failed to set up dirs: %v", err) @@ -360,7 +363,7 @@ func TestGitRepositoryReconcile(t *testing.T) { "update": { giteaClient: mockGitea{ getRepo: func() (*gitea.Repository, *gitea.Response, error) { - return &gitea.Repository{CloneURL: dir}, nil, nil + return &gitea.Repository{CloneURL: updateDir}, nil, nil }, }, input: v1alpha1.GitRepository{ @@ -378,7 +381,7 @@ func TestGitRepositoryReconcile(t *testing.T) { }, expect: expect{ resource: v1alpha1.GitRepositoryStatus{ - ExternalGitRepositoryUrl: dir, + ExternalGitRepositoryUrl: updateDir, Synced: true, InternalGitRepositoryUrl: "http://cnoe.io/giteaAdmin/test-test.git", }, @@ -403,7 +406,6 @@ func TestGitRepositoryReconcile(t *testing.T) { if v.expect.resource.LatestCommit.Hash == "" { v.expect.resource.LatestCommit.Hash = v.input.Status.LatestCommit.Hash } - time.Sleep(100 * time.Millisecond) assert.Equal(t, v.input.Status, v.expect.resource) } }) diff --git a/pkg/controllers/gitrepository/git_repository.go b/pkg/controllers/gitrepository/git_repository.go index e218e1ee..a476ae99 100644 --- a/pkg/controllers/gitrepository/git_repository.go +++ b/pkg/controllers/gitrepository/git_repository.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/google/go-github/v61/github" ) @@ -44,5 +45,5 @@ type gitProvider interface { getProviderCredentials(ctx context.Context, repo *v1alpha1.GitRepository) (gitProviderCredentials, error) getRepository(ctx context.Context, repo *v1alpha1.GitRepository) (repoInfo, error) setProviderCredentials(ctx context.Context, repo *v1alpha1.GitRepository, creds gitProviderCredentials) error - updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error + updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error } diff --git a/pkg/controllers/gitrepository/gitea.go b/pkg/controllers/gitrepository/gitea.go index 35d7a381..d71afdda 100644 --- a/pkg/controllers/gitrepository/gitea.go +++ b/pkg/controllers/gitrepository/gitea.go @@ -84,7 +84,7 @@ func (g *giteaProvider) setProviderCredentials(ctx context.Context, repo *v1alph func (g *giteaProvider) getRepository(ctx context.Context, repo *v1alpha1.GitRepository) (repoInfo, error) { resp, repoResp, err := g.giteaClient.GetRepo(getOrganizationName(*repo), getRepositoryName(*repo)) if err != nil { - if repoResp.StatusCode == 404 { + if repoResp != nil && repoResp.StatusCode == 404 { return repoInfo{}, notFoundError{} } return repoInfo{}, err @@ -98,12 +98,12 @@ func (g *giteaProvider) getRepository(ctx context.Context, repo *v1alpha1.GitRep }, nil } -func (g *giteaProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { +func (g *giteaProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { switch repo.Spec.Source.Type { case v1alpha1.SourceTypeLocal, v1alpha1.SourceTypeEmbedded: - return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, repoMap) case v1alpha1.SourceTypeRemote: - return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds) + return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds, tmpDir, repoMap) default: return nil } diff --git a/pkg/controllers/gitrepository/github.go b/pkg/controllers/gitrepository/github.go index 1f3613e2..28ec79e0 100644 --- a/pkg/controllers/gitrepository/github.go +++ b/pkg/controllers/gitrepository/github.go @@ -102,8 +102,8 @@ func (g *gitHubProvider) setProviderCredentials(ctx context.Context, repo *v1alp return g.gitHubClient.setToken(creds.accessToken) } -func (g *gitHubProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { - return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) +func (g *gitHubProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, repoMap) } func newGitHubClient(httpClient *http.Client) gitHubClient { diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index 77186e7d..08ba9309 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -41,6 +41,8 @@ type LocalbuildReconciler struct { ExitOnSync bool shouldShutdown bool Config util.CorePackageTemplateConfig + TempDir string + RepoMap *util.RepoMap } type subReconciler func(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild) (ctrl.Result, error) @@ -302,7 +304,11 @@ func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resour Path: remote.Path(), } - wt, _, err := util.CloneRemoteRepoToMemory(ctx, rs, 1, false) + cloneDir := util.RepoDir(rs.Url, r.TempDir) + st := r.RepoMap.Get(rs.Url, cloneDir) + st.MU.Lock() + defer st.MU.Unlock() + wt, _, err := util.CloneRemoteRepoToDir(ctx, rs, 1, false, cloneDir, "") if err != nil { return ctrl.Result{}, fmt.Errorf("cloning repo, %s: %w", pkgUrl, err) } diff --git a/pkg/controllers/run.go b/pkg/controllers/run.go index f72262f8..2df33c3b 100644 --- a/pkg/controllers/run.go +++ b/pkg/controllers/run.go @@ -12,9 +12,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ctxCancel context.CancelFunc, exitOnSync bool, cfg util.CorePackageTemplateConfig) error { +func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ctxCancel context.CancelFunc, exitOnSync bool, cfg util.CorePackageTemplateConfig, tmpDir string) error { logger := log.FromContext(ctx) + repoMap := util.NewRepoLock() + // Run Localbuild controller if err := (&localbuild.LocalbuildReconciler{ Client: mgr.GetClient(), @@ -22,6 +24,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ExitOnSync: exitOnSync, CancelFunc: ctxCancel, Config: cfg, + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create localbuild controller") return err @@ -33,6 +37,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, Recorder: mgr.GetEventRecorderFor("gitrepository-controller"), Config: cfg, GitProviderFunc: gitrepository.GetGitProvider, + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr, nil) if err != nil { logger.Error(err, "unable to create repo controller") @@ -42,6 +48,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("custompackage-controller"), + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr) if err != nil { logger.Error(err, "unable to create custom package controller") diff --git a/pkg/util/files.go b/pkg/util/files.go index 616c4807..5a5cdf5c 100644 --- a/pkg/util/files.go +++ b/pkg/util/files.go @@ -2,20 +2,11 @@ package util import ( "bytes" - "context" "fmt" "io" "os" "path/filepath" - "strings" "text/template" - - "github.com/cnoe-io/idpbuilder/api/v1alpha1" - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/storage/memory" ) func CopyDirectory(scrDir, dest string) error { @@ -117,124 +108,3 @@ func ApplyTemplate(in []byte, templateData any) ([]byte, error) { return ret.Bytes(), nil } - -// returns all files with yaml or yml suffix from a worktree -func GetWorktreeYamlFiles(parent string, wt billy.Filesystem, recurse bool) ([]string, error) { - if strings.HasSuffix(parent, "/") { - parent = strings.TrimSuffix(parent, "/") - } - paths := make([]string, 0, 10) - ents, err := wt.ReadDir(parent) - if err != nil { - return nil, err - } - for i := range ents { - ent := ents[i] - if ent.IsDir() && recurse { - dir := fmt.Sprintf("%s/%s", parent, ent.Name()) - rPaths, dErr := GetWorktreeYamlFiles(dir, wt, recurse) - if dErr != nil { - return nil, fmt.Errorf("reading %s : %w", ent.Name(), dErr) - } - paths = append(paths, rPaths...) - } - if ent.Mode().IsRegular() && (strings.HasSuffix(ent.Name(), "yaml") || strings.HasSuffix(ent.Name(), "yml")) { - paths = append(paths, fmt.Sprintf("%s/%s", parent, ent.Name())) - } - } - return paths, nil -} - -func ReadWorktreeFile(wt billy.Filesystem, path string) ([]byte, error) { - f, fErr := wt.Open(path) - if fErr != nil { - return nil, fmt.Errorf("opening %s", path) - } - defer f.Close() - - b := new(bytes.Buffer) - _, fErr = b.ReadFrom(f) - if fErr != nil { - return nil, fmt.Errorf("reading %s", path) - } - - return b.Bytes(), nil -} - -func CloneRemoteRepoToMemory(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool) (billy.Filesystem, *git.Repository, error) { - cloneOptions := &git.CloneOptions{ - URL: remote.Url, - Depth: depth, - ShallowSubmodules: true, - SingleBranch: true, - Tags: git.AllTags, - InsecureSkipTLS: insecureSkipTLS, - } - if remote.CloneSubmodules { - cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth - } - - if remote.Ref != "" { - cloneOptions.ReferenceName = plumbing.NewTagReferenceName(remote.Ref) - } - - wt := memfs.New() - var cloned *git.Repository - cloned, err := git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) - if err != nil { - cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(remote.Ref) - cloned, err = git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) - if err != nil { - return nil, nil, err - } - } - return wt, cloned, nil -} - -func CopyTreeToTree(srcWT, dstWT billy.Filesystem, srcPath, dstPath string) error { - files, err := srcWT.ReadDir(srcPath) - if err != nil { - return err - } - - for i := range files { - srcFile := files[i] - fullSrcPath := filepath.Join(srcPath, srcFile.Name()) - fullDstPath := filepath.Join(dstPath, srcFile.Name()) - if srcFile.Mode().IsRegular() { - cErr := CopyWTFile(srcWT, dstWT, fullSrcPath, fullDstPath) - if cErr != nil { - return cErr - } - continue - } - - if srcFile.IsDir() { - dErr := CopyTreeToTree(srcWT, dstWT, fullSrcPath, fullDstPath) - if dErr != nil { - return dErr - } - } - } - return nil -} - -func CopyWTFile(srcWT, dstWT billy.Filesystem, srcFile, dstFile string) error { - newFile, err := dstWT.Create(dstFile) - if err != nil { - return fmt.Errorf("creating file %s: %w", dstFile, err) - } - defer newFile.Close() - - srcF, err := srcWT.Open(srcFile) - if err != nil { - return fmt.Errorf("reading file %s: %w", srcFile, err) - } - defer srcF.Close() - - _, err = io.Copy(newFile, srcF) - if err != nil { - return fmt.Errorf("copying file %s: %w", srcFile, err) - } - return nil -} diff --git a/pkg/util/fs_test.go b/pkg/util/fs_test.go index 1a6d1b57..da0a692e 100644 --- a/pkg/util/fs_test.go +++ b/pkg/util/fs_test.go @@ -1,21 +1,15 @@ package util import ( - "context" "fmt" "io/fs" "os" "path/filepath" - "strings" "testing" "testing/fstest" "github.com/cnoe-io/idpbuilder/globals" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/storage/memory" "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" ) func TestWriteFS(t *testing.T) { @@ -78,30 +72,3 @@ func TestWriteFS(t *testing.T) { }) } } - -func TestGetWorktreeYamlFiles(t *testing.T) { - filepath.Join() - cloneOptions := &git.CloneOptions{ - URL: "https://github.com/cnoe-io/idpbuilder", - Depth: 1, - ShallowSubmodules: true, - } - - wt := memfs.New() - _, err := git.CloneContext(context.Background(), memory.NewStorage(), wt, cloneOptions) - if err != nil { - t.Fatalf(err.Error()) - } - - paths, err := GetWorktreeYamlFiles("./pkg", wt, true) - - assert.Equal(t, nil, err) - assert.NotEqual(t, 0, len(paths)) - for _, s := range paths { - assert.Equal(t, true, strings.HasSuffix(s, "yaml") || strings.HasSuffix(s, "yml")) - } - - paths, err = GetWorktreeYamlFiles("./pkg", wt, false) - assert.Equal(t, nil, err) - assert.Equal(t, 0, len(paths)) -} diff --git a/pkg/util/git_repository.go b/pkg/util/git_repository.go new file mode 100644 index 00000000..5a78298e --- /dev/null +++ b/pkg/util/git_repository.go @@ -0,0 +1,260 @@ +package util + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "sync" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" +) + +type RepoMap struct { + repos sync.Map +} + +func (r *RepoMap) Get(repoName, dir string) *RepoState { + v, _ := r.repos.LoadOrStore(repoName, &RepoState{Dir: dir}) + return v.(*RepoState) +} + +type RepoState struct { + MU sync.Mutex + Dir string +} + +func NewRepoLock() *RepoMap { + return &RepoMap{ + repos: sync.Map{}, + } +} + +func RepoUrlHash(repoUrl string) string { + sha := sha256.New() + sha.Write([]byte(repoUrl)) + return hex.EncodeToString(sha.Sum(nil)) +} + +func RepoDir(repoUrl, parent string) string { + if parent == "" { + panic("no") + } + + return filepath.Join(parent, RepoUrlHash(repoUrl)) +} + +// returns all files with yaml or yml suffix from a worktree +func GetWorktreeYamlFiles(parent string, wt billy.Filesystem, recurse bool) ([]string, error) { + if strings.HasSuffix(parent, "/") { + parent = strings.TrimSuffix(parent, "/") + } + paths := make([]string, 0, 10) + ents, err := wt.ReadDir(parent) + if err != nil { + return nil, err + } + for i := range ents { + ent := ents[i] + if ent.IsDir() && recurse { + dir := fmt.Sprintf("%s/%s", parent, ent.Name()) + rPaths, dErr := GetWorktreeYamlFiles(dir, wt, recurse) + if dErr != nil { + return nil, fmt.Errorf("reading %s : %w", ent.Name(), dErr) + } + paths = append(paths, rPaths...) + } + if ent.Mode().IsRegular() && (strings.HasSuffix(ent.Name(), "yaml") || strings.HasSuffix(ent.Name(), "yml")) { + paths = append(paths, fmt.Sprintf("%s/%s", parent, ent.Name())) + } + } + return paths, nil +} + +func ReadWorktreeFile(wt billy.Filesystem, path string) ([]byte, error) { + f, fErr := wt.Open(path) + if fErr != nil { + return nil, fmt.Errorf("opening %s", path) + } + defer f.Close() + + b := new(bytes.Buffer) + _, fErr = b.ReadFrom(f) + if fErr != nil { + return nil, fmt.Errorf("reading %s", path) + } + + return b.Bytes(), nil +} + +func CloneRemoteRepoToMemory(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool) (billy.Filesystem, *git.Repository, error) { + cloneOptions := &git.CloneOptions{ + URL: remote.Url, + Depth: depth, + ShallowSubmodules: true, + SingleBranch: true, + Tags: git.AllTags, + InsecureSkipTLS: insecureSkipTLS, + } + if remote.CloneSubmodules { + cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth + } + + if remote.Ref != "" { + cloneOptions.ReferenceName = plumbing.NewTagReferenceName(remote.Ref) + } + + wt := memfs.New() + var cloned *git.Repository + cloned, err := git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(remote.Ref) + cloned, err = git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + return nil, nil, err + } + } + return wt, cloned, nil +} + +func CloneRemoteRepoToDir(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool, dir, fallbackUrl string) (billy.Filesystem, *git.Repository, error) { + repo, err := git.PlainOpen(dir) + if err != nil { + if errors.Is(err, git.ErrRepositoryNotExists) { + cloneOptions := &git.CloneOptions{ + URL: remote.Url, + Depth: depth, + ShallowSubmodules: true, + Tags: git.AllTags, + InsecureSkipTLS: insecureSkipTLS, + } + if remote.CloneSubmodules { + cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth + } + repo, err = git.PlainCloneContext(ctx, dir, false, cloneOptions) + if err != nil { + if fallbackUrl != "" { + cloneOptions.URL = fallbackUrl + repo, err = git.PlainCloneContext(ctx, dir, false, cloneOptions) + if err != nil { + return nil, nil, fmt.Errorf("cloning repo with fall back url: %w", err) + } + } + return nil, nil, fmt.Errorf("cloning repo: %w", err) + } + } else { + return nil, nil, fmt.Errorf("opening repo at %s %w", dir, err) + } + } + + wt, err := repo.Worktree() + if err != nil { + return nil, nil, fmt.Errorf("getting repo worktree: %w", err) + } + if remote.Ref != "" { + cErr := checkoutCommitOrRef(ctx, wt, remote.Ref) + if cErr != nil { + return nil, nil, fmt.Errorf("checkout %s: %w", remote.Ref, cErr) + } + } + + return wt.Filesystem, repo, nil +} + +func CopyTreeToTree(srcWT, dstWT billy.Filesystem, srcPath, dstPath string) error { + files, err := srcWT.ReadDir(srcPath) + if err != nil { + return err + } + + for i := range files { + srcFile := files[i] + fullSrcPath := filepath.Join(srcPath, srcFile.Name()) + fullDstPath := filepath.Join(dstPath, srcFile.Name()) + if srcFile.Mode().IsRegular() { + cErr := CopyWTFile(srcWT, dstWT, fullSrcPath, fullDstPath) + if cErr != nil { + return cErr + } + continue + } + + if srcFile.IsDir() { + dErr := CopyTreeToTree(srcWT, dstWT, fullSrcPath, fullDstPath) + if dErr != nil { + return dErr + } + } + } + return nil +} + +func CopyWTFile(srcWT, dstWT billy.Filesystem, srcFile, dstFile string) error { + newFile, err := dstWT.Create(dstFile) + if err != nil { + return fmt.Errorf("creating file %s: %w", dstFile, err) + } + defer newFile.Close() + + srcF, err := srcWT.Open(srcFile) + if err != nil { + return fmt.Errorf("reading file %s: %w", srcFile, err) + } + defer srcF.Close() + + _, err = io.Copy(newFile, srcF) + if err != nil { + return fmt.Errorf("copying file %s: %w", srcFile, err) + } + return nil +} + +// ref could be anything. Check if hash, tag, or branch in that order +func checkoutCommitOrRef(ctx context.Context, wt *git.Worktree, ref string) error { + var refName plumbing.ReferenceName + opts := &git.CheckoutOptions{ + Hash: plumbing.NewHash(ref), + } + + err := wt.Checkout(opts) + if err != nil { + refName = plumbing.NewTagReferenceName(ref) + opts = &git.CheckoutOptions{ + Branch: refName, + } + err := wt.Checkout(opts) + if err != nil { + refName = plumbing.NewBranchReferenceName(ref) + opts = &git.CheckoutOptions{ + Branch: refName, + } + err := wt.Checkout(opts) + if err != nil { + return err + } + } + } + pullOpts := &git.PullOptions{ + RemoteName: "origin", + } + + if opts.Hash.IsZero() { + pullOpts.ReferenceName = refName + err = wt.PullContext(ctx, pullOpts) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return fmt.Errorf("pulling latest %s: %w", ref, err) + } + } + + return nil +} diff --git a/pkg/util/git_repository_test.go b/pkg/util/git_repository_test.go new file mode 100644 index 00000000..7866ec12 --- /dev/null +++ b/pkg/util/git_repository_test.go @@ -0,0 +1,97 @@ +package util + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/assert" +) + +func TestCopyToDir(t *testing.T) { + spec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: "examples/basic", + Url: "../..", + Ref: "v0.3.0", + } + dir, _ := os.MkdirTemp("", "idpbuilder-test") + defer os.RemoveAll(dir) + // new clone + _, _, err := CloneRemoteRepoToDir(context.Background(), spec, 0, false, dir, "") + assert.Nil(t, err) + // existing + _, _, err = CloneRemoteRepoToDir(context.Background(), spec, 0, false, dir, "") + assert.Nil(t, err) +} + +func TestCopyTreeToTree(t *testing.T) { + spec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: "examples/basic", + Url: "https://github.com/cnoe-io/idpbuilder", + Ref: "", + } + + dst := memfs.New() + src, _, err := CloneRemoteRepoToMemory(context.Background(), spec, 1, false) + assert.Nil(t, err) + + err = CopyTreeToTree(src, dst, spec.Path, ".") + assert.Nil(t, err) + testCopiedFiles(t, src, dst, spec.Path, ".") +} + +func testCopiedFiles(t *testing.T, src, dst billy.Filesystem, srcStartPath, dstStartPath string) { + files, err := src.ReadDir(srcStartPath) + assert.Nil(t, err) + + for i := range files { + file := files[i] + if file.Mode().IsRegular() { + srcB, err := ReadWorktreeFile(src, filepath.Join(srcStartPath, file.Name())) + assert.Nil(t, err) + + dstB, err := ReadWorktreeFile(dst, filepath.Join(dstStartPath, file.Name())) + assert.Nil(t, err) + assert.Equal(t, srcB, dstB) + } + if file.IsDir() { + testCopiedFiles(t, src, dst, filepath.Join(srcStartPath, file.Name()), filepath.Join(dstStartPath, file.Name())) + } + } +} + +func TestGetWorktreeYamlFiles(t *testing.T) { + filepath.Join() + cloneOptions := &git.CloneOptions{ + URL: "https://github.com/cnoe-io/idpbuilder", + Depth: 1, + ShallowSubmodules: true, + } + + wt := memfs.New() + _, err := git.CloneContext(context.Background(), memory.NewStorage(), wt, cloneOptions) + if err != nil { + t.Fatalf(err.Error()) + } + + paths, err := GetWorktreeYamlFiles("./pkg", wt, true) + + assert.Equal(t, nil, err) + assert.NotEqual(t, 0, len(paths)) + for _, s := range paths { + assert.Equal(t, true, strings.HasSuffix(s, "yaml") || strings.HasSuffix(s, "yml")) + } + + paths, err = GetWorktreeYamlFiles("./pkg", wt, false) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(paths)) +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index ac154e65..10dec69c 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,15 +1,8 @@ package util import ( - "context" - "path/filepath" "strconv" "testing" - - "github.com/cnoe-io/idpbuilder/api/v1alpha1" - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/stretchr/testify/assert" ) var specialCharMap = make(map[string]struct{}) @@ -49,40 +42,3 @@ func TestGeneratePassword(t *testing.T) { } } } - -func TestCopyTreeToTree(t *testing.T) { - spec := v1alpha1.RemoteRepositorySpec{ - CloneSubmodules: false, - Path: "examples/basic", - Url: "https://github.com/cnoe-io/idpbuilder", - Ref: "", - } - - dst := memfs.New() - src, _, err := CloneRemoteRepoToMemory(context.Background(), spec, 1, false) - assert.Nil(t, err) - - err = CopyTreeToTree(src, dst, spec.Path, ".") - assert.Nil(t, err) - testCopiedFiles(t, src, dst, spec.Path, ".") -} - -func testCopiedFiles(t *testing.T, src, dst billy.Filesystem, srcStartPath, dstStartPath string) { - files, err := src.ReadDir(srcStartPath) - assert.Nil(t, err) - - for i := range files { - file := files[i] - if file.Mode().IsRegular() { - srcB, err := ReadWorktreeFile(src, filepath.Join(srcStartPath, file.Name())) - assert.Nil(t, err) - - dstB, err := ReadWorktreeFile(dst, filepath.Join(dstStartPath, file.Name())) - assert.Nil(t, err) - assert.Equal(t, srcB, dstB) - } - if file.IsDir() { - testCopiedFiles(t, src, dst, filepath.Join(srcStartPath, file.Name()), filepath.Join(dstStartPath, file.Name())) - } - } -}