diff --git a/README.md b/README.md index c56a3cd9..9d434150 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,10 @@ Run the following commands for available flags and subcommands: ### Custom Packages -Idpbuilder supports specifying custom packages using the flag `--package-dir` flag. This flag expects a directory containing ArgoCD application files. +Idpbuilder supports specifying custom packages using the flag `--package-dir` flag. +This flag expects a directory (local or remote) containing ArgoCD application files. +In case of a remote directory, it must be a directory in a git repository, +and the URL format must be a [kustomize remote URL format](https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md). Examples of using custom packages are available in the [example](./examples) directory. Let's take a look at [this example](examples/basic). This defines two custom package directories to deploy to the cluster. @@ -158,6 +161,12 @@ To deploy these packages, run the following commands from this repository's root ./idpbuilder create --package-dir examples/basic/package1 --package-dir examples/basic/package2 ``` +Alternatively, you can use the URL format: + +``` +./idpbuilder create --package-dir https://github.com/cnoe-io/idpbuilder//examples/basic/package1 --package-dir https://github.com/cnoe-io/idpbuilder//examples/basic/package2 +``` + Running this command should create three additional ArgoCD applications in your cluster. ```sh diff --git a/api/v1alpha1/custom_package_types.go b/api/v1alpha1/custom_package_types.go index 04620c6c..71c39c57 100644 --- a/api/v1alpha1/custom_package_types.go +++ b/api/v1alpha1/custom_package_types.go @@ -4,6 +4,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + CNOEURIScheme = "cnoe://" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status type CustomPackage struct { @@ -30,12 +34,23 @@ type CustomPackageSpec struct { GitServerAuthSecretRef SecretReference `json:"gitServerAuthSecretRef"` // InternalGitServeURL specifies the base URL for the git server accessible within the cluster. // for example, http://my-gitea-http.gitea.svc.cluster.local:3000 - InternalGitServeURL string `json:"internalGitServeURL"` + InternalGitServeURL string `json:"internalGitServeURL"` + RemoteRepository RemoteRepositorySpec `json:"remoteRepository"` // Replicate specifies whether to replicate remote or local contents to the local gitea server. // +kubebuilder:default:=false Replicate bool `json:"replicate"` } +// RemoteRepositorySpec specifies information about remote repositories. +type RemoteRepositorySpec struct { + CloneSubmodules bool `json:"cloneSubmodules"` + Path string `json:"path"` + // Url specifies the url to the repository containing the ArgoCD application file + Url string `json:"url"` + // Ref specifies the specific ref supported by git fetch + Ref string `json:"ref"` +} + type ArgoCDPackageSpec struct { // ApplicationFile specifies the absolute path to the ArgoCD application file ApplicationFile string `json:"applicationFile"` diff --git a/api/v1alpha1/gitrepository_types.go b/api/v1alpha1/gitrepository_types.go index 457c110c..d3ac32af 100644 --- a/api/v1alpha1/gitrepository_types.go +++ b/api/v1alpha1/gitrepository_types.go @@ -8,6 +8,9 @@ const ( GitProviderGitea = "gitea" GitProviderGitHub = "github" GiteaAdminUserName = "giteaAdmin" + SourceTypeLocal = "local" + SourceTypeRemote = "remote" + SourceTypeEmbedded = "embedded" ) type GitRepositorySpec struct { @@ -27,9 +30,10 @@ type GitRepositorySource struct { // Path is the absolute path to directory that contains Kustomize structure or raw manifests. // This is required when Type is set to local. // +kubebuilder:validation:Optional - Path string `json:"path"` + Path string `json:"path"` + RemoteRepository RemoteRepositorySpec `json:"remoteRepository"` // Type is the source type. - // +kubebuilder:validation:Enum:=local;embedded + // +kubebuilder:validation:Enum:=local;embedded;remote // +kubebuilder:default:=embedded Type string `json:"type"` } diff --git a/api/v1alpha1/localbuild_types.go b/api/v1alpha1/localbuild_types.go index bd08f86a..fe27e6d1 100644 --- a/api/v1alpha1/localbuild_types.go +++ b/api/v1alpha1/localbuild_types.go @@ -38,6 +38,7 @@ type PackageConfigsSpec struct { Argo ArgoPackageConfigSpec `json:"argoPackageConfigs,omitempty"` EmbeddedArgoApplications EmbeddedArgoApplicationsPackageConfigSpec `json:"embeddedArgoApplicationsPackageConfigs,omitempty"` CustomPackageDirs []string `json:"customPackageDirs,omitempty"` + CustomPackageUrls []string `json:"customPackageUrls,omitempty"` } type LocalbuildSpec struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c3cf1482..1061d56d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -148,6 +148,7 @@ func (in *CustomPackageSpec) DeepCopyInto(out *CustomPackageSpec) { *out = *in out.ArgoCD = in.ArgoCD out.GitServerAuthSecretRef = in.GitServerAuthSecretRef + out.RemoteRepository = in.RemoteRepository } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomPackageSpec. @@ -264,6 +265,7 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitRepositorySource) DeepCopyInto(out *GitRepositorySource) { *out = *in + out.RemoteRepository = in.RemoteRepository } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositorySource. @@ -459,6 +461,11 @@ func (in *PackageConfigsSpec) DeepCopyInto(out *PackageConfigsSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CustomPackageUrls != nil { + in, out := &in.CustomPackageUrls, &out.CustomPackageUrls + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackageConfigsSpec. @@ -501,6 +508,21 @@ func (in *Provider) DeepCopy() *Provider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteRepositorySpec) DeepCopyInto(out *RemoteRepositorySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteRepositorySpec. +func (in *RemoteRepositorySpec) DeepCopy() *RemoteRepositorySpec { + if in == nil { + return nil + } + out := new(RemoteRepositorySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretReference) DeepCopyInto(out *SecretReference) { *out = *in diff --git a/go.mod b/go.mod index c1465bd3..a8699204 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,12 @@ module github.com/cnoe-io/idpbuilder go 1.21.3 -toolchain go1.21.5 - require ( code.gitea.io/sdk/gitea v0.16.0 github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d github.com/docker/docker v24.0.7+incompatible - github.com/go-git/go-git/v5 v5.10.0 + github.com/go-git/go-billy/v5 v5.5.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/go-logr/logr v1.4.1 github.com/google/go-cmp v0.6.0 github.com/google/go-github/v61 v61.0.0 @@ -29,12 +28,11 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect @@ -49,7 +47,6 @@ require ( github.com/go-errors/errors v1.4.2 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect @@ -90,9 +87,9 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/sergi/go-diff v1.1.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skeema/knownhosts v1.2.0 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect diff --git a/go.sum b/go.sum index dfa88a66..17caef73 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,8 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -25,8 +23,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d h1:cmcSrS0OYTLlGsBFshaAG29qC+PC5LBtKRhnEknlzgU= github.com/cnoe-io/argocd-api v0.0.0-20240125015729-416a35fe855d/go.mod h1:IXG3LiEAeckMfjdwJnt6qC0ee4J4U5bleMuk1HN82ZA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -59,8 +58,8 @@ github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1 github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= @@ -71,8 +70,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ= -github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -140,8 +139,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -190,13 +187,13 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= diff --git a/pkg/build/build.go b/pkg/build/build.go index d6b78ecd..47036f02 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -3,9 +3,11 @@ package build import ( "context" "fmt" + "os" "time" "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/globals" "github.com/cnoe-io/idpbuilder/pkg/controllers" "github.com/cnoe-io/idpbuilder/pkg/kind" "github.com/cnoe-io/idpbuilder/pkg/util" @@ -32,28 +34,42 @@ type Build struct { kubeVersion string extraPortsMapping string customPackageDirs []string + customPackageUrls []string packageCustomization map[string]v1alpha1.PackageCustomization exitOnSync bool scheme *runtime.Scheme CancelFunc context.CancelFunc } -func NewBuild(name, kubeVersion, kubeConfigPath, kindConfigPath, extraPortsMapping string, cfg util.CorePackageTemplateConfig, - customPackageDirs []string, exitOnSync bool, scheme *runtime.Scheme, ctxCancel context.CancelFunc, - packageCustomization map[string]v1alpha1.PackageCustomization) *Build { +type NewBuildOptions struct { + Name string + TemplateData util.CorePackageTemplateConfig + KindConfigPath string + KubeConfigPath string + KubeVersion string + ExtraPortsMapping string + CustomPackageDirs []string + CustomPackageUrls []string + PackageCustomization map[string]v1alpha1.PackageCustomization + ExitOnSync bool + Scheme *runtime.Scheme + CancelFunc context.CancelFunc +} +func NewBuild(opts NewBuildOptions) *Build { return &Build{ - name: name, - kindConfigPath: kindConfigPath, - kubeConfigPath: kubeConfigPath, - kubeVersion: kubeVersion, - extraPortsMapping: extraPortsMapping, - customPackageDirs: customPackageDirs, - packageCustomization: packageCustomization, - exitOnSync: exitOnSync, - scheme: scheme, - cfg: cfg, - CancelFunc: ctxCancel, + name: opts.Name, + kindConfigPath: opts.KindConfigPath, + kubeConfigPath: opts.KubeConfigPath, + kubeVersion: opts.KubeVersion, + extraPortsMapping: opts.ExtraPortsMapping, + customPackageDirs: opts.CustomPackageDirs, + customPackageUrls: opts.CustomPackageUrls, + packageCustomization: opts.PackageCustomization, + exitOnSync: opts.ExitOnSync, + scheme: opts.Scheme, + cfg: opts.TemplateData, + CancelFunc: opts.CancelFunc, } } @@ -106,8 +122,8 @@ func (b *Build) ReconcileCRDs(ctx context.Context, kubeClient client.Client) err return nil } -func (b *Build) RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error) error { - return controllers.RunControllers(ctx, mgr, exitCh, b.CancelFunc, b.exitOnSync, b.cfg) +func (b *Build) RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, tmpDir string) error { + return controllers.RunControllers(ctx, mgr, exitCh, b.CancelFunc, b.exitOnSync, b.cfg, tmpDir) } func (b *Build) Run(ctx context.Context, recreateCluster bool) error { @@ -148,8 +164,16 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { return err } + dir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", globals.ProjectName, b.name)) + if err != nil { + setupLog.Error(err, "creating temp dir") + return err + } + defer os.RemoveAll(dir) + setupLog.V(1).Info("Created temp directory for cloning repositories", "dir", dir) + setupLog.V(1).Info("Running controllers") - if err := b.RunControllers(ctx, mgr, managerExit); err != nil { + if err := b.RunControllers(ctx, mgr, managerExit, dir); err != nil { setupLog.Error(err, "Error running controllers") return err } @@ -178,6 +202,7 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error { PackageCustomization: b.packageCustomization, }, CustomPackageDirs: b.customPackageDirs, + CustomPackageUrls: b.customPackageUrls, }, } diff --git a/pkg/cmd/create/root.go b/pkg/cmd/create/root.go index 333627ef..6408050c 100644 --- a/pkg/cmd/create/root.go +++ b/pkg/cmd/create/root.go @@ -56,7 +56,7 @@ func init() { CreateCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "Protocol to use to access web UIs. http or https.") CreateCmd.PersistentFlags().StringVar(&port, "port", "8443", "Port number under which idpBuilder tools are accessible.") CreateCmd.PersistentFlags().BoolVar(&pathRouting, "use-path-routing", false, "When set to true, web UIs are exposed under single domain name.") - CreateCmd.Flags().StringSliceVarP(&extraPackagesDirs, "package-dir", "p", []string{}, "Paths to custom packages") + CreateCmd.Flags().StringSliceVarP(&extraPackagesDirs, "package-dir", "p", []string{}, "Paths to directories containing custom packages") CreateCmd.Flags().StringSliceVarP(&packageCustomizationFiles, "package-custom-file", "c", []string{}, "Name of the package and the path to file to customize the package with. e.g. argocd:/tmp/argocd.yaml") // idpbuilder related flags CreateCmd.Flags().BoolVarP(&noExit, "no-exit", "n", true, "When set, idpbuilder will not exit after all packages are synced. Useful for continuously syncing local directories.") @@ -84,12 +84,15 @@ func create(cmd *cobra.Command, args []string) error { } var absDirPaths []string + var remotePaths []string + if len(extraPackagesDirs) > 0 { - p, err := helpers.GetAbsFilePaths(extraPackagesDirs, true) - if err != nil { - return err + r, l, pErr := helpers.ParsePackageStrings(extraPackagesDirs) + if pErr != nil { + return pErr } - absDirPaths = p + absDirPaths = l + remotePaths = r } o := make(map[string]v1alpha1.PackageCustomization) @@ -106,17 +109,31 @@ func create(cmd *cobra.Command, args []string) error { exitOnSync = !noExit } - b := build.NewBuild( - buildName, kubeVersion, kubeConfigPath, kindConfigPath, extraPortsMapping, - util.CorePackageTemplateConfig{ + opts := build.NewBuildOptions{ + Name: buildName, + KubeVersion: kubeVersion, + KubeConfigPath: kubeConfigPath, + KindConfigPath: kindConfigPath, + ExtraPortsMapping: extraPortsMapping, + + TemplateData: util.CorePackageTemplateConfig{ Protocol: protocol, Host: host, IngressHost: ingressHost, Port: port, UsePathRouting: pathRouting, }, - absDirPaths, exitOnSync, k8s.GetScheme(), ctxCancel, o, - ) + + CustomPackageDirs: absDirPaths, + CustomPackageUrls: remotePaths, + ExitOnSync: exitOnSync, + PackageCustomization: o, + + Scheme: k8s.GetScheme(), + CancelFunc: ctxCancel, + } + + b := build.NewBuild(opts) if err := b.Run(ctx, recreateCluster); err != nil { return err @@ -153,7 +170,9 @@ func validate() error { return pErr } } - return nil + + _, _, err = helpers.ParsePackageStrings(extraPackagesDirs) + return err } func getPackageCustomFile(input string) (v1alpha1.PackageCustomization, error) { diff --git a/pkg/cmd/get/secrets.go b/pkg/cmd/get/secrets.go index b5c4ec70..cd97ce0b 100644 --- a/pkg/cmd/get/secrets.go +++ b/pkg/cmd/get/secrets.go @@ -13,7 +13,6 @@ import ( "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/build" "github.com/cnoe-io/idpbuilder/pkg/k8s" - "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -59,8 +58,13 @@ func getSecretsE(cmd *cobra.Command, args []string) error { defer ctxCancel() kubeConfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config") - b := build.NewBuild("", "", kubeConfigPath, "", "", - util.CorePackageTemplateConfig{}, []string{}, false, k8s.GetScheme(), ctxCancel, nil) + opts := build.NewBuildOptions{ + KubeConfigPath: kubeConfigPath, + Scheme: k8s.GetScheme(), + CancelFunc: ctxCancel, + } + + b := build.NewBuild(opts) kubeConfig, err := b.GetKubeConfig() if err != nil { diff --git a/pkg/cmd/helpers/validation.go b/pkg/cmd/helpers/validation.go index 5e398873..68a425a2 100644 --- a/pkg/cmd/helpers/validation.go +++ b/pkg/cmd/helpers/validation.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/cnoe-io/idpbuilder/pkg/util" "sigs.k8s.io/kustomize/kyaml/kio" ) @@ -34,27 +35,53 @@ func ValidateKubernetesYamlFile(absPath string) error { return nil } +func ParsePackageStrings(pkgStrings []string) ([]string, []string, error) { + remote, local := make([]string, 0, 2), make([]string, 0, 2) + for i := range pkgStrings { + loc := pkgStrings[i] + _, err := util.NewKustomizeRemote(loc) + if err == nil { + remote = append(remote, loc) + continue + } + + absPath, err := getAbsPath(loc, true) + if err == nil { + local = append(local, absPath) + continue + } + return nil, nil, err + } + + return remote, local, nil +} + +func getAbsPath(path string, isDir bool) (string, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to validate path %s : %w", path, err) + } + f, err := os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("failed to validate path %s : %w", absPath, err) + } + if isDir && !f.IsDir() { + return "", fmt.Errorf("given path is not a directory. %s", absPath) + } + if !isDir && !f.Mode().IsRegular() { + return "", fmt.Errorf("give path is not a file. %s", absPath) + } + return absPath, nil +} + func GetAbsFilePaths(paths []string, isDir bool) ([]string, error) { out := make([]string, len(paths)) for i := range paths { - path := paths[i] - absPath, err := filepath.Abs(path) - if err != nil { - return nil, fmt.Errorf("failed to validate path %s : %w", path, err) - } - f, err := os.Stat(absPath) + absPath, err := getAbsPath(paths[i], isDir) if err != nil { - return nil, fmt.Errorf("failed to validate path %s : %w", absPath, err) - } - if isDir && !f.IsDir() { - return nil, fmt.Errorf("given path is not a directory. %s", absPath) - } - if !isDir && !f.Mode().IsRegular() { - return nil, fmt.Errorf("give path is not a file. %s", absPath) + return nil, err } - out[i] = absPath } - return out, nil } diff --git a/pkg/cmd/helpers/validation_test.go b/pkg/cmd/helpers/validation_test.go index 1134ce7a..e4d454be 100644 --- a/pkg/cmd/helpers/validation_test.go +++ b/pkg/cmd/helpers/validation_test.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/stretchr/testify/assert" ) func TestValidateKubernetesYaml(t *testing.T) { @@ -33,3 +35,40 @@ func TestValidateKubernetesYaml(t *testing.T) { } } } + +func TestParsePackageStrings(t *testing.T) { + cases := map[string]struct { + expectErr bool + inputPaths []string + remote int + local int + }{ + "allLocal": {expectErr: false, inputPaths: []string{"test-data", "."}, remote: 0, local: 2}, + "allRemote": {expectErr: false, inputPaths: []string{ + "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + "git@github.com:owner/repo//examples", + }, remote: 2, local: 0}, + "mix": {expectErr: false, inputPaths: []string{ + "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + "test-data", + }, remote: 1, local: 1}, + "invalidLocalPath": {expectErr: true, inputPaths: []string{ + "does-not-exist", + }, remote: 0, local: 0}, + "invalidRemotePath": {expectErr: true, inputPaths: []string{ + "https:// github.com/kubernetes-sigs/kustomize//examples", + }, remote: 0, local: 0}, + } + + for k := range cases { + c := cases[k] + remote, local, err := ParsePackageStrings(c.inputPaths) + if cases[k].expectErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, c.remote, len(remote)) + assert.Equal(t, c.local, len(local)) + } +} diff --git a/pkg/controllers/custompackage/controller.go b/pkg/controllers/custompackage/controller.go index a0c65685..b53d9f1e 100644 --- a/pkg/controllers/custompackage/controller.go +++ b/pkg/controllers/custompackage/controller.go @@ -8,11 +8,10 @@ import ( "strings" "time" - "github.com/cnoe-io/idpbuilder/api/v1alpha1" - "github.com/cnoe-io/idpbuilder/pkg/util" - argov1alpha1 "github.com/cnoe-io/argocd-api/api/argo/application/v1alpha1" + "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/k8s" + "github.com/cnoe-io/idpbuilder/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -32,6 +31,8 @@ type Reconciler struct { Recorder record.EventRecorder Scheme *runtime.Scheme Config util.CorePackageTemplateConfig + TempDir string + RepoMap *util.RepoMap } func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -71,7 +72,7 @@ func (r *Reconciler) postProcessReconcile(ctx context.Context, req ctrl.Request, // create an in-cluster repository CR, update the application spec, then apply func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alpha1.CustomPackage) (ctrl.Result, error) { - b, err := os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) + b, err := r.getArgoCDAppFile(ctx, resource) if err != nil { return ctrl.Result{}, fmt.Errorf("reading file %s: %w", resource.Spec.ArgoCD.ApplicationFile, err) } @@ -91,17 +92,17 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp app, ok := objs[0].(*argov1alpha1.Application) if !ok { - return ctrl.Result{}, fmt.Errorf("object is not an PackageSpec application %s", resource.Spec.ArgoCD.ApplicationFile) + return ctrl.Result{}, fmt.Errorf("object is not an ArgoCD application %s", resource.Spec.ArgoCD.ApplicationFile) } appName := app.GetName() if resource.Spec.Replicate { - repoRefs := make([]v1alpha1.ObjectRef, 0, 1) synced := true + repoRefs := make([]v1alpha1.ObjectRef, 0, 1) if app.Spec.HasMultipleSources() { for j := range app.Spec.Sources { s := &app.Spec.Sources[j] - res, repo, sErr := r.reconcileArgocdSource(ctx, resource, appName, resource.Spec.ArgoCD.ApplicationFile, s.RepoURL) + res, repo, sErr := r.reconcileArgoCDSource(ctx, resource, s, appName) if sErr != nil { return res, sErr } @@ -119,7 +120,7 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp } } else { s := app.Spec.Source - res, repo, sErr := r.reconcileArgocdSource(ctx, resource, appName, resource.Spec.ArgoCD.ApplicationFile, s.RepoURL) + res, repo, sErr := r.reconcileArgoCDSource(ctx, resource, s, appName) if sErr != nil { return res, sErr } @@ -161,37 +162,84 @@ func (r *Reconciler) reconcileCustomPackage(ctx context.Context, resource *v1alp return ctrl.Result{RequeueAfter: requeueTime}, nil } -func (r *Reconciler) reconcileArgocdSource(ctx context.Context, resource *v1alpha1.CustomPackage, appName, pkgDir, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { - logger := log.FromContext(ctx) - - process, absPath, err := isCNOEDirectory(pkgDir, repoURL) - if err != nil { - logger.Error(err, "processing argocd app source", "dir", pkgDir, "repoURL", repoURL) - return ctrl.Result{}, nil, err +func (r *Reconciler) reconcileArgoCDSource(ctx context.Context, resource *v1alpha1.CustomPackage, appSource *argov1alpha1.ApplicationSource, appName string) (ctrl.Result, *v1alpha1.GitRepository, error) { + if isCNOEScheme(appSource.RepoURL) { + if resource.Spec.RemoteRepository.Url == "" { + return r.reconcileArgoCDSourceFromLocal(ctx, resource, appName, appSource.RepoURL) + } + return r.reconcileArgoCDSourceFromRemote(ctx, resource, appName, appSource.RepoURL) } - if !process { - return ctrl.Result{}, nil, nil + return ctrl.Result{}, nil, nil +} + +func (r *Reconciler) reconcileArgoCDSourceFromRemote(ctx context.Context, resource *v1alpha1.CustomPackage, appName, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { + relativePath := strings.TrimPrefix(repoURL, v1alpha1.CNOEURIScheme) + // no guarantee that this path exists + dirPath := filepath.Join(resource.Spec.RemoteRepository.Path, relativePath) + + repo := &v1alpha1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: remoteRepoName(appName, dirPath, resource.Spec.RemoteRepository), + Namespace: resource.Namespace, + }, } - repo, err := r.reconcileGitRepo(ctx, resource, repoName(appName, absPath), absPath) - if err != nil { + cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { + if err := controllerutil.SetControllerReference(resource, repo, r.Scheme); err != nil { + return err + } + + if repo.ObjectMeta.Annotations == nil { + repo.ObjectMeta.Annotations = make(map[string]string) + } + util.SetCLIStartTimeAnnotationValue(repo.ObjectMeta.Annotations, cliStartTime) + + repo.Spec = v1alpha1.GitRepositorySpec{ + Source: v1alpha1.GitRepositorySource{ + Type: v1alpha1.SourceTypeRemote, + RemoteRepository: resource.Spec.RemoteRepository, + Path: dirPath, + }, + Provider: v1alpha1.Provider{ + Name: v1alpha1.GitProviderGitea, + GitURL: resource.Spec.GitServerURL, + InternalGitURL: resource.Spec.InternalGitServeURL, + OrganizationName: v1alpha1.GiteaAdminUserName, + }, + SecretRef: resource.Spec.GitServerAuthSecretRef, + } + + return nil + }) + + if err != nil && !errors.IsAlreadyExists(err) { return ctrl.Result{}, nil, err } return ctrl.Result{}, repo, nil } -func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.CustomPackage, repoName, absPath string) (*v1alpha1.GitRepository, error) { +func (r *Reconciler) reconcileArgoCDSourceFromLocal(ctx context.Context, resource *v1alpha1.CustomPackage, appName, repoURL string) (ctrl.Result, *v1alpha1.GitRepository, error) { + logger := log.FromContext(ctx) + + absPath, err := getCNOEAbsPath(resource.Spec.ArgoCD.ApplicationFile, repoURL) + if err != nil { + logger.Error(err, "processing argocd app source", "dir", resource.Spec.ArgoCD.ApplicationFile, "repoURL", repoURL) + return ctrl.Result{}, nil, err + } + repo := &v1alpha1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: repoName, + Name: localRepoName(appName, absPath), Namespace: resource.Namespace, }, } cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, repo, func() error { if err := controllerutil.SetControllerReference(resource, repo, r.Scheme); err != nil { return err } @@ -203,7 +251,7 @@ func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.Cu repo.Spec = v1alpha1.GitRepositorySpec{ Source: v1alpha1.GitRepositorySource{ - Type: "local", + Type: v1alpha1.SourceTypeLocal, Path: absPath, }, Provider: v1alpha1.Provider{ @@ -220,10 +268,10 @@ func (r *Reconciler) reconcileGitRepo(ctx context.Context, resource *v1alpha1.Cu // it's possible for an application to specify the same directory multiple times in the spec. // if there is a repository already created for this package, no further action is necessary. if !errors.IsAlreadyExists(err) { - return repo, err + return ctrl.Result{}, repo, err } - return repo, nil + return ctrl.Result{}, repo, nil } func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -232,27 +280,48 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func isCNOEDirectory(fPath, repoURL string) (bool, string, error) { - if strings.HasPrefix(repoURL, "cnoe://") { - parentDir := filepath.Dir(fPath) - relativePath := strings.TrimPrefix(repoURL, "cnoe://") - absPath, err := filepath.Abs(filepath.Join(parentDir, relativePath)) - if err != nil { - return false, "", err - } +func (r *Reconciler) getArgoCDAppFile(ctx context.Context, resource *v1alpha1.CustomPackage) ([]byte, error) { + if resource.Spec.RemoteRepository.Url == "" { + return os.ReadFile(resource.Spec.ArgoCD.ApplicationFile) + } - f, err := os.Stat(absPath) - if err != nil { - return false, "", err - } - if !f.IsDir() { - return false, "", fmt.Errorf("path not a directory: %s", absPath) - } - return true, absPath, err + cloneDir := util.RepoDir(resource.Spec.RemoteRepository.Url, r.TempDir) + st := r.RepoMap.LoadOrStore(resource.Spec.RemoteRepository.Url, cloneDir) + st.MU.Lock() + wt, _, err := util.CloneRemoteRepoToDir(ctx, resource.Spec.RemoteRepository, 1, false, cloneDir, "") + defer st.MU.Unlock() + if err != nil { + return nil, fmt.Errorf("cloning repo, %s: %w", resource.Spec.RemoteRepository.Url, err) } - return false, "", nil + return util.ReadWorktreeFile(wt, resource.Spec.ArgoCD.ApplicationFile) } -func repoName(appName, dir string) string { +func localRepoName(appName, dir string) string { return fmt.Sprintf("%s-%s", appName, filepath.Base(dir)) } + +func remoteRepoName(appName, pathToPkg string, repo v1alpha1.RemoteRepositorySpec) string { + return fmt.Sprintf("%s-%s", appName, filepath.Base(pathToPkg)) +} + +func isCNOEScheme(repoURL string) bool { + return strings.HasPrefix(repoURL, v1alpha1.CNOEURIScheme) +} + +func getCNOEAbsPath(fPath, repoURL string) (string, error) { + parentDir := filepath.Dir(fPath) + relativePath := strings.TrimPrefix(repoURL, v1alpha1.CNOEURIScheme) + absPath, err := filepath.Abs(filepath.Join(parentDir, relativePath)) + if err != nil { + return "", err + } + + f, err := os.Stat(absPath) + if err != nil { + return "", err + } + if !f.IsDir() { + return "", fmt.Errorf("path not a directory: %s", absPath) + } + return absPath, err +} diff --git a/pkg/controllers/custompackage/controller_test.go b/pkg/controllers/custompackage/controller_test.go index 6e9afa88..53baecef 100644 --- a/pkg/controllers/custompackage/controller_test.go +++ b/pkg/controllers/custompackage/controller_test.go @@ -156,7 +156,7 @@ func TestReconcileCustomPkg(t *testing.T) { c := mgr.GetClient() repo := v1alpha1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: repoName("my-app", "test/resources/customPackages/testDir/busybox"), + Name: localRepoName("my-app", "test/resources/customPackages/testDir/busybox"), Namespace: "test", }, } @@ -197,8 +197,8 @@ func TestReconcileCustomPkg(t *testing.T) { if err != nil { t.Fatalf("failed getting my-app %v", err) } - if strings.HasPrefix(localApp.Spec.Source.RepoURL, "cnoe://") { - t.Fatalf("cnoe:// prefix should be removed") + if strings.HasPrefix(localApp.Spec.Source.RepoURL, v1alpha1.CNOEURIScheme) { + t.Fatalf("%s prefix should be removed", v1alpha1.CNOEURIScheme) } for _, n := range []string{"guestbook", "guestbook2"} { @@ -219,7 +219,7 @@ func TestReconcileCustomPkg(t *testing.T) { t.Fatalf("failed getting my-app2 %v", err) } - if strings.HasPrefix(localApp2.Spec.Sources[0].RepoURL, "cnoe://") { - t.Fatalf("cnoe:// prefix should be removed") + if strings.HasPrefix(localApp2.Spec.Sources[0].RepoURL, v1alpha1.CNOEURIScheme) { + t.Fatalf("%s prefix should be removed", v1alpha1.CNOEURIScheme) } } diff --git a/pkg/controllers/gitrepository/controller.go b/pkg/controllers/gitrepository/controller.go index d98a7da7..d08740dd 100644 --- a/pkg/controllers/gitrepository/controller.go +++ b/pkg/controllers/gitrepository/controller.go @@ -7,14 +7,13 @@ import ( "fmt" "net" "net/http" - "os" - "path/filepath" "time" "code.gitea.io/sdk/gitea" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" @@ -47,6 +46,8 @@ type RepositoryReconciler struct { Scheme *runtime.Scheme Config util.CorePackageTemplateConfig GitProviderFunc gitProviderFunc + TempDir string + RepoMap *util.RepoMap } type gitProviderFunc func(context.Context, *v1alpha1.GitRepository, client.Client, *runtime.Scheme, util.CorePackageTemplateConfig) (gitProvider, error) @@ -65,6 +66,10 @@ func getOrganizationName(repo v1alpha1.GitRepository) string { return repo.Spec.Provider.OrganizationName } +func getFallbackRepositoryURL(repo *v1alpha1.GitRepository, info repoInfo) string { + return fmt.Sprintf("%s/%s.git", repo.Spec.Provider.GitURL, info.fullName) +} + func GetGitProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) (gitProvider, error) { switch repo.Spec.Provider.Name { case v1alpha1.GitProviderGitea: @@ -107,9 +112,6 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) } defer r.postProcessReconcile(ctx, req, &gitRepo) - if !r.shouldProcess(gitRepo) { - return ctrl.Result{Requeue: false}, nil - } logger.V(1).Info("reconciling GitRepository", "name", req.Name, "namespace", req.Namespace) result, err := r.reconcileGitRepo(ctx, &gitRepo) @@ -124,7 +126,6 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *RepositoryReconciler) postProcessReconcile(ctx context.Context, req ctrl.Request, repo *v1alpha1.GitRepository) { logger := log.FromContext(ctx) - err := r.Status().Update(ctx, repo) if err != nil { logger.Error(err, "failed updating repo status") @@ -155,6 +156,7 @@ func (r *RepositoryReconciler) reconcileGitRepo(ctx context.Context, repo *v1alp if err != nil { return ctrl.Result{}, fmt.Errorf("setting git provider credentials: %w", err) } + var providerRepo repoInfo p, err := provider.getRepository(ctx, repo) if err != nil { @@ -171,7 +173,7 @@ func (r *RepositoryReconciler) reconcileGitRepo(ctx context.Context, repo *v1alp providerRepo = p } - err = provider.updateRepoContent(ctx, repo, providerRepo, creds) + err = provider.updateRepoContent(ctx, repo, providerRepo, creds, r.TempDir, r.RepoMap) if err != nil { return ctrl.Result{}, fmt.Errorf("updating repository contents: %w", err) } @@ -189,94 +191,163 @@ func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager, notifyChan cha Complete(r) } -func (r *RepositoryReconciler) shouldProcess(repo v1alpha1.GitRepository) bool { - if repo.Spec.Source.Type == "local" && !filepath.IsAbs(repo.Spec.Source.Path) { - return false +func addAllAndCommit(path string, gitRepo *git.Repository) (plumbing.Hash, bool, error) { + tree, err := gitRepo.Worktree() + if err != nil { + return plumbing.Hash{}, false, fmt.Errorf("getting git worktree: %w", err) } - return true -} -func updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) error { - logger := log.FromContext(ctx) + err = tree.AddGlob("*") + if err != nil { + return plumbing.Hash{}, false, fmt.Errorf("adding git files: %w", err) + } - tempDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s", repo.Name, repo.Namespace)) - defer os.RemoveAll(tempDir) + status, err := tree.Status() if err != nil { - return fmt.Errorf("creating temporary directory: %w", err) + return plumbing.Hash{}, false, fmt.Errorf("getting git status: %w", err) + } + + if status.IsClean() { + h, _ := gitRepo.Head() + return h.Hash(), false, nil } + h, err := tree.Commit(fmt.Sprintf("updated from %s", path), &git.CommitOptions{ + All: true, + AllowEmptyCommits: false, + Author: &object.Signature{ + Name: gitCommitAuthorName, + Email: gitCommitAuthorEmail, + When: time.Now(), + }, + }) + return h, true, nil +} + +func pushToRemote(ctx context.Context, remoteRepo *git.Repository, creds gitProviderCredentials) error { auth, err := getBasicAuth(creds) if err != nil { return fmt.Errorf("getting basic auth: %w", err) } - - cloneOptions := &git.CloneOptions{ + return remoteRepo.PushContext(ctx, &git.PushOptions{ Auth: &auth, - URL: repoInfo.cloneUrl, - NoCheckout: true, InsecureSkipTLS: true, + }) +} + +// add files from local fs to target repository (gitea for now) +func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig, tmpDir string, repoMap *util.RepoMap) error { + logger := log.FromContext(ctx) + tgtCloneDir := util.RepoDir(tgtRepo.cloneUrl, tmpDir) + + st := repoMap.LoadOrStore(tgtRepo.cloneUrl, tgtCloneDir) + st.MU.Lock() + defer st.MU.Unlock() + + tgtRepoSpec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: ".", + Url: tgtRepo.cloneUrl, + Ref: "", } - clonedRepo, err := git.PlainCloneContext(ctx, tempDir, false, cloneOptions) + logger.V(1).Info("cloning repo", "repoUrl", tgtRepoSpec.Url, "fallbackUrl", getFallbackRepositoryURL(repo, tgtRepo), "cloneDir", tgtCloneDir) + _, tgtRepository, err := util.CloneRemoteRepoToDir(ctx, tgtRepoSpec, 1, true, tgtCloneDir, getFallbackRepositoryURL(repo, tgtRepo)) if err != nil { - // if we cannot clone with gitea's configured url, then we fallback to using the url provided in spec. - logger.V(1).Info("failed cloning with returned clone URL. Falling back to default url.", "err", err) - - cloneOptions.URL = fmt.Sprintf("%s/%s.git", repo.Spec.Provider.GitURL, repoInfo.fullName) - c, retErr := git.PlainCloneContext(ctx, tempDir, false, cloneOptions) - if retErr != nil { - return fmt.Errorf("cloning repo with fall back url: %w", retErr) - } - clonedRepo = c + return fmt.Errorf("cloning repo %s: %w", tgtRepoSpec.Url, err) } - err = writeRepoContents(repo, tempDir, tmplConfig, scheme) + err = writeRepoContents(repo, tgtCloneDir, tmplConfig, scheme) if err != nil { return fmt.Errorf("writing repo contents: %w", err) } - tree, err := clonedRepo.Worktree() + hash, push, err := addAllAndCommit(repo.Spec.Source.Path, tgtRepository) if err != nil { - return fmt.Errorf("getting git worktree: %w", err) + return fmt.Errorf("add and commit %w", err) } - err = tree.AddGlob("*") - if err != nil { - return fmt.Errorf("adding git files: %w", err) + if push { + remoteUrl, err := util.FirstRemoteURL(tgtRepository) + if err != nil { + return fmt.Errorf("getting remote url %w", err) + } + + logger.V(1).Info("pushing to remote url %s", remoteUrl) + err = pushToRemote(ctx, tgtRepository, creds) + if err != nil { + return fmt.Errorf("pushing to git: %w", err) + } + + repo.Status.LatestCommit.Hash = hash.String() + return nil } - status, err := tree.Status() + repo.Status.LatestCommit.Hash = hash.String() + return nil +} + +// add files from another repository at specified path to target repository (gitea for now) +func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error { + logger := log.FromContext(ctx) + srcRepo := repo.Spec.Source.RemoteRepository + cloneDir := util.RepoDir(srcRepo.Url, tmpDir) + + st := repoMap.LoadOrStore(srcRepo.Url, cloneDir) + st.MU.Lock() + defer st.MU.Unlock() + + logger.V(1).Info("cloning repo", "repoUrl", srcRepo.Url, "fallbackUrl", "", "cloneDir", cloneDir) + remoteWT, _, err := util.CloneRemoteRepoToDir(ctx, srcRepo, 1, false, cloneDir, "") if err != nil { - return fmt.Errorf("getting git status: %w", err) + return fmt.Errorf("cloning repo, %s: %w", srcRepo.Url, err) } - if status.IsClean() { - h, _ := clonedRepo.Head() - repo.Status.LatestCommit.Hash = h.Hash().String() - return nil + tgtRepoSpec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: ".", + Url: tgtRepo.cloneUrl, + Ref: "", } - commit, err := tree.Commit(fmt.Sprintf("updated from %s", repo.Spec.Source.Path), &git.CommitOptions{ - All: true, - AllowEmptyCommits: false, - Author: &object.Signature{ - Name: gitCommitAuthorName, - Email: gitCommitAuthorEmail, - When: time.Now(), - }, - }) + tgtCloneDir := util.RepoDir(tgtRepo.cloneUrl, tmpDir) + lst := repoMap.LoadOrStore(tgtRepoSpec.Url, tgtCloneDir) + + lst.MU.Lock() + defer lst.MU.Unlock() + + logger.V(1).Info("cloning repo", "repoUrl", tgtRepoSpec.Url, "fallbackUrl", getFallbackRepositoryURL(repo, tgtRepo), "cloneDir", tgtCloneDir) + tgtRepoWT, tgtRepository, err := util.CloneRemoteRepoToDir(ctx, tgtRepoSpec, 1, true, tgtCloneDir, getFallbackRepositoryURL(repo, tgtRepo)) if err != nil { - return fmt.Errorf("committing git files: %w", err) + return fmt.Errorf("cloning repo %s: %w", srcRepo.Url, err) } - err = clonedRepo.Push(&git.PushOptions{ - Auth: &auth, - InsecureSkipTLS: true, - }) + err = util.CopyTreeToTree(remoteWT, tgtRepoWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".") + if err != nil { + return fmt.Errorf("copying contents, %s: %w", tgtRepo.cloneUrl, err) + } + + hash, push, err := addAllAndCommit(repo.Spec.Source.Path, tgtRepository) if err != nil { - return fmt.Errorf("pushing to git: %w", err) + return fmt.Errorf("add and commit %w", err) + } + + if push { + remoteUrl, err := util.FirstRemoteURL(tgtRepository) + if err != nil { + return fmt.Errorf("getting remote url %w", err) + } + + logger.V(1).Info("pushing to remote url %s", remoteUrl) + err = pushToRemote(ctx, tgtRepository, creds) + if err != nil { + return fmt.Errorf("pushing to git: %w", err) + } + + repo.Status.LatestCommit.Hash = hash.String() + return nil } - repo.Status.LatestCommit.Hash = commit.String() + repo.Status.LatestCommit.Hash = hash.String() return nil } diff --git a/pkg/controllers/gitrepository/controller_test.go b/pkg/controllers/gitrepository/controller_test.go index f8e8eabc..41f98410 100644 --- a/pkg/controllers/gitrepository/controller_test.go +++ b/pkg/controllers/gitrepository/controller_test.go @@ -196,18 +196,21 @@ func setupDir() (string, error) { func TestGitRepositoryContentReconcile(t *testing.T) { ctx := context.Background() - dir, _, err := setUpLocalRepo() - defer os.RemoveAll(dir) + localRepoDir, _, err := setUpLocalRepo() + defer os.RemoveAll(localRepoDir) if err != nil { t.Fatalf("failed setting up local git repo: %v", err) } - addDir, err := setupDir() - defer os.RemoveAll(addDir) + srcDir, err := setupDir() + defer os.RemoveAll(srcDir) if err != nil { t.Fatalf("failed to set up dirs: %v", err) } + testCloneDir, _ := os.MkdirTemp("", "gitrepo-test") + defer os.RemoveAll(testCloneDir) + m := metav1.ObjectMeta{ Name: "test", Namespace: "test", @@ -216,7 +219,7 @@ func TestGitRepositoryContentReconcile(t *testing.T) { ObjectMeta: m, Spec: v1alpha1.GitRepositorySpec{ Source: v1alpha1.GitRepositorySource{ - Path: addDir, + Path: srcDir, Type: "local", }, }, @@ -228,38 +231,39 @@ func TestGitRepositoryContentReconcile(t *testing.T) { giteaClient: mockGitea{}, } // add file to source directory, reconcile, clone the repo and check if the added file exists - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: localRepoDir}, gitProviderCredentials{}, testCloneDir, util.NewRepoLock()) if err != nil { t.Fatalf("failed adding %v", err) } - tmpDir, _ := os.MkdirTemp("", "add") - defer os.RemoveAll(tmpDir) - repo, _ := git.PlainClone(tmpDir, false, &git.CloneOptions{ - URL: dir, + + repo, _ := git.PlainClone(testCloneDir, false, &git.CloneOptions{ + URL: localRepoDir, }) - c, err := os.ReadFile(filepath.Join(tmpDir, "add")) + c, err := os.ReadFile(filepath.Join(testCloneDir, "add")) if err != nil { - t.Fatalf("failed to read file at %s. %v", filepath.Join(tmpDir, "add"), err) + t.Fatalf("failed to read file at %s. %v", filepath.Join(testCloneDir, "add"), err) } if string(c) != addFileContent { t.Fatalf("expected %s, got %s", addFileContent, c) } // remove added file, reconcile, pull, check if the file is removed - err = os.Remove(filepath.Join(addDir, "add")) + err = os.Remove(filepath.Join(srcDir, "add")) if err != nil { t.Fatalf("failed to remove added file %v", err) } - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: localRepoDir}, gitProviderCredentials{}, testCloneDir, util.NewRepoLock()) if err != nil { t.Fatalf("failed removing %v", err) } + w, _ := repo.Worktree() err = w.Pull(&git.PullOptions{}) if err != nil { t.Fatalf("failed pulling changes %v", err) } - _, err = os.Stat(filepath.Join(tmpDir, "add")) + + _, err = os.Stat(filepath.Join(testCloneDir, "add")) if err == nil { t.Fatalf("file should not exist") } @@ -271,12 +275,15 @@ func TestGitRepositoryContentReconcile(t *testing.T) { func TestGitRepositoryContentReconcileEmbedded(t *testing.T) { ctx := context.Background() - dir, _, err := setUpLocalRepo() - defer os.RemoveAll(dir) + localRepoDir, _, err := setUpLocalRepo() + defer os.RemoveAll(localRepoDir) if err != nil { t.Fatalf("failed setting up local git repo: %v", err) } + tmpDir, _ := os.MkdirTemp("", "add") + defer os.RemoveAll(tmpDir) + m := metav1.ObjectMeta{ Name: "test", Namespace: "test", @@ -299,7 +306,7 @@ func TestGitRepositoryContentReconcileEmbedded(t *testing.T) { Client: &fakeClient{}, giteaClient: mockGitea{}, } - err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: dir}, gitProviderCredentials{}) + err = p.updateRepoContent(ctx, &resource, repoInfo{cloneUrl: localRepoDir}, gitProviderCredentials{}, tmpDir, util.NewRepoLock()) if err != nil { t.Fatalf("failed adding %v", err) } @@ -307,8 +314,8 @@ func TestGitRepositoryContentReconcileEmbedded(t *testing.T) { } func TestGitRepositoryReconcile(t *testing.T) { - dir, hash, err := setUpLocalRepo() - defer os.RemoveAll(dir) + localReoDir, hash, err := setUpLocalRepo() + defer os.RemoveAll(localReoDir) if err != nil { t.Fatalf("failed setting up local git repo: %v", err) } @@ -316,13 +323,19 @@ func TestGitRepositoryReconcile(t *testing.T) { if err != nil { t.Fatalf("failed to get absolute path: %v", err) } + updateDir, _, _ := setUpLocalRepo() + defer os.RemoveAll(updateDir) addDir, err := setupDir() + fmt.Println(addDir) defer os.RemoveAll(addDir) if err != nil { t.Fatalf("failed to set up dirs: %v", err) } + tmpDir, _ := os.MkdirTemp("", "gitrepo-test") + defer os.RemoveAll(tmpDir) + m := metav1.ObjectMeta{ Name: "test", Namespace: "test", @@ -332,7 +345,7 @@ func TestGitRepositoryReconcile(t *testing.T) { "no op": { giteaClient: mockGitea{ getRepo: func() (*gitea.Repository, *gitea.Response, error) { - return &gitea.Repository{CloneURL: dir}, nil, nil + return &gitea.Repository{CloneURL: localReoDir}, nil, nil }, }, input: v1alpha1.GitRepository{ @@ -350,7 +363,7 @@ func TestGitRepositoryReconcile(t *testing.T) { }, expect: expect{ resource: v1alpha1.GitRepositoryStatus{ - ExternalGitRepositoryUrl: dir, + ExternalGitRepositoryUrl: localReoDir, LatestCommit: v1alpha1.Commit{Hash: hash}, Synced: true, InternalGitRepositoryUrl: "http://cnoe.io/giteaAdmin/test-test.git", @@ -360,7 +373,7 @@ func TestGitRepositoryReconcile(t *testing.T) { "update": { giteaClient: mockGitea{ getRepo: func() (*gitea.Repository, *gitea.Response, error) { - return &gitea.Repository{CloneURL: dir}, nil, nil + return &gitea.Repository{CloneURL: updateDir}, nil, nil }, }, input: v1alpha1.GitRepository{ @@ -378,7 +391,7 @@ func TestGitRepositoryReconcile(t *testing.T) { }, expect: expect{ resource: v1alpha1.GitRepositoryStatus{ - ExternalGitRepositoryUrl: dir, + ExternalGitRepositoryUrl: updateDir, Synced: true, InternalGitRepositoryUrl: "http://cnoe.io/giteaAdmin/test-test.git", }, @@ -394,6 +407,8 @@ func TestGitRepositoryReconcile(t *testing.T) { r := RepositoryReconciler{ Client: &fakeClient{}, GitProviderFunc: v.giteaProvider, + TempDir: tmpDir, + RepoMap: util.NewRepoLock(), } _, err := r.reconcileGitRepo(ctx, &v.input) if v.expect.err == nil && err != nil { @@ -403,7 +418,6 @@ func TestGitRepositoryReconcile(t *testing.T) { if v.expect.resource.LatestCommit.Hash == "" { v.expect.resource.LatestCommit.Hash = v.input.Status.LatestCommit.Hash } - time.Sleep(100 * time.Millisecond) assert.Equal(t, v.input.Status, v.expect.resource) } }) @@ -411,8 +425,12 @@ func TestGitRepositoryReconcile(t *testing.T) { func TestGitRepositoryPostReconcile(t *testing.T) { c := fakeClient{} + tmpDir, _ := os.MkdirTemp("", "repo-updates-test") + defer os.RemoveAll(tmpDir) reconciler := RepositoryReconciler{ - Client: &c, + Client: &c, + TempDir: tmpDir, + RepoMap: util.NewRepoLock(), } testTime := time.Now().Format(time.RFC3339Nano) repo := v1alpha1.GitRepository{ diff --git a/pkg/controllers/gitrepository/git_repository.go b/pkg/controllers/gitrepository/git_repository.go index e218e1ee..a476ae99 100644 --- a/pkg/controllers/gitrepository/git_repository.go +++ b/pkg/controllers/gitrepository/git_repository.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/pkg/util" "github.com/google/go-github/v61/github" ) @@ -44,5 +45,5 @@ type gitProvider interface { getProviderCredentials(ctx context.Context, repo *v1alpha1.GitRepository) (gitProviderCredentials, error) getRepository(ctx context.Context, repo *v1alpha1.GitRepository) (repoInfo, error) setProviderCredentials(ctx context.Context, repo *v1alpha1.GitRepository, creds gitProviderCredentials) error - updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error + updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error } diff --git a/pkg/controllers/gitrepository/gitea.go b/pkg/controllers/gitrepository/gitea.go index 288c1cc4..2a9db6c9 100644 --- a/pkg/controllers/gitrepository/gitea.go +++ b/pkg/controllers/gitrepository/gitea.go @@ -40,6 +40,7 @@ func (g *giteaProvider) createRepository(ctx context.Context, repo *v1alpha1.Git DefaultBranch: DefaultBranchName, AutoInit: true, }) + if err != nil { return repoInfo{}, fmt.Errorf("failed to create git repository: %w", err) } @@ -97,8 +98,22 @@ func (g *giteaProvider) getRepository(ctx context.Context, repo *v1alpha1.GitRep }, nil } -func (g *giteaProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { - return updateRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) +func (g *giteaProvider) updateRepoContent( + ctx context.Context, + repo *v1alpha1.GitRepository, + repoInfo repoInfo, + creds gitProviderCredentials, + tmpDir string, + repoMap *util.RepoMap, +) error { + switch repo.Spec.Source.Type { + case v1alpha1.SourceTypeLocal, v1alpha1.SourceTypeEmbedded: + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, tmpDir, repoMap) + case v1alpha1.SourceTypeRemote: + return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds, tmpDir, repoMap) + default: + return nil + } } func writeRepoContents(repo *v1alpha1.GitRepository, dstPath string, config util.CorePackageTemplateConfig, scheme *runtime.Scheme) error { diff --git a/pkg/controllers/gitrepository/github.go b/pkg/controllers/gitrepository/github.go index ba21a27b..2a2ac0c3 100644 --- a/pkg/controllers/gitrepository/github.go +++ b/pkg/controllers/gitrepository/github.go @@ -102,8 +102,15 @@ func (g *gitHubProvider) setProviderCredentials(ctx context.Context, repo *v1alp return g.gitHubClient.setToken(creds.accessToken) } -func (g *gitHubProvider) updateRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, repoInfo repoInfo, creds gitProviderCredentials) error { - return updateRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config) +func (g *gitHubProvider) updateRepoContent( + ctx context.Context, + repo *v1alpha1.GitRepository, + repoInfo repoInfo, + creds gitProviderCredentials, + tmpDir string, + repoMap *util.RepoMap, +) error { + return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, tmpDir, repoMap) } func newGitHubClient(httpClient *http.Client) gitHubClient { diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index 47b67182..aaeb22f8 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -8,16 +8,17 @@ import ( "strings" "time" + argocdapp "github.com/cnoe-io/argocd-api/api/argo/application" "github.com/cnoe-io/idpbuilder/pkg/util" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" argov1alpha1 "github.com/cnoe-io/argocd-api/api/argo/application/v1alpha1" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/globals" "github.com/cnoe-io/idpbuilder/pkg/resources/localbuild" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -42,6 +43,8 @@ type LocalbuildReconciler struct { ExitOnSync bool shouldShutdown bool Config util.CorePackageTemplateConfig + TempDir string + RepoMap *util.RepoMap } type subReconciler func(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild) (ctrl.Result, error) @@ -140,12 +143,18 @@ func (r *LocalbuildReconciler) ReconcileArgoAppsWithGitea(ctx context.Context, r return result, fmt.Errorf("reconciling bootstrap apps %w", err) } } - if resource.Spec.PackageConfigs.CustomPackageDirs != nil { - for i := range resource.Spec.PackageConfigs.CustomPackageDirs { - result, err := r.reconcileCustomPkg(ctx, resource, resource.Spec.PackageConfigs.CustomPackageDirs[i]) - if err != nil { - return result, err - } + + for _, s := range resource.Spec.PackageConfigs.CustomPackageDirs { + result, err := r.reconcileCustomPkgDir(ctx, resource, s) + if err != nil { + return result, err + } + } + + for _, s := range resource.Spec.PackageConfigs.CustomPackageUrls { + result, err := r.reconcileCustomPkgUrl(ctx, resource, s) + if err != nil { + return result, err } } @@ -180,7 +189,7 @@ func (r *LocalbuildReconciler) reconcileEmbeddedApp(ctx context.Context, appName } err = r.Client.Get(ctx, client.ObjectKeyFromObject(app), app) - if err != nil && errors.IsNotFound(err) { + if err != nil && k8serrors.IsNotFound(err) { localbuild.SetApplicationSpec( app, repo.Status.InternalGitRepositoryUrl, @@ -220,7 +229,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.Annotations) if err != nil { - return true, err + return false, err } repos := &v1alpha1.GitRepositoryList{} @@ -228,6 +237,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a if err != nil { return false, fmt.Errorf("listing repositories %w", err) } + for i := range repos.Items { repo := repos.Items[i] @@ -258,7 +268,6 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a if err != nil { return false, fmt.Errorf("listing custom packages %w", err) } - for i := range pkgs.Items { pkg := pkgs.Items[i] startTimeAnnotation, gErr := util.GetCLIStartTimeAnnotationValue(pkg.ObjectMeta.Annotations) @@ -267,7 +276,7 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a } if startTimeAnnotation != cliStartTime { - continue + return false, nil } observedTime, gErr := util.GetLastObservedSyncTimeAnnotationValue(pkg.ObjectMeta.Annotations) @@ -275,7 +284,6 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a logger.Info(gErr.Error()) return false, nil } - if !pkg.Status.Synced || cliStartTime != observedTime { return false, nil } @@ -284,7 +292,116 @@ func (r *LocalbuildReconciler) shouldShutDown(ctx context.Context, resource *v1a return true, nil } -func (r *LocalbuildReconciler) reconcileCustomPkg(ctx context.Context, resource *v1alpha1.Localbuild, pkgDir string) (ctrl.Result, error) { +func (r *LocalbuildReconciler) reconcileCustomPkg( + ctx context.Context, + resource *v1alpha1.Localbuild, + b []byte, + filePath string, + remote *util.KustomizeRemote, +) error { + o := &unstructured.Unstructured{} + _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) + if fErr != nil { + return fErr + } + + if isSupportedArgoCDTypes(gvk) { + appName := o.GetName() + appNS := o.GetNamespace() + customPkg := &v1alpha1.CustomPackage{ + ObjectMeta: metav1.ObjectMeta{ + Name: getCustomPackageName(filepath.Base(filePath), appName), + Namespace: globals.GetProjectNamespace(resource.Name), + }, + } + + cliStartTime, _ := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) + + _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { + if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { + return err + } + if customPkg.ObjectMeta.Annotations == nil { + customPkg.ObjectMeta.Annotations = make(map[string]string) + } + + util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) + + customPkg.Spec = v1alpha1.CustomPackageSpec{ + Replicate: true, + GitServerURL: resource.Status.Gitea.ExternalURL, + InternalGitServeURL: resource.Status.Gitea.InternalURL, + GitServerAuthSecretRef: v1alpha1.SecretReference{ + Name: resource.Status.Gitea.AdminUserSecretName, + Namespace: resource.Status.Gitea.AdminUserSecretNamespace, + }, + ArgoCD: v1alpha1.ArgoCDPackageSpec{ + ApplicationFile: filePath, + Name: appName, + Namespace: appNS, + }, + } + + if remote != nil { + customPkg.Spec.RemoteRepository = v1alpha1.RemoteRepositorySpec{ + Url: remote.CloneUrl(), + Ref: remote.Ref, + CloneSubmodules: remote.Submodules, + Path: remote.Path(), + } + } + + return nil + }) + return fErr + } + return nil +} + +func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resource *v1alpha1.Localbuild, pkgUrl string) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + remote, err := util.NewKustomizeRemote(pkgUrl) + if err != nil { + return ctrl.Result{}, fmt.Errorf("parsing url, %s: %w", pkgUrl, err) + } + rs := v1alpha1.RemoteRepositorySpec{ + Url: remote.CloneUrl(), + Ref: remote.Ref, + CloneSubmodules: remote.Submodules, + Path: remote.Path(), + } + + cloneDir := util.RepoDir(rs.Url, r.TempDir) + st := r.RepoMap.LoadOrStore(rs.Url, cloneDir) + st.MU.Lock() + defer st.MU.Unlock() + wt, _, err := util.CloneRemoteRepoToDir(ctx, rs, 1, false, cloneDir, "") + if err != nil { + return ctrl.Result{}, fmt.Errorf("cloning repo, %s: %w", pkgUrl, err) + } + + yamlFiles, err := util.GetWorktreeYamlFiles(remote.Path(), wt, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("getting yaml files from repo, %s: %w", pkgUrl, err) + } + + for _, yamlFile := range yamlFiles { + b, fErr := util.ReadWorktreeFile(wt, yamlFile) + if fErr != nil { + logger.V(1).Info("processing", "file", yamlFile, "err", fErr) + continue + } + + rErr := r.reconcileCustomPkg(ctx, resource, b, yamlFile, remote) + if rErr != nil { + logger.Error(rErr, "reconciling custom pkg", "file", yamlFile, "pkgUrl", pkgUrl) + } + } + return ctrl.Result{}, nil +} + +func (r *LocalbuildReconciler) reconcileCustomPkgDir(ctx context.Context, resource *v1alpha1.Localbuild, pkgDir string) (ctrl.Result, error) { logger := log.FromContext(ctx) files, err := os.ReadDir(pkgDir) @@ -305,56 +422,9 @@ func (r *LocalbuildReconciler) reconcileCustomPkg(ctx context.Context, resource continue } - o := &unstructured.Unstructured{} - _, gvk, fErr := scheme.Codecs.UniversalDeserializer().Decode(b, nil, o) - if fErr != nil { - continue - } - if gvk.Kind == "Application" && gvk.Group == "argoproj.io" { - appName := o.GetName() - appNS := o.GetNamespace() - customPkg := &v1alpha1.CustomPackage{ - ObjectMeta: metav1.ObjectMeta{ - Name: getCustomPackageName(file.Name(), appName), - Namespace: globals.GetProjectNamespace(resource.Name), - }, - } - - cliStartTime, err := util.GetCLIStartTimeAnnotationValue(resource.ObjectMeta.Annotations) - if err != nil { - logger.Error(err, "this resource may not sync correctly") - } - - _, fErr = controllerutil.CreateOrUpdate(ctx, r.Client, customPkg, func() error { - if err := controllerutil.SetControllerReference(resource, customPkg, r.Scheme); err != nil { - return err - } - if customPkg.ObjectMeta.Annotations == nil { - customPkg.ObjectMeta.Annotations = make(map[string]string) - } - - util.SetCLIStartTimeAnnotationValue(customPkg.ObjectMeta.Annotations, cliStartTime) - - customPkg.Spec = v1alpha1.CustomPackageSpec{ - Replicate: true, - GitServerURL: resource.Status.Gitea.ExternalURL, - InternalGitServeURL: resource.Status.Gitea.InternalURL, - GitServerAuthSecretRef: v1alpha1.SecretReference{ - Name: resource.Status.Gitea.AdminUserSecretName, - Namespace: resource.Status.Gitea.AdminUserSecretNamespace, - }, - ArgoCD: v1alpha1.ArgoCDPackageSpec{ - ApplicationFile: filePath, - Name: appName, - Namespace: appNS, - }, - } - return nil - }) - if fErr != nil { - logger.Error(fErr, "failed creating custom package object", "name", appName, "namespace", appNS) - continue - } + rErr := r.reconcileCustomPkg(ctx, resource, b, filePath, nil) + if rErr != nil { + logger.Error(rErr, "reconciling custom pkg", "file", filePath, "pkgDir", pkgDir) } } @@ -400,7 +470,7 @@ func (r *LocalbuildReconciler) reconcileGitRepo(ctx context.Context, resource *v }, } - if repoType == "embedded" { + if repoType == v1alpha1.SourceTypeEmbedded { repo.Spec.Source.EmbeddedAppName = embeddedName } else { repo.Spec.Source.Path = absPath @@ -423,6 +493,13 @@ func getCustomPackageName(fileName, appName string) string { return fmt.Sprintf("%s-%s", strings.ToLower(s[0]), appName) } +func isSupportedArgoCDTypes(gvk *schema.GroupVersionKind) bool { + if gvk == nil { + return false + } + return gvk.Kind == argocdapp.ApplicationKind && gvk.Group == argocdapp.Group +} + func GetEmbeddedRawInstallResources(name string, templateData any, config v1alpha1.PackageCustomization, scheme *runtime.Scheme) ([][]byte, error) { switch name { case "argocd": diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml index 953cc204..507dd2f8 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_custompackages.yaml @@ -74,6 +74,27 @@ spec: InternalGitServeURL specifies the base URL for the git server accessible within the cluster. for example, http://my-gitea-http.gitea.svc.cluster.local:3000 type: string + remoteRepository: + description: RemoteRepositorySpec specifies information about remote + repositories. + properties: + cloneSubmodules: + type: boolean + path: + type: string + ref: + description: Ref specifies the specific ref supported by git fetch + type: string + url: + description: Url specifies the url to the repository containing + the ArgoCD application file + type: string + required: + - cloneSubmodules + - path + - ref + - url + type: object replicate: default: false description: Replicate specifies whether to replicate remote or local @@ -83,6 +104,7 @@ spec: - gitServerAuthSecretRef - gitServerURL - internalGitServeURL + - remoteRepository - replicate type: object status: diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml index b334960a..498b932c 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_gitrepositories.yaml @@ -100,14 +100,38 @@ spec: Path is the absolute path to directory that contains Kustomize structure or raw manifests. This is required when Type is set to local. type: string + remoteRepository: + description: RemoteRepositorySpec specifies information about + remote repositories. + properties: + cloneSubmodules: + type: boolean + path: + type: string + ref: + description: Ref specifies the specific ref supported by git + fetch + type: string + url: + description: Url specifies the url to the repository containing + the ArgoCD application file + type: string + required: + - cloneSubmodules + - path + - ref + - url + type: object type: default: embedded description: Type is the source type. enum: - local - embedded + - remote type: string required: + - remoteRepository - type type: object required: diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml index 6ac9b1be..b215d3b1 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml @@ -52,6 +52,10 @@ spec: items: type: string type: array + customPackageUrls: + items: + type: string + type: array embeddedArgoApplicationsPackageConfigs: description: EmbeddedArgoApplicationsPackageConfigSpec Controls the installation of the embedded argo applications. diff --git a/pkg/controllers/run.go b/pkg/controllers/run.go index f72262f8..19eae91f 100644 --- a/pkg/controllers/run.go +++ b/pkg/controllers/run.go @@ -12,9 +12,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ctxCancel context.CancelFunc, exitOnSync bool, cfg util.CorePackageTemplateConfig) error { +func RunControllers( + ctx context.Context, + mgr manager.Manager, + exitCh chan error, + ctxCancel context.CancelFunc, + exitOnSync bool, + cfg util.CorePackageTemplateConfig, + tmpDir string, +) error { logger := log.FromContext(ctx) + repoMap := util.NewRepoLock() + // Run Localbuild controller if err := (&localbuild.LocalbuildReconciler{ Client: mgr.GetClient(), @@ -22,6 +32,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, ExitOnSync: exitOnSync, CancelFunc: ctxCancel, Config: cfg, + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create localbuild controller") return err @@ -33,6 +45,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, Recorder: mgr.GetEventRecorderFor("gitrepository-controller"), Config: cfg, GitProviderFunc: gitrepository.GetGitProvider, + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr, nil) if err != nil { logger.Error(err, "unable to create repo controller") @@ -42,6 +56,8 @@ func RunControllers(ctx context.Context, mgr manager.Manager, exitCh chan error, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("custompackage-controller"), + TempDir: tmpDir, + RepoMap: repoMap, }).SetupWithManager(mgr) if err != nil { logger.Error(err, "unable to create custom package controller") diff --git a/pkg/util/git_repository.go b/pkg/util/git_repository.go new file mode 100644 index 00000000..a5949938 --- /dev/null +++ b/pkg/util/git_repository.go @@ -0,0 +1,271 @@ +package util + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "sync" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" +) + +type RepoMap struct { + repos sync.Map +} + +func (r *RepoMap) LoadOrStore(repoName, dir string) *RepoState { + v, _ := r.repos.LoadOrStore(repoName, &RepoState{Dir: dir}) + return v.(*RepoState) +} + +type RepoState struct { + MU sync.Mutex + Dir string +} + +func NewRepoLock() *RepoMap { + return &RepoMap{ + repos: sync.Map{}, + } +} + +func RepoUrlHash(repoUrl string) string { + sha := sha256.New() + sha.Write([]byte(repoUrl)) + return hex.EncodeToString(sha.Sum(nil)) +} + +func RepoDir(repoUrl, parent string) string { + return filepath.Join(parent, RepoUrlHash(repoUrl)) +} + +func FirstRemoteURL(repo *git.Repository) (string, error) { + remotes, err := repo.Remotes() + if err != nil { + return "", err + } + if len(remotes) <= 0 { + return "", fmt.Errorf("no remotes found") + } + r := remotes[0].Config().URLs + if len(r) <= 0 { + return "", fmt.Errorf("no remote url found") + } + return r[0], nil +} + +// returns all files with yaml or yml suffix from a worktree +func GetWorktreeYamlFiles(parent string, wt billy.Filesystem, recurse bool) ([]string, error) { + if strings.HasSuffix(parent, "/") { + parent = strings.TrimSuffix(parent, "/") + } + paths := make([]string, 0, 10) + ents, err := wt.ReadDir(parent) + if err != nil { + return nil, err + } + for i := range ents { + ent := ents[i] + if ent.IsDir() && recurse { + dir := fmt.Sprintf("%s/%s", parent, ent.Name()) + rPaths, dErr := GetWorktreeYamlFiles(dir, wt, recurse) + if dErr != nil { + return nil, fmt.Errorf("reading %s : %w", ent.Name(), dErr) + } + paths = append(paths, rPaths...) + } + if ent.Mode().IsRegular() && (strings.HasSuffix(ent.Name(), "yaml") || strings.HasSuffix(ent.Name(), "yml")) { + paths = append(paths, fmt.Sprintf("%s/%s", parent, ent.Name())) + } + } + return paths, nil +} + +func ReadWorktreeFile(wt billy.Filesystem, path string) ([]byte, error) { + f, fErr := wt.Open(path) + if fErr != nil { + return nil, fmt.Errorf("opening %s", path) + } + defer f.Close() + + b := new(bytes.Buffer) + _, fErr = b.ReadFrom(f) + if fErr != nil { + return nil, fmt.Errorf("reading %s", path) + } + + return b.Bytes(), nil +} + +func CloneRemoteRepoToMemory(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool) (billy.Filesystem, *git.Repository, error) { + cloneOptions := &git.CloneOptions{ + URL: remote.Url, + Depth: depth, + ShallowSubmodules: true, + SingleBranch: true, + Tags: git.AllTags, + InsecureSkipTLS: insecureSkipTLS, + } + if remote.CloneSubmodules { + cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth + } + + if remote.Ref != "" { + cloneOptions.ReferenceName = plumbing.NewTagReferenceName(remote.Ref) + } + + wt := memfs.New() + var cloned *git.Repository + cloned, err := git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(remote.Ref) + cloned, err = git.CloneContext(ctx, memory.NewStorage(), wt, cloneOptions) + if err != nil { + return nil, nil, err + } + } + return wt, cloned, nil +} + +func CloneRemoteRepoToDir(ctx context.Context, remote v1alpha1.RemoteRepositorySpec, depth int, insecureSkipTLS bool, dir, fallbackUrl string) (billy.Filesystem, *git.Repository, error) { + repo, err := git.PlainOpen(dir) + if err != nil { + if errors.Is(err, git.ErrRepositoryNotExists) { + cloneOptions := &git.CloneOptions{ + URL: remote.Url, + Depth: depth, + ShallowSubmodules: true, + Tags: git.AllTags, + InsecureSkipTLS: insecureSkipTLS, + } + if remote.CloneSubmodules { + cloneOptions.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth + } + repo, err = git.PlainCloneContext(ctx, dir, false, cloneOptions) + if err != nil { + if fallbackUrl != "" { + cloneOptions.URL = fallbackUrl + repo, err = git.PlainCloneContext(ctx, dir, false, cloneOptions) + if err != nil { + return nil, nil, fmt.Errorf("cloning repo with fall back url: %w", err) + } + } + return nil, nil, fmt.Errorf("cloning repo: %w", err) + } + } else { + return nil, nil, fmt.Errorf("opening repo at %s %w", dir, err) + } + } + + wt, err := repo.Worktree() + if err != nil { + return nil, nil, fmt.Errorf("getting repo worktree: %w", err) + } + if remote.Ref != "" { + cErr := checkoutCommitOrRef(ctx, wt, remote.Ref) + if cErr != nil { + return nil, nil, fmt.Errorf("checkout %s: %w", remote.Ref, cErr) + } + } + + return wt.Filesystem, repo, nil +} + +func CopyTreeToTree(srcWT, dstWT billy.Filesystem, srcPath, dstPath string) error { + files, err := srcWT.ReadDir(srcPath) + if err != nil { + return err + } + + for i := range files { + srcFile := files[i] + fullSrcPath := filepath.Join(srcPath, srcFile.Name()) + fullDstPath := filepath.Join(dstPath, srcFile.Name()) + if srcFile.Mode().IsRegular() { + cErr := CopyWTFile(srcWT, dstWT, fullSrcPath, fullDstPath) + if cErr != nil { + return cErr + } + continue + } + + if srcFile.IsDir() { + dErr := CopyTreeToTree(srcWT, dstWT, fullSrcPath, fullDstPath) + if dErr != nil { + return dErr + } + } + } + return nil +} + +func CopyWTFile(srcWT, dstWT billy.Filesystem, srcFile, dstFile string) error { + newFile, err := dstWT.Create(dstFile) + if err != nil { + return fmt.Errorf("creating file %s: %w", dstFile, err) + } + defer newFile.Close() + + srcF, err := srcWT.Open(srcFile) + if err != nil { + return fmt.Errorf("reading file %s: %w", srcFile, err) + } + defer srcF.Close() + + _, err = io.Copy(newFile, srcF) + if err != nil { + return fmt.Errorf("copying file %s: %w", srcFile, err) + } + return nil +} + +// ref could be anything. Check if hash, tag, or branch in that order +func checkoutCommitOrRef(ctx context.Context, wt *git.Worktree, ref string) error { + var refName plumbing.ReferenceName + opts := &git.CheckoutOptions{ + Hash: plumbing.NewHash(ref), + } + + err := wt.Checkout(opts) + if err != nil { + refName = plumbing.NewTagReferenceName(ref) + opts = &git.CheckoutOptions{ + Branch: refName, + } + err := wt.Checkout(opts) + if err != nil { + refName = plumbing.NewBranchReferenceName(ref) + opts = &git.CheckoutOptions{ + Branch: refName, + } + err := wt.Checkout(opts) + if err != nil { + return err + } + } + } + pullOpts := &git.PullOptions{ + RemoteName: "origin", + } + + if opts.Hash.IsZero() { + pullOpts.ReferenceName = refName + err = wt.PullContext(ctx, pullOpts) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return fmt.Errorf("pulling latest %s: %w", ref, err) + } + } + + return nil +} diff --git a/pkg/util/git_repository_test.go b/pkg/util/git_repository_test.go new file mode 100644 index 00000000..6e224657 --- /dev/null +++ b/pkg/util/git_repository_test.go @@ -0,0 +1,116 @@ +package util + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/assert" +) + +func TestCloneRemoteRepoToDir(t *testing.T) { + spec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: "examples/basic", + Url: "https://github.com/cnoe-io/idpbuilder", + Ref: "v0.3.0", + } + dir, _ := os.MkdirTemp("", "TestCopyToDir") + defer os.RemoveAll(dir) + // new clone + _, _, err := CloneRemoteRepoToDir(context.Background(), spec, 0, false, dir, "") + assert.Nil(t, err) + testDir, _ := os.MkdirTemp("", "TestCopyToDir") + defer os.RemoveAll(testDir) + + repo, err := git.PlainClone(testDir, false, &git.CloneOptions{URL: dir}) + assert.Nil(t, err) + ref, err := repo.Head() + assert.Nil(t, err) + assert.Equal(t, "52783df3a8942cc882ebeb6168f80e1876a2f129", ref.Hash().String()) + + // existing + spec.Ref = "v0.4.0" + testDir2, _ := os.MkdirTemp("", "TestCopyToDir") + defer os.RemoveAll(testDir2) + + _, _, err = CloneRemoteRepoToDir(context.Background(), spec, 0, false, dir, "") + repo, err = git.PlainClone(testDir2, false, &git.CloneOptions{URL: dir}) + assert.Nil(t, err) + ref, err = repo.Head() + assert.Nil(t, err) + assert.Equal(t, "11eccd57fde9f4ef6de8bfa1fc11d168a4d30fe1", ref.Hash().String()) + + assert.Nil(t, err) +} + +func TestCopyTreeToTree(t *testing.T) { + spec := v1alpha1.RemoteRepositorySpec{ + CloneSubmodules: false, + Path: "examples/basic", + Url: "https://github.com/cnoe-io/idpbuilder", + Ref: "", + } + + dst := memfs.New() + src, _, err := CloneRemoteRepoToMemory(context.Background(), spec, 1, false) + assert.Nil(t, err) + + err = CopyTreeToTree(src, dst, spec.Path, ".") + assert.Nil(t, err) + testCopiedFiles(t, src, dst, spec.Path, ".") +} + +func testCopiedFiles(t *testing.T, src, dst billy.Filesystem, srcStartPath, dstStartPath string) { + files, err := src.ReadDir(srcStartPath) + assert.Nil(t, err) + + for i := range files { + file := files[i] + if file.Mode().IsRegular() { + srcB, err := ReadWorktreeFile(src, filepath.Join(srcStartPath, file.Name())) + assert.Nil(t, err) + + dstB, err := ReadWorktreeFile(dst, filepath.Join(dstStartPath, file.Name())) + assert.Nil(t, err) + assert.Equal(t, srcB, dstB) + } + if file.IsDir() { + testCopiedFiles(t, src, dst, filepath.Join(srcStartPath, file.Name()), filepath.Join(dstStartPath, file.Name())) + } + } +} + +func TestGetWorktreeYamlFiles(t *testing.T) { + filepath.Join() + cloneOptions := &git.CloneOptions{ + URL: "https://github.com/cnoe-io/idpbuilder", + Depth: 1, + ShallowSubmodules: true, + } + + wt := memfs.New() + _, err := git.CloneContext(context.Background(), memory.NewStorage(), wt, cloneOptions) + if err != nil { + t.Fatalf(err.Error()) + } + + paths, err := GetWorktreeYamlFiles("./pkg", wt, true) + + assert.Equal(t, nil, err) + assert.NotEqual(t, 0, len(paths)) + for _, s := range paths { + assert.Equal(t, true, strings.HasSuffix(s, "yaml") || strings.HasSuffix(s, "yml")) + } + + paths, err = GetWorktreeYamlFiles("./pkg", wt, false) + assert.Equal(t, nil, err) + assert.Equal(t, 0, len(paths)) +} diff --git a/pkg/util/url.go b/pkg/util/url.go new file mode 100644 index 00000000..0fb20fb3 --- /dev/null +++ b/pkg/util/url.go @@ -0,0 +1,187 @@ +package util + +import ( + "fmt" + "net/url" + "strconv" + "strings" + "time" +) + +// constants from remote target parameters supported by Kustomize +// https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md +const ( + QueryStringRef = "ref" + QueryStringVersion = "version" + QueryStringTimeout = "timeout" + QueryStringSubmodules = "submodules" + + RepoUrlDelimiter = "//" + SCPDelimiter = ":" + UserDelimiter = "@" + + defaultTimeout = time.Second * 27 + defaultCloneSubmodule = true + + errMsgUrlUnsupported = "url must have // after the repository url. example: https://github.com/kubernetes-sigs/kustomize//examples" + errMsgUrlColon = "first path segment in URL cannot contain colon" +) + +type KustomizeRemote struct { + raw string + + Scheme string + User string + Password string + Host string + Port string + RepoPath string + + FilePath string + + Ref string + Submodules bool + Timeout time.Duration +} + +func (g *KustomizeRemote) CloneUrl() string { + sb := strings.Builder{} + if g.Scheme != "" { + sb.WriteString(fmt.Sprintf("%s://", g.Scheme)) + } + if g.User != "" { + sb.WriteString(g.User) + if g.Password != "" { + sb.WriteString(fmt.Sprintf(":%s", g.Password)) + } + sb.Write([]byte(UserDelimiter)) + } + + sb.WriteString(g.Host) + if g.Port != "" { + sb.WriteString(fmt.Sprintf(":%s", g.Port)) + } + if g.Scheme == "" { + sb.WriteString(":") + } else { + sb.WriteString("/") + } + + sb.WriteString(g.RepoPath) + return sb.String() +} + +func (g *KustomizeRemote) Path() string { + return g.FilePath +} + +func (g *KustomizeRemote) parseQuery() error { + _, query, _ := strings.Cut(g.raw, "?") + values, err := url.ParseQuery(query) + + if err != nil { + return err + } + + // if empty, it means we checkout the default branch + version := values.Get(QueryStringVersion) + ref := values.Get(QueryStringRef) + // ref has higher priority per kustomize doc + if ref != "" { + version = ref + } + + duration := defaultTimeout + timeoutString := values.Get(QueryStringTimeout) + if timeoutString != "" { + timeoutInt, sErr := strconv.Atoi(timeoutString) + if sErr == nil { + duration = time.Duration(timeoutInt) * time.Second + } else { + t, sErr := time.ParseDuration(timeoutString) + if sErr == nil { + duration = t + } + } + } + + cloneSubmodules := defaultCloneSubmodule + submodule := values.Get(QueryStringSubmodules) + if submodule != "" { + v, pErr := strconv.ParseBool(submodule) + if pErr == nil { + cloneSubmodules = v + } + } + + g.Ref = version + g.Submodules = cloneSubmodules + g.Timeout = duration + + return nil +} + +func (g *KustomizeRemote) parse() error { + parsed, err := url.Parse(g.raw) + if err != nil { + if strings.Contains(err.Error(), errMsgUrlColon) { + return g.parseSCPStyle() + } + return err + } + + g.Scheme, g.User, g.Host = parsed.Scheme, parsed.User.Username(), parsed.Host + p, ok := parsed.User.Password() + if ok { + g.Password = p + } + + err = g.parseQuery() + if err != nil { + return fmt.Errorf("parsing query parameters in package url: %s: %w", g.raw, err) + } + + return g.parsePath(parsed.Path) +} + +func (g *KustomizeRemote) parseSCPStyle() error { + // example git@github.com:owner/repo + cIndex := strings.Index(g.raw, SCPDelimiter) + if cIndex == -1 { + return fmt.Errorf("not a valid SCP style URL") + } + + uIndex := strings.Index(g.raw[:cIndex], UserDelimiter) + if uIndex != -1 { + g.User = g.raw[:uIndex] + } + g.Host = g.raw[uIndex+1 : cIndex] + err := g.parseQuery() + if err != nil { + return fmt.Errorf("parsing query parameters in package url: %s: %w", g.raw, err) + } + + pathEnd := len(g.raw) + qIndex := strings.Index(g.raw, "?") + if qIndex != -1 { + pathEnd = qIndex + } + return g.parsePath(g.raw[cIndex+1 : pathEnd]) +} + +func (g *KustomizeRemote) parsePath(path string) error { + // example kubernetes-sigs/kustomize//examples/multibases/dev/ + index := strings.Index(path, RepoUrlDelimiter) + if index == -1 { + return fmt.Errorf(errMsgUrlUnsupported) + } + + g.RepoPath = strings.TrimPrefix(path[:index], "/") + g.FilePath = strings.TrimSuffix(path[index+2:], "/") + return nil +} + +func NewKustomizeRemote(uri string) (*KustomizeRemote, error) { + r := &KustomizeRemote{raw: uri} + return r, r.parse() +} diff --git a/pkg/util/url_test.go b/pkg/util/url_test.go new file mode 100644 index 00000000..d4204be2 --- /dev/null +++ b/pkg/util/url_test.go @@ -0,0 +1,82 @@ +package util + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestURLParse(t *testing.T) { + + type expect struct { + cloneUrl string + path string + ref string + submodule bool + timeout time.Duration + err bool + } + + type testCase struct { + expect expect + input string + } + + cases := []testCase{ + { + input: "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + expect: expect{ + cloneUrl: "https://github.com/kubernetes-sigs/kustomize", + path: "examples/multibases/dev", + ref: "v3.3.1", + submodule: true, + timeout: 120 * time.Second, + }, + }, + { + input: "git@github.com:owner/repo//examples?timeout=120&version=v3.3.1", + expect: expect{ + cloneUrl: "git@github.com:owner/repo", + path: "examples", + ref: "v3.3.1", + submodule: true, + timeout: 120 * time.Second, + }, + }, + { + input: "https:// /(@kubernetes-sigs/kustomize//examples/multibases/dev/?timeout=120&ref=v3.3.1", + expect: expect{ + err: true, + }, + }, + { + input: "https://my.github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?version=v3.3.1&submodules=false&timeout=1s", + expect: expect{ + cloneUrl: "https://my.github.com/kubernetes-sigs/kustomize", + path: "examples/multibases/dev", + ref: "v3.3.1", + submodule: false, + timeout: 1 * time.Second, + }, + }, + } + + for i := range cases { + c := cases[i] + + r, err := NewKustomizeRemote(c.input) + if err != nil { + if !c.expect.err { + assert.Fail(t, err.Error()) + } else { + continue + } + } + assert.Equal(t, c.expect.path, r.Path()) + assert.Equal(t, c.expect.cloneUrl, r.CloneUrl()) + assert.Equal(t, c.expect.timeout, r.Timeout) + assert.Equal(t, c.expect.ref, r.Ref) + assert.Equal(t, c.expect.submodule, r.Submodules) + } +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 32cb85c9..e5ae343f 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -8,7 +8,6 @@ import ( var specialCharMap = make(map[string]struct{}) func TestGeneratePassword(t *testing.T) { - for i := range specialChars { specialCharMap[string(specialChars[i])] = struct{}{} }