diff --git a/api/v1/gitrepository_types.go b/api/v1/gitrepository_types.go index 4475acba4..5a5cf2a4e 100644 --- a/api/v1/gitrepository_types.go +++ b/api/v1/gitrepository_types.go @@ -78,6 +78,11 @@ type GitRepositorySpec struct { // +optional Verification *GitRepositoryVerification `json:"verify,omitempty"` + // Proxy specifies the proxy configuration to use while contacting + // the Git server. + // +optional + Proxy *GitRepositoryProxy `json:"proxy,omitempty"` + // Ignore overrides the set of excluded patterns in the .sourceignore format // (which is the same as .gitignore). If not provided, a default will be used, // consult the documentation for your version to find out what those are. @@ -175,6 +180,13 @@ type GitRepositoryVerification struct { SecretRef meta.LocalObjectReference `json:"secretRef"` } +type GitRepositoryProxy struct { + // SecretRef specifies the Secret containing the proxy address and optionally + // the proxy username and password. + // +required + SecretRef meta.LocalObjectReference `json:"secretRef"` +} + // GitRepositoryStatus records the observed state of a Git repository. type GitRepositoryStatus struct { // ObservedGeneration is the last observed generation of the GitRepository diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0b0fde694..78b144514 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -130,6 +130,22 @@ func (in *GitRepositoryList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitRepositoryProxy) DeepCopyInto(out *GitRepositoryProxy) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepositoryProxy. +func (in *GitRepositoryProxy) DeepCopy() *GitRepositoryProxy { + if in == nil { + return nil + } + out := new(GitRepositoryProxy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitRepositoryRef) DeepCopyInto(out *GitRepositoryRef) { *out = *in @@ -169,6 +185,11 @@ func (in *GitRepositorySpec) DeepCopyInto(out *GitRepositorySpec) { *out = new(GitRepositoryVerification) **out = **in } + if in.Proxy != nil { + in, out := &in.Proxy, &out.Proxy + *out = new(GitRepositoryProxy) + **out = **in + } if in.Ignore != nil { in, out := &in.Ignore, &out.Ignore *out = new(string) diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 3097292ca..1aa011c1d 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -90,6 +90,23 @@ spec: description: Interval at which to check the GitRepository for updates. pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ type: string + proxy: + description: Proxy specifies the proxy configuration to use while + contacting the Git server. + properties: + secretRef: + description: SecretRef specifies the Secret containing the proxy + address and optionally the proxy username and password. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + required: + - secretRef + type: object recurseSubmodules: description: RecurseSubmodules enables the initialization of all submodules within the GitRepository as cloned from the URL, using their default diff --git a/docs/spec/v1/gitrepositories.md b/docs/spec/v1/gitrepositories.md index 5a634b7fe..c9ff6c0ae 100644 --- a/docs/spec/v1/gitrepositories.md +++ b/docs/spec/v1/gitrepositories.md @@ -451,10 +451,53 @@ controller with the argument `--feature-gates=OptimizedGitClones=false`. NB: GitRepository objects configured for SemVer or Commit clones are not affected by this functionality. -#### Proxy support +### Proxy -When a proxy is configured in the source-controller Pod through the appropriate -environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. +`.spec.proxy.secretRef.name` is an optional field to specify a name of a Secret +containing the proxy settings that need to be used for all remote Git operations +for the particular GitRepository object. The Secret can contain three keys: + +- `address`: The address of the proxy server. This is a required key. +- `username`: The username to use if the proxy server is protected by basic + authentication. This is an optinal key. +- `password`: The password to use if the proxy server is protected by basic + authentication. This is an optinal key. + +The proxy server must be either HTTP/S or SOCKS5. You can use a SOCKS5 proxy +with a HTTP/S Git repository url. + +Examples: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: http-proxy +type: Opaque +stringData: + address: http://proxy.com + username: mandalorian + password: grogu +``` + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssh-proxy +type: Opaque +stringData: + address: socks5://proxy.com + username: mandalorian + password: grogu +``` + +Proxying can also be configured in the source-controller pod directly by using +the standard environment variables such as `HTTPS_PROXY`, `ALL_PROXY`, etc. + +`.spec.proxy.secretRef.name` takes precedence over all environment variables. ### Recurse submodules diff --git a/go.mod b/go.mod index dae94ccd4..815e908ee 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,9 @@ replace github.com/docker/docker => github.com/docker/docker v23.0.6+incompatibl // is compatible with github.com/google/go-containerregistry v0.15.x. replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.14.1-0.20230409045903-ed5c185df419 +// temp replace directive to enable the use of `WithProxy()` +replace github.com/fluxcd/pkg/git/gogit => github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae + require ( cloud.google.com/go/storage v1.30.1 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 diff --git a/go.sum b/go.sum index a5e17e8dc..86d6c313e 100644 --- a/go.sum +++ b/go.sum @@ -382,8 +382,8 @@ github.com/fluxcd/pkg/apis/meta v1.1.0 h1:vYU1mvUzztnQyTzZOLHQ3wm/tXd7E1QZ2V91zu github.com/fluxcd/pkg/apis/meta v1.1.0/go.mod h1:/QwCotRKL/BT6RSa4O75FlYW14fU8eRfKnoagzbkmL4= github.com/fluxcd/pkg/git v0.12.2 h1:96xH3hy3WfwiD0DioyJZcGapYT3lmPc2s7jU5UM8buw= github.com/fluxcd/pkg/git v0.12.2/go.mod h1:9TG4fEfGCF1XHLt9Xs7X2YOmkmWOiwfjH9tdGIQs8/8= -github.com/fluxcd/pkg/git/gogit v0.11.1 h1:17UbHEPQovLOhlrsPaDoJa3J7jX0I7G92TWXeEDf2eU= -github.com/fluxcd/pkg/git/gogit v0.11.1/go.mod h1:Hh358WYfwmvGf6Aaj1wjGZMN2AWlAcXRR6aubMQYq8M= +github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae h1:JCU+9lTKHPIE7bS5Q9DUoGNa2QuGNhIch/3rt5v8Pso= +github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae/go.mod h1:Kn+GfYfZBBIaXmQj39cQvrDxT/6y8leQxXZ5/B+YYTQ= github.com/fluxcd/pkg/gittestserver v0.8.4 h1:rA/QUZnfH77ZZG+5xfMqjgEHJdzeeE6Nn1o8cops/bU= github.com/fluxcd/pkg/gittestserver v0.8.4/go.mod h1:i3Vng3Stl5zOuGhN4+RuP2NWf5snJCeGUKA7pzAvcHU= github.com/fluxcd/pkg/helmtestserver v0.13.0 h1:bRzOO955nDKWKJZvDORfmDvRdb/558BX4ffgx1vT4LI= diff --git a/internal/controller/gitrepository_controller.go b/internal/controller/gitrepository_controller.go index 622b540c3..225d0e618 100644 --- a/internal/controller/gitrepository_controller.go +++ b/internal/controller/gitrepository_controller.go @@ -28,6 +28,7 @@ import ( securejoin "github.com/cyphar/filepath-securejoin" "github.com/fluxcd/pkg/runtime/logger" + "github.com/go-git/go-git/v5/plumbing/transport" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -473,24 +474,50 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch conditions.Delete(obj, sourcev1.SourceVerifiedCondition) } + var proxyOpts *transport.ProxyOptions + if obj.Spec.Proxy != nil { + proxySecretName := obj.Spec.Proxy.SecretRef.Name + proxyData, err := r.getSecretData(ctx, proxySecretName, obj.GetNamespace()) + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to get secret '%s/%s': %w", proxySecretName, obj.GetNamespace(), err), + sourcev1.AuthenticationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + // Return error as the world as observed may change + return sreconcile.ResultEmpty, e + } + address, ok := proxyData["address"] + if !ok { + e := serror.NewGeneric( + fmt.Errorf("invalid proxy secret '%s/%s': key 'address' is missing", proxySecretName, obj.GetNamespace()), + sourcev1.AuthenticationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + // Return error as the world as observed may change + return sreconcile.ResultEmpty, e + } + proxyOpts = &transport.ProxyOptions{ + URL: string(address), + } + proxyOpts.Username = string(proxyData["username"]) + proxyOpts.Password = string(proxyData["password"]) + } + var authData map[string][]byte if obj.Spec.SecretRef != nil { // Attempt to retrieve secret - name := types.NamespacedName{ - Namespace: obj.GetNamespace(), - Name: obj.Spec.SecretRef.Name, - } - var secret corev1.Secret - if err := r.Client.Get(ctx, name, &secret); err != nil { + var err error + authData, err = r.getSecretData(ctx, obj.Spec.SecretRef.Name, obj.GetNamespace()) + if err != nil { e := serror.NewGeneric( - fmt.Errorf("failed to get secret '%s': %w", name.String(), err), + fmt.Errorf("failed to get secret '%s/%s': %w", obj.Spec.SecretRef.Name, obj.GetNamespace(), err), sourcev1.AuthenticationFailedReason, ) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) // Return error as the world as observed may change return sreconcile.ResultEmpty, e } - authData = secret.Data } u, err := url.Parse(obj.Spec.URL) @@ -541,7 +568,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch optimizedClone = true } - c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone) + c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, optimizedClone) if err != nil { return sreconcile.ResultEmpty, err } @@ -583,7 +610,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch // If we can't skip the reconciliation, checkout again without any // optimization. - c, err := r.gitCheckout(ctx, obj, authOpts, dir, false) + c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, false) if err != nil { return sreconcile.ResultEmpty, err } @@ -611,6 +638,18 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch return sreconcile.ResultSuccess, nil } +func (r *GitRepositoryReconciler) getSecretData(ctx context.Context, name, namespace string) (map[string][]byte, error) { + key := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + var secret corev1.Secret + if err := r.Client.Get(ctx, key, &secret); err != nil { + return nil, err + } + return secret.Data, nil +} + // reconcileArtifact archives a new Artifact to the Storage, if the current // (Status) data on the object does not match the given. // @@ -782,8 +821,8 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, sp *patc // gitCheckout builds checkout options with the given configurations and // performs a git checkout. func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, - obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string, - optimized bool) (*git.Commit, error) { + obj *sourcev1.GitRepository, authOpts *git.AuthOptions, proxyOpts *transport.ProxyOptions, + dir string, optimized bool) (*git.Commit, error) { // Configure checkout strategy. cloneOpts := repository.CloneConfig{ RecurseSubmodules: obj.Spec.RecurseSubmodules, @@ -813,6 +852,9 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, if authOpts.Transport == git.HTTP { clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP()) } + if proxyOpts != nil { + clientOpts = append(clientOpts, gogit.WithProxy(*proxyOpts)) + } gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...) if err != nil {