From 4d3ef253a0d4cf45941fbcfc6acaacb991b9665f Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 25 Jan 2018 16:54:59 +0000 Subject: [PATCH 1/2] Support basic auth and HTTP for image registries The image registry code assumed - all image registries use HTTPS - all image registries use token authentication ..but if you run your own registry in your cluster, neither of these things is likely to be true. To support basic authentication, all we need to do is add a handler in when constructing the registry client; it will get used if the authentication challenge indicates so. Supporting HTTP is (oddly) a bit trickier, since there's no indication from an image name (which is all we have) whether the registry uses HTTP or HTTPS. Registries will tend to redirect any HTTP requests to HTTPS, so we _could_ try HTTP first and follow any redirection. However, if a registry supported both HTTP and HTTPS, and didn't redirect, we'd end up sending credentials over an insecure connection unnecessarily. Instead of that, make it possible to tell the daemon which registries to use HTTP for, with the (multiply-valued) argument `--registry-insecure-host`. --- cmd/fluxd/main.go | 8 ++-- registry/client.go | 13 ++++--- registry/client_factory.go | 77 ++++++++++++++++++++++++-------------- 3 files changed, 61 insertions(+), 37 deletions(-) diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go index 4229f5607..feb04cbd0 100644 --- a/cmd/fluxd/main.go +++ b/cmd/fluxd/main.go @@ -94,6 +94,7 @@ func main() { registryRPS = fs.Int("registry-rps", 200, "maximum registry requests per second per host") registryBurst = fs.Int("registry-burst", defaultRemoteConnections, "maximum number of warmer connections to remote and memcache") registryTrace = fs.Bool("registry-trace", false, "output trace of image registry requests to log") + registryInsecure = fs.StringSlice("registry-insecure-host", []string{}, "use HTTP for this image registry domain (e.g., registry.cluster.local), instead of HTTPS") // k8s-secret backed ssh keyring configuration k8sSecretName = fs.String("k8s-secret-name", "flux-git-deploy", "Name of the k8s secret used to store the private SSH key") @@ -257,9 +258,10 @@ func main() { Burst: *registryBurst, } remoteFactory := ®istry.RemoteClientFactory{ - Logger: registryLogger, - Limiters: registryLimits, - Trace: *registryTrace, + Logger: registryLogger, + Limiters: registryLimits, + Trace: *registryTrace, + InsecureHosts: *registryInsecure, } // Warmer diff --git a/registry/client.go b/registry/client.go index d7a3f8ab5..603bb3c1a 100644 --- a/registry/client.go +++ b/registry/client.go @@ -37,6 +37,7 @@ type ClientFactory interface { type Remote struct { transport http.RoundTripper repo image.CanonicalName + base string } // Adapt to docker distribution `reference.Named`. @@ -44,17 +45,17 @@ type named struct { image.CanonicalName } -// Name returns the name of the repository. These values are used to -// build API URLs, and (it turns out) are _not_ expected to include a -// domain (e.g., quay.io). Hence, the implementation here just returns -// the path. +// Name returns the name of the repository. These values are used by +// the docker distribution client package to build API URLs, and (it +// turns out) are _not_ expected to include a domain (e.g., +// quay.io). Hence, the implementation here just returns the path. func (n named) Name() string { return n.Image } // Return the tags for this repository. func (a *Remote) Tags(ctx context.Context) ([]string, error) { - repository, err := client.NewRepository(named{a.repo}, "https://"+a.repo.Domain, a.transport) + repository, err := client.NewRepository(named{a.repo}, a.base, a.transport) if err != nil { return nil, err } @@ -64,7 +65,7 @@ func (a *Remote) Tags(ctx context.Context) ([]string, error) { // Manifest fetches the metadata for an image reference; currently // assumed to be in the same repo as that provided to `NewRemote(...)` func (a *Remote) Manifest(ctx context.Context, ref string) (image.Info, error) { - repository, err := client.NewRepository(named{a.repo}, "https://"+a.repo.Domain, a.transport) + repository, err := client.NewRepository(named{a.repo}, a.base, a.transport) if err != nil { return image.Info{}, err } diff --git a/registry/client_factory.go b/registry/client_factory.go index 65733ec07..1a93a1942 100644 --- a/registry/client_factory.go +++ b/registry/client_factory.go @@ -15,11 +15,13 @@ import ( ) type RemoteClientFactory struct { - Logger log.Logger - Limiters *middleware.RateLimiters - Trace bool + Logger log.Logger + Limiters *middleware.RateLimiters + Trace bool + InsecureHosts []string + + mu sync.Mutex challengeManager challenge.Manager - mx sync.Mutex } type logging struct { @@ -43,47 +45,66 @@ func (f *RemoteClientFactory) ClientFor(repo image.CanonicalName, creds Credenti tx = &logging{f.Logger, tx} } - f.mx.Lock() + f.mu.Lock() if f.challengeManager == nil { f.challengeManager = challenge.NewSimpleManager() } - f.mx.Unlock() manager := f.challengeManager + f.mu.Unlock() + + scheme := "https" + for _, h := range f.InsecureHosts { + if repo.Domain == h { + scheme = "http" + } + } - pingURL := url.URL{ - Scheme: "https", + registryURL := url.URL{ + Scheme: scheme, Host: repo.Domain, Path: "/v2/", } + // Before we know how to authorise, need to establish which - // authorisation challenges the host will send. - if cs, err := manager.GetChallenges(pingURL); err == nil { - if len(cs) == 0 { - req, err := http.NewRequest("GET", pingURL.String(), nil) - if err != nil { - return nil, err - } - res, err := (&http.Client{ - Transport: tx, - }).Do(req) - if err != nil { - return nil, err - } - if err = manager.AddResponse(res); err != nil { - return nil, err - } + // authorisation challenges the host will send. See if we've been + // here before. + cs, err := manager.GetChallenges(registryURL) + if err != nil { + return nil, err + } + if len(cs) == 0 { + // No prior challenge; try pinging the registry endpoint to + // get a challenge. `http.Client` will follow redirects, so + // even if we thought it was an insecure (HTTP) host, we may + // end up requesting HTTPS. + req, err := http.NewRequest("GET", registryURL.String(), nil) + if err != nil { + return nil, err + } + res, err := (&http.Client{ + Transport: tx, + }).Do(req) + if err != nil { + return nil, err + } + if err = manager.AddResponse(res); err != nil { + return nil, err } + registryURL = *res.Request.URL // <- the URL after any redirection } cred := creds.credsFor(repo.Domain) if f.Trace { - f.Logger.Log("repo", repo.String(), "auth", cred.String()) + f.Logger.Log("repo", repo.String(), "auth", cred.String(), "api", registryURL.String()) } - handler := auth.NewTokenHandler(tx, &store{cred}, repo.Image, "pull") - tx = transport.NewTransport(tx, auth.NewAuthorizer(manager, handler)) + tokenHandler := auth.NewTokenHandler(tx, &store{cred}, repo.Image, "pull") + basicauthHandler := auth.NewBasicHandler(&store{cred}) + tx = transport.NewTransport(tx, auth.NewAuthorizer(manager, tokenHandler, basicauthHandler)) - client := &Remote{transport: tx, repo: repo} + // For the API base we want only the scheme and host. + registryURL.Path = "" + client := &Remote{transport: tx, repo: repo, base: registryURL.String()} return NewInstrumentedClient(client), nil } From ae9d26746eee029a9a094feaed698dc8739a1274 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 25 Jan 2018 17:08:18 +0000 Subject: [PATCH 2/2] Document new arg and limitation on localhost --- site/daemon.md | 1 + site/requirements.md | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/site/daemon.md b/site/daemon.md index 9a2b05a3a..bb2c3cdab 100644 --- a/site/daemon.md +++ b/site/daemon.md @@ -61,6 +61,7 @@ fluxd requires setup and offers customization though a multitude of flags. |--registry-poll-interval| `5 minutes` | period at which to poll registry for new images| |--registry-rps | `200` | maximum registry requests per second per host| |--registry-burst | `125` | maximum number of warmer connections to remote and memcache| +|--registry-insecure-host| [] | registry hosts to use HTTP for (instead of HTTPS) | |**k8s-secret backed ssh keyring configuration** | | | |--k8s-secret-name | `flux-git-deploy` | name of the k8s secret used to store the private SSH key| |--k8s-secret-volume-mount-path | `/etc/fluxd/ssh` | mount location of the k8s secret storing the private SSH key| diff --git a/site/requirements.md b/site/requirements.md index 465551bae..f5b92fb32 100644 --- a/site/requirements.md +++ b/site/requirements.md @@ -29,3 +29,11 @@ It is _not_ a requirement that the files are arranged in any particular way into directories. Flux will look in subdirectories for YAML files recursively, but does not infer any meaning from the directory structure. + +Flux uses the Docker Registry API to collect metadata about the images +running in the cluster. This comes with at least one limitation: + + * Since Flux runs in a container in your cluster, it may not be able + to resolve all hostnames that you or Kubernetes can resolve. In + particular, it won't be able to get image metadata for images in a + private image registry that's made available at `localhost`.