Support /etc/containerd/certs.d/<HOST:PORT>/hosts.toml

cmd/nerdctl/compose.go

@@ -103,6 +103,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	if err != nil {
 		return nil, err
 	}
+	hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir")
+	if err != nil {
+		return nil, err
+	}
+
 	o := composer.Options{
 		ProjectOptions: composecli.ProjectOptions{
 			WorkingDir:  projectDirectory,
@@ -186,7 +191,7 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 				pullMode, ocispecPlatforms, nil, quiet)
 		} else {
 			_, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), cmd.ErrOrStderr(), snapshotter, imageName,
-				pullMode, insecure, ocispecPlatforms, nil, quiet)
+				pullMode, insecure, hostsDirs, ocispecPlatforms, nil, quiet)
 		}
 		return imgErr
 	}

cmd/nerdctl/compose_up_linux_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/docker/go-connections/nat"
 
 	"github.com/containerd/nerdctl/pkg/testutil"
+	"github.com/containerd/nerdctl/pkg/testutil/nettestutil"
 
 	"gotest.tools/v3/assert"
 )
@@ -81,7 +82,7 @@ func testComposeUp(t *testing.T, base *testutil.Base, dockerComposeYAML string)
 	base.Cmd("network", "inspect", fmt.Sprintf("%s_default", projectName)).AssertOK()
 
 	checkWordpress := func() error {
-		resp, err := httpGet("http://127.0.0.1:8080", 10)
+		resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 10, false)
 		if err != nil {
 			return err
 		}
@@ -144,7 +145,7 @@ COPY index.html /usr/share/nginx/html/index.html
 	base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d", "--build").AssertOK()
 	defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()
 
-	resp, err := httpGet("http://127.0.0.1:8080", 50)
+	resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 50, false)
 	assert.NilError(t, err)
 	respBody, err := io.ReadAll(resp.Body)
 	assert.NilError(t, err)

cmd/nerdctl/image_encrypt_linux_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/containerd/nerdctl/pkg/testutil"
+	"github.com/containerd/nerdctl/pkg/testutil/testregistry"
 	"gotest.tools/v3/assert"
 )
 
@@ -73,10 +74,10 @@ func TestImageEncryptJWE(t *testing.T) {
 	defer keyPair.cleanup()
 	base := testutil.NewBase(t)
 	tID := testutil.Identifier(t)
-	reg := newTestRegistry(base)
-	defer reg.cleanup()
+	reg := testregistry.NewPlainHTTP(base)
+	defer reg.Cleanup()
 	base.Cmd("pull", testutil.CommonImage).AssertOK()
-	encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.listenPort, tID)
+	encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.ListenPort, tID)
 	defer base.Cmd("rmi", encryptImageRef).Run()
 	base.Cmd("image", "encrypt", "--recipient=jwe:"+keyPair.pub, testutil.CommonImage, encryptImageRef).AssertOK()
 	base.Cmd("image", "inspect", "--mode=native", "--format={{len .Index.Manifests}}", encryptImageRef).AssertOutExactly("1\n")

cmd/nerdctl/ipfs_compose_linux_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/containerd/nerdctl/pkg/testutil"
+	"github.com/containerd/nerdctl/pkg/testutil/nettestutil"
 	"gotest.tools/v3/assert"
 )
 
@@ -129,7 +130,7 @@ COPY index.html /usr/share/nginx/html/index.html
 	defer base.Cmd("ipfs", "registry", "down").AssertOK()
 	defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()
 
-	resp, err := httpGet("http://127.0.0.1:8080", 50)
+	resp, err := nettestutil.HTTPGet("http://127.0.0.1:8080", 50, false)
 	assert.NilError(t, err)
 	respBody, err := io.ReadAll(resp.Body)
 	assert.NilError(t, err)

cmd/nerdctl/login.go

@@ -21,16 +21,20 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"runtime"
+	"net/http"
+	"net/url"
 	"strings"
 
-	"github.com/containerd/nerdctl/pkg/version"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/remotes/docker"
+	dockerconfig "github.com/containerd/containerd/remotes/docker/config"
+	"github.com/containerd/nerdctl/pkg/imgutil/dockerconfigresolver"
 	dockercliconfig "github.com/docker/cli/cli/config"
 	clitypes "github.com/docker/cli/cli/config/types"
 	dockercliconfigtypes "github.com/docker/cli/cli/config/types"
 	"github.com/docker/docker/api/types"
-	registrytypes "github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/registry"
+	"golang.org/x/net/context/ctxhttp"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -76,7 +80,7 @@ func loginAction(cmd *cobra.Command, args []string) error {
 		serverAddress = options.serverAddress
 	}
 
-	var response registrytypes.AuthenticateOKBody
+	var responseIdentityToken string
 	ctx := cmd.Context()
 	isDefaultRegistry := serverAddress == registry.IndexServer
 	authConfig, err := GetDefaultAuthConfig(options.username == "" && options.password == "", serverAddress, isDefaultRegistry)
@@ -85,7 +89,7 @@ func loginAction(cmd *cobra.Command, args []string) error {
 	}
 	if err == nil && authConfig.Username != "" && authConfig.Password != "" {
 		//login With StoreCreds
-		response, err = loginClientSide(ctx, cmd, *authConfig)
+		responseIdentityToken, err = loginClientSide(ctx, cmd, *authConfig)
 	}
 
 	if err != nil || authConfig.Username == "" || authConfig.Password == "" {
@@ -94,16 +98,16 @@ func loginAction(cmd *cobra.Command, args []string) error {
 			return err
 		}
 
-		response, err = loginClientSide(ctx, cmd, *authConfig)
+		responseIdentityToken, err = loginClientSide(ctx, cmd, *authConfig)
 		if err != nil {
 			return err
 		}
 
 	}
 
-	if response.IdentityToken != "" {
+	if responseIdentityToken != "" {
 		authConfig.Password = ""
-		authConfig.IdentityToken = response.IdentityToken
+		authConfig.IdentityToken = responseIdentityToken
 	}
 
 	dockerConfigFile, err := dockercliconfig.Load("")
@@ -115,9 +119,10 @@ func loginAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("error saving credentials: %w", err)
 	}
 
-	if response.Status != "" {
-		fmt.Fprintln(cmd.OutOrStdout(), response.Status)
+	if _, err = loginClientSide(ctx, cmd, *authConfig); err != nil {
+		return err
 	}
+	fmt.Fprintln(cmd.OutOrStdout(), "Login Succeeded")
 
 	return nil
 }
@@ -169,33 +174,119 @@ func GetDefaultAuthConfig(checkCredStore bool, serverAddress string, isDefaultRe
 	return &res, nil
 }
 
-// Code from github.com/cli/cli/command/registry/login.go
-func loginClientSide(ctx context.Context, cmd *cobra.Command, auth types.AuthConfig) (registrytypes.AuthenticateOKBody, error) {
+func loginClientSide(ctx context.Context, cmd *cobra.Command, auth types.AuthConfig) (string, error) {
+	host := auth.ServerAddress
+	if strings.Contains(host, "://") {
+		u, err := url.Parse(host)
+		if err != nil {
+			return "", err
+		}
+		host = u.Host
+	}
 
-	var insecureRegistries []string
-	insecureRegistry, err := cmd.Flags().GetBool("insecure-registry")
+	var dOpts []dockerconfigresolver.Opt
+	insecure, err := cmd.Flags().GetBool("insecure-registry")
+	if err != nil {
+		return "", err
+	}
+	if insecure {
+		logrus.Warnf("skipping verifying HTTPS certs for %q", host)
+		dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true))
+	}
+	hostsDirs, err := cmd.Flags().GetStringSlice("hosts-dir")
 	if err != nil {
-		return registrytypes.AuthenticateOKBody{}, err
+		return "", err
 	}
-	if insecureRegistry {
-		insecureRegistries = append(insecureRegistries, auth.ServerAddress)
+	dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(hostsDirs))
+
+	authCreds := func(acArg string) (string, string, error) {
+		if acArg == host {
+			if auth.RegistryToken != "" {
+				// Even containerd/CRI does not support RegistryToken as of v1.4.3,
+				// so, nobody is actually using RegistryToken?
+				logrus.Warnf("RegistryToken (for %q) is not supported yet (FIXME)", host)
+			}
+			if auth.IdentityToken != "" {
+				return "", auth.IdentityToken, nil
+			}
+			return auth.Username, auth.Password, nil
+		}
+		return "", "", fmt.Errorf("expected acArg to be %q, got %q", host, acArg)
 	}
-	svc, err := registry.NewService(registry.ServiceOptions{
-		InsecureRegistries: insecureRegistries,
-	})
 
+	dOpts = append(dOpts, dockerconfigresolver.WithAuthCreds(authCreds))
+	ho, err := dockerconfigresolver.NewHostOptions(ctx, host, dOpts...)
+	if err != nil {
+		return "", err
+	}
+	fetchedRefreshTokens := make(map[string]string) // key: req.URL.Host
+	// onFetchRefreshToken is called when tryLoginWithRegHost calls rh.Authorizer.Authorize()
+	onFetchRefreshToken := func(ctx context.Context, s string, req *http.Request) {
+		fetchedRefreshTokens[req.URL.Host] = s
+	}
+	ho.AuthorizerOpts = append(ho.AuthorizerOpts, docker.WithFetchRefreshToken(onFetchRefreshToken))
+	regHosts, err := dockerconfig.ConfigureHosts(ctx, *ho)(host)
 	if err != nil {
-		return registrytypes.AuthenticateOKBody{}, err
+		return "", err
+	}
+	logrus.Debugf("len(regHosts)=%d", len(regHosts))
+	if len(regHosts) == 0 {
+		return "", fmt.Errorf("got empty []docker.RegistryHost for %q", host)
+	}
+	for i, rh := range regHosts {
+		err = tryLoginWithRegHost(ctx, rh)
+		identityToken := fetchedRefreshTokens[rh.Host] // can be empty
+		if err == nil {
+			return identityToken, nil
+		}
+		logrus.WithError(err).WithField("i", i).Error("failed to call tryLoginWithRegHost")
 	}
+	return "", err
+}
 
-	userAgent := fmt.Sprintf("Docker-Client/nerdctl-%s (%s)", version.Version, runtime.GOOS)
+func tryLoginWithRegHost(ctx context.Context, rh docker.RegistryHost) error {
+	if rh.Authorizer == nil {
+		return errors.New("got nil Authorizer")
+	}
+	u := url.URL{
+		Scheme: rh.Scheme,
+		Host:   rh.Host,
+		Path:   rh.Path,
+	}
+	ctx = docker.WithScope(ctx, "")
+	var ress []*http.Response
+	for i := 0; i < 10; i++ {
+		req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+		if err != nil {
+			return err
+		}
+		for k, v := range rh.Header.Clone() {
+			for _, vv := range v {
+				req.Header.Add(k, vv)
+			}
+		}
+		if err := rh.Authorizer.Authorize(ctx, req); err != nil {
+			return fmt.Errorf("failed to call rh.Authorizer.Authorize: %w", err)
+		}
+		res, err := ctxhttp.Do(ctx, rh.Client, req)
+		if err != nil {
+			return fmt.Errorf("failed to call rh.Client.Do: %w", err)
+		}
+		ress = append(ress, res)
+		if res.StatusCode == 401 {
+			if err := rh.Authorizer.AddResponses(ctx, ress); err != nil && !errdefs.IsNotImplemented(err) {
+				return fmt.Errorf("failed to call rh.Authorizer.AddResponses: %w", err)
+			}
+			continue
+		}
+		if res.StatusCode/100 != 2 {
+			return fmt.Errorf("unexpected status code %d", res.StatusCode)
+		}
 
-	status, token, err := svc.Auth(ctx, &auth, userAgent)
+		return nil
+	}
 
-	return registrytypes.AuthenticateOKBody{
-		Status:        status,
-		IdentityToken: token,
-	}, err
+	return errors.New("too many 401 (probably)")
 }
 
 func ConfigureAuthentification(authConfig *types.AuthConfig, options *loginOptions) error {

cmd/nerdctl/login_linux_test.go

@@ -0,0 +1,43 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/containerd/nerdctl/pkg/testutil"
+	"github.com/containerd/nerdctl/pkg/testutil/testregistry"
+)
+
+func TestLogin(t *testing.T) {
+	// Skip docker, because Docker doesn't have `--hosts-dir` option, and we don't want to contaminate the global /etc/docker/certs.d during this test
+	testutil.DockerIncompatible(t)
+
+	base := testutil.NewBase(t)
+	reg := testregistry.NewHTTPS(base, "admin", "validTestPassword")
+	defer reg.Cleanup()
+
+	regHost := net.JoinHostPort(reg.IP.String(), strconv.Itoa(reg.ListenPort))
+
+	t.Logf("Good password")
+	base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "validTestPassword", regHost).AssertOK()
+
+	t.Logf("Bad password")
+	base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "invalidTestPassword", regHost).AssertFail()
+}

cmd/nerdctl/main.go

@@ -87,16 +87,17 @@ func xmain() error {
 // Config corresponds to nerdctl.toml .
 // See docs/config.md .
 type Config struct {
-	Debug            bool   `toml:"debug"`
-	DebugFull        bool   `toml:"debug_full"`
-	Address          string `toml:"address"`
-	Namespace        string `toml:"namespace"`
-	Snapshotter      string `toml:"snapshotter"`
-	CNIPath          string `toml:"cni_path"`
-	CNINetConfPath   string `toml:"cni_netconfpath"`
-	DataRoot         string `toml:"data_root"`
-	CgroupManager    string `toml:"cgroup_manager"`
-	InsecureRegistry bool   `toml:"insecure_registry"`
+	Debug            bool     `toml:"debug"`
+	DebugFull        bool     `toml:"debug_full"`
+	Address          string   `toml:"address"`
+	Namespace        string   `toml:"namespace"`
+	Snapshotter      string   `toml:"snapshotter"`
+	CNIPath          string   `toml:"cni_path"`
+	CNINetConfPath   string   `toml:"cni_netconfpath"`
+	DataRoot         string   `toml:"data_root"`
+	CgroupManager    string   `toml:"cgroup_manager"`
+	InsecureRegistry bool     `toml:"insecure_registry"`
+	HostsDir         []string `toml:"hosts_dir"`
 }
 
 // NewConfig creates a default Config object statically,
@@ -113,6 +114,7 @@ func NewConfig() *Config {
 		DataRoot:         ncdefaults.DataRoot(),
 		CgroupManager:    ncdefaults.CgroupManager(),
 		InsecureRegistry: false,
+		HostsDir:         ncdefaults.HostsDirs(),
 	}
 }
 
@@ -148,6 +150,8 @@ func initRootCmdFlags(rootCmd *cobra.Command, tomlPath string) error {
 	rootCmd.PersistentFlags().String("cgroup-manager", cfg.CgroupManager, `Cgroup manager to use ("cgroupfs"|"systemd")`)
 	rootCmd.RegisterFlagCompletionFunc("cgroup-manager", shellCompleteCgroupManagerNames)
 	rootCmd.PersistentFlags().Bool("insecure-registry", cfg.InsecureRegistry, "skips verifying HTTPS certs, and allows falling back to plain HTTP")
+	// hosts-dir is defined as StringSlice, not StringArray, to allow specifying "--hosts-dir=/etc/containerd/certs.d,/etc/docker/certs.d"
+	rootCmd.PersistentFlags().StringSlice("hosts-dir", cfg.HostsDir, "A directory that contains <HOST:PORT>/hosts.toml (containerd style) or <HOST:PORT>/{ca.cert, cert.pem, key.pem} (docker style)")
 	return nil
 }