From fadec7b519abba4b5a60dda2bbb62b56ded19dbe Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Thu, 7 Jan 2021 16:43:41 -0800 Subject: [PATCH] Add support for custom config mounting This adds support to mount a custom ConfigMap containing files into /etc/config which can then be used for tuning the buildkitd configuration. For example, if your registry uses self-signed or non-standard certs, you can pass those in and configure buildkitd to trust them. This also fixes a subtle glitch where we should be passing in the root path as a flag instead of config file setting, thus making it easier for users to BYO config file and not run into problems. --- README.md | 26 +++++++++++++++ pkg/cmd/create.go | 3 ++ pkg/driver/kubernetes/driver.go | 1 - pkg/driver/kubernetes/factory.go | 2 ++ pkg/driver/kubernetes/manifest/manifest.go | 37 ++++++++++++++++++++-- 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 685f14fc..e441c77d 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,32 @@ You can then build using the registry cache with the command: kubectl build -t myimage --cache-to=type=registry,ref=registry:5000/cache --cache-from=type=registry,ref=registry:5000/cache . ``` +## Custom Certs for Registries + +If you happen to run a container image registry with non-standard certs (self signed, or signed by a private CA) +you can configure buildkitd to trust those certificates. + +First load your certs into a ConfigMap: +```sh +kubectl create configmap custom-root-certs --from-file=root_ca1.pem --from-file=root_ca2.pem +``` + +Then create a `buildkitd.toml` file for your registries where the certs above will be mounted into `/etc/config` +``` +debug = false +[worker.containerd] + namespace = "k8s.io" +[registry."myregistry1.acme.local"] + ca=["/etc/config/root_ca1.pem"] +[registry."myregistry2.acme.local:5001"] + ca=["/etc/config/root_ca2.pem"] +``` + +You can then create a customized builder with +``` +kubectl buildkit create --custom-config=custom-root-certs --config ./buildkitd.toml +``` + ## Contributing Check out our [contributing](./CONTRIBUTING.md) for guidance on how to help contribute to the project. diff --git a/pkg/cmd/create.go b/pkg/cmd/create.go index 12d34286..20123aa0 100644 --- a/pkg/cmd/create.go +++ b/pkg/cmd/create.go @@ -41,6 +41,7 @@ type createOptions struct { flags string configFile string progress string + customConfig string } func runCreate(streams genericclioptions.IOStreams, in createOptions, rootOpts *rootOptions) error { @@ -76,6 +77,7 @@ func runCreate(streams genericclioptions.IOStreams, in createOptions, rootOpts * "containerd-sock": in.containerdSock, "docker-sock": in.dockerSock, "runtime": in.runtime, + "custom-config": in.customConfig, } d, err := driver.GetDriver(ctx, in.name, driverFactory, rootOpts.KubeClientConfig, flags, in.configFile, driverOpts, "" /*contextPathHash*/) @@ -137,6 +139,7 @@ Driver Specific Usage: flags.BoolVar(&options.rootless, "rootless", false, "Run in rootless mode") flags.StringVar(&options.loadbalance, "loadbalance", "random", "Load balancing strategy [random, sticky]") flags.StringVar(&options.worker, "worker", "auto", "Worker backend [auto, runc, containerd]") + flags.StringVar(&options.customConfig, "custom-config", "", "Name of a ConfigMap containing custom files (e.g., certs), mounted in /etc/config/ - use 'kubectl create configmap ... --from-file=...'") return cmd } diff --git a/pkg/driver/kubernetes/driver.go b/pkg/driver/kubernetes/driver.go index cd5f7831..3971261c 100644 --- a/pkg/driver/kubernetes/driver.go +++ b/pkg/driver/kubernetes/driver.go @@ -51,7 +51,6 @@ const ( // TODO - consider adding other default values here to aid users in fine-tuning by editing the configmap post deployment DefaultConfigFileTemplate = `# Default buildkitd configuration. Use --config to override during create debug = false -root = "/var/lib/buildkit/{{ .Name }}" [worker.containerd] namespace = "{{ .ContainerdNamespace }}" ` diff --git a/pkg/driver/kubernetes/factory.go b/pkg/driver/kubernetes/factory.go index 71251989..b244fa16 100644 --- a/pkg/driver/kubernetes/factory.go +++ b/pkg/driver/kubernetes/factory.go @@ -176,6 +176,8 @@ func (d *Driver) initDriverFromConfig() error { return errors.Errorf("invalid runtime %q", v) } deploymentOpt.ContainerRuntime = v + case "custom-config": + deploymentOpt.CustomConfig = v default: return errors.Errorf("invalid driver option %s for driver %s", k, DriverName) } diff --git a/pkg/driver/kubernetes/manifest/manifest.go b/pkg/driver/kubernetes/manifest/manifest.go index a160f732..e6897b78 100644 --- a/pkg/driver/kubernetes/manifest/manifest.go +++ b/pkg/driver/kubernetes/manifest/manifest.go @@ -23,6 +23,7 @@ type DeploymentOpt struct { ContainerdSockHostPath string DockerSockHostPath string ContainerRuntime string + CustomConfig string } const ( @@ -127,6 +128,11 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) { return nil, err } } + if opt.CustomConfig != "" { + if err := addCustomConfigMount(d, opt); err != nil { + return nil, err + } + } return d, nil } @@ -146,10 +152,12 @@ func toRootless(d *appsv1.Deployment) error { func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error { labels := labels(opt) + buildkitRoot := "/var/lib/buildkit/" + opt.Name d.Spec.Template.Spec.Containers[0].Args = append( d.Spec.Template.Spec.Containers[0].Args, "--oci-worker=false", "--containerd-worker=true", + "--root", buildkitRoot, ) mountPropagationBidirectional := corev1.MountPropagationBidirectional d.Spec.Template.Spec.Containers[0].VolumeMounts = append( @@ -160,7 +168,7 @@ func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error { }, corev1.VolumeMount{ Name: "var-lib-buildkit", - MountPath: "/var/lib/buildkit/" + opt.Name, + MountPath: buildkitRoot, MountPropagation: &mountPropagationBidirectional, }, corev1.VolumeMount{ @@ -204,7 +212,7 @@ func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error { Name: "var-lib-buildkit", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/buildkit/" + opt.Name, + Path: buildkitRoot, Type: &hostPathDirectoryOrCreate, }}, }, @@ -305,6 +313,31 @@ func addDockerSockMount(d *appsv1.Deployment, opt *DeploymentOpt) error { return nil } +func addCustomConfigMount(d *appsv1.Deployment, opt *DeploymentOpt) error { + d.Spec.Template.Spec.Containers[0].VolumeMounts = append( + d.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "custom-config", + MountPath: "/etc/config/", + }, + ) + d.Spec.Template.Spec.Volumes = append( + d.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "custom-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: opt.CustomConfig, + }, + }, + }, + }, + ) + + return nil +} + func NewConfigMap(opt *DeploymentOpt, contents []byte) *corev1.ConfigMap { return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{