Add support for custom config mounting

This adds support to mount a custom ConfigMap containing files
into /etc/config which can then be used for tuning the buildkitd
configuration.  For example, if your registry uses self-signed or
non-standard certs, you can pass those in and configure buildkitd
to trust them.

This also fixes a subtle glitch where we should be passing in the root
path as a flag instead of config file setting, thus making it easier for
users to BYO config file and not run into problems.

README.md

@@ -122,6 +122,32 @@ You can then build using the registry cache with the command:
 kubectl build -t myimage --cache-to=type=registry,ref=registry:5000/cache --cache-from=type=registry,ref=registry:5000/cache .
 ```
 
+## Custom Certs for Registries
+
+If you happen to run a container image registry with non-standard certs (self signed, or signed by a private CA)
+you can configure buildkitd to trust those certificates.
+
+First load your certs into a ConfigMap:
+```sh
+kubectl create configmap custom-root-certs --from-file=root_ca1.pem --from-file=root_ca2.pem
+```
+
+Then create a `buildkitd.toml` file for your registries where the certs above will be mounted into `/etc/config`
+```
+debug = false
+[worker.containerd]
+  namespace = "k8s.io"
+[registry."myregistry1.acme.local"]
+    ca=["/etc/config/root_ca1.pem"]
+[registry."myregistry2.acme.local:5001"]
+    ca=["/etc/config/root_ca2.pem"]
+```
+
+You can then create a customized builder with
+```
+kubectl buildkit create --custom-config=custom-root-certs --config ./buildkitd.toml
+```
+
 ## Contributing
 
 Check out our [contributing](./CONTRIBUTING.md) for guidance on how to help contribute to the project.

pkg/cmd/create.go

@@ -41,6 +41,7 @@ type createOptions struct {
 	flags               string
 	configFile          string
 	progress            string
+	customConfig        string
 }
 
 func runCreate(streams genericclioptions.IOStreams, in createOptions, rootOpts *rootOptions) error {
@@ -76,6 +77,7 @@ func runCreate(streams genericclioptions.IOStreams, in createOptions, rootOpts *
 		"containerd-sock":      in.containerdSock,
 		"docker-sock":          in.dockerSock,
 		"runtime":              in.runtime,
+		"custom-config":        in.customConfig,
 	}
 
 	d, err := driver.GetDriver(ctx, in.name, driverFactory, rootOpts.KubeClientConfig, flags, in.configFile, driverOpts, "" /*contextPathHash*/)
@@ -137,6 +139,7 @@ Driver Specific Usage:
 	flags.BoolVar(&options.rootless, "rootless", false, "Run in rootless mode")
 	flags.StringVar(&options.loadbalance, "loadbalance", "random", "Load balancing strategy [random, sticky]")
 	flags.StringVar(&options.worker, "worker", "auto", "Worker backend [auto, runc, containerd]")
+	flags.StringVar(&options.customConfig, "custom-config", "", "Name of a ConfigMap containing custom files (e.g., certs), mounted in /etc/config/ - use 'kubectl create configmap ... --from-file=...'")
 
 	return cmd
 }

pkg/driver/kubernetes/driver.go

@@ -51,7 +51,6 @@ const (
 	// TODO - consider adding other default values here to aid users in fine-tuning by editing the configmap post deployment
 	DefaultConfigFileTemplate = `# Default buildkitd configuration.  Use --config <path/to/file> to override during create
 debug = false
-root = "/var/lib/buildkit/{{ .Name }}"
 [worker.containerd]
   namespace = "{{ .ContainerdNamespace }}"
 `

pkg/driver/kubernetes/factory.go

@@ -176,6 +176,8 @@ func (d *Driver) initDriverFromConfig() error {
 				return errors.Errorf("invalid runtime %q", v)
 			}
 			deploymentOpt.ContainerRuntime = v
+		case "custom-config":
+			deploymentOpt.CustomConfig = v
 		default:
 			return errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
 		}

pkg/driver/kubernetes/manifest/manifest.go

@@ -23,6 +23,7 @@ type DeploymentOpt struct {
 	ContainerdSockHostPath string
 	DockerSockHostPath     string
 	ContainerRuntime       string
+	CustomConfig           string
 }
 
 const (
@@ -127,6 +128,11 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
 			return nil, err
 		}
 	}
+	if opt.CustomConfig != "" {
+		if err := addCustomConfigMount(d, opt); err != nil {
+			return nil, err
+		}
+	}
 	return d, nil
 }
 
@@ -146,10 +152,12 @@ func toRootless(d *appsv1.Deployment) error {
 
 func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error {
 	labels := labels(opt)
+	buildkitRoot := "/var/lib/buildkit/" + opt.Name
 	d.Spec.Template.Spec.Containers[0].Args = append(
 		d.Spec.Template.Spec.Containers[0].Args,
 		"--oci-worker=false",
 		"--containerd-worker=true",
+		"--root", buildkitRoot,
 	)
 	mountPropagationBidirectional := corev1.MountPropagationBidirectional
 	d.Spec.Template.Spec.Containers[0].VolumeMounts = append(
@@ -160,7 +168,7 @@ func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error {
 		},
 		corev1.VolumeMount{
 			Name:             "var-lib-buildkit",
-			MountPath:        "/var/lib/buildkit/" + opt.Name,
+			MountPath:        buildkitRoot,
 			MountPropagation: &mountPropagationBidirectional,
 		},
 		corev1.VolumeMount{
@@ -204,7 +212,7 @@ func toContainerdWorker(d *appsv1.Deployment, opt *DeploymentOpt) error {
 			Name: "var-lib-buildkit",
 			VolumeSource: corev1.VolumeSource{
 				HostPath: &corev1.HostPathVolumeSource{
-					Path: "/var/lib/buildkit/" + opt.Name,
+					Path: buildkitRoot,
 					Type: &hostPathDirectoryOrCreate,
 				}},
 		},
@@ -305,6 +313,31 @@ func addDockerSockMount(d *appsv1.Deployment, opt *DeploymentOpt) error {
 	return nil
 }
 
+func addCustomConfigMount(d *appsv1.Deployment, opt *DeploymentOpt) error {
+	d.Spec.Template.Spec.Containers[0].VolumeMounts = append(
+		d.Spec.Template.Spec.Containers[0].VolumeMounts,
+		corev1.VolumeMount{
+			Name:      "custom-config",
+			MountPath: "/etc/config/",
+		},
+	)
+	d.Spec.Template.Spec.Volumes = append(
+		d.Spec.Template.Spec.Volumes,
+		corev1.Volume{
+			Name: "custom-config",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: opt.CustomConfig,
+					},
+				},
+			},
+		},
+	)
+
+	return nil
+}
+
 func NewConfigMap(opt *DeploymentOpt, contents []byte) *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{