diff --git a/conf-files/k8s_registry.yaml b/conf-files/k8s_registry.yaml deleted file mode 100644 index 40b23e1..0000000 --- a/conf-files/k8s_registry.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 - spec: - containers: - - name: registry - image: registry:2.8.3 - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - ports: - - containerPort: 5000 - name: registry - protocol: TCP - ---- - -apiVersion: v1 -kind: Service -metadata: - name: kube-registry - namespace: kube-system - labels: - k8s-app: kube-registry -spec: - selector: - k8s-app: kube-registry - ports: - - name: registry - port: 5000 - protocol: TCP - ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-registry-proxy - namespace: kube-system - labels: - k8s-app: kube-registry - kubernetes.io/cluster-service: "true" - version: v0.4 -spec: - selector: - matchLabels: - k8s-app: kube-registry - version: v0.4 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0.4 - spec: - containers: - - name: kube-registry-proxy - image: gcr.io/google_containers/kube-registry-proxy:0.4 - env: - - name: REGISTRY_HOST - value: kube-registry.kube-system.svc.cluster.local - - name: REGISTRY_PORT - value: "5000" - ports: - - name: registry - containerPort: 80 - hostPort: 5000 diff --git a/docs/registry.md b/docs/registry.md index 07ad667..4736f2b 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -1,8 +1,8 @@ # Using a Local Registry -In order to use a local image registry we need to configure both `containerd` -and `Kata` to like our home-baked registry. In addition, Kata does not seem to -be able to use HTTP registries inside the guest, so we need to go an extra +In order to use a local image registry we need to configure `containerd`, +`Kata`, and `containerd` to like our home-baked registry. In addition, Kata does +not seem to be able to use HTTP registries inside the guest, so we need to go an extra step and configure HTTPS certificates for our registry too. To this extent, we first create a self-signed certificate, and give it the @@ -13,9 +13,13 @@ registry name. Second, we need to update the docker config to include our certificates for this registry, as well as containerd's. -Finally, we need to include both the updated `/etc/hosts` file with the DNS +Third, we need to include both the updated `/etc/hosts` file with the DNS entries, as well as the certificate, inside the agent's `initrd`. +Finally, we need to configure Knative to accept self-signed certificates. To +do so, we need to update the `controller` deployment by applying a [patch]( +./conf-files/knative_controller_custom_certs.yaml.j2). + All this process is automated when we start the local registry with the provided task: diff --git a/tasks/registry.py b/tasks/registry.py index fb30385..ab27a17 100644 --- a/tasks/registry.py +++ b/tasks/registry.py @@ -3,14 +3,11 @@ from os.path import exists, join from subprocess import run from tasks.util.docker import is_ctr_running -from tasks.util.env import CONF_FILES_DIR, K8S_CONFIG_DIR, LOCAL_REGISTRY_URL - -# TODO: rename and move this method elsewhere -from tasks.util.env import get_kbs_url +from tasks.util.env import K8S_CONFIG_DIR, LOCAL_REGISTRY_URL +from tasks.util.env import get_node_url from tasks.util.kata import replace_agent from tasks.util.knative import configure_self_signed_certs from tasks.util.kubeadm import run_kubectl_command -from tasks.util.pid import get_pid from tasks.util.toml import update_toml HOST_CERT_DIR = join(K8S_CONFIG_DIR, "local-registry") @@ -23,13 +20,15 @@ REGISTRY_IMAGE_TAG = "registry:2.7" +K8S_SECRET_NAME = "csg-coco-registry-customca" + @task def start(ctx): """ Configure a local container registry reachable from CoCo guests in K8s """ - this_ip = get_kbs_url() + this_ip = get_node_url() # ---------- # DNS Config @@ -168,29 +167,31 @@ def start(ctx): # ---------- # First, create a k8s secret with the credentials - secret_name = "csg-coco-registry-customca" kube_cmd = ( "-n knative-serving create secret generic {} --from-file=ca.crt={}".format( - secret_name, HOST_CERT_PATH + K8S_SECRET_NAME, HOST_CERT_PATH ) ) run_kubectl_command(kube_cmd) # Second, patch the controller deployment - configure_self_signed_certs(HOST_CERT_PATH, secret_name) + configure_self_signed_certs(HOST_CERT_PATH, K8S_SECRET_NAME) @task def stop(ctx): """ Remove the container registry in the k8s cluster - """ - # First, kill the prot-forward process running in the background - pid = get_pid("kubectl") - run("kill -9 {}".format(pid), shell=True, check=True) - registry_k8s_file = join(CONF_FILES_DIR, "k8s_registry.yaml") - # TODO: is this enough to clean the images? - run_kubectl_command("delete -f {}".format(registry_k8s_file)) + We follow the steps in start in reverse order, paying particular interest + to the steps that are not idempotent (e.g. creating a k8s secret). + """ + # For Knative, we only need to delete the secret, as the other bit is a + # patch to the controller deployment that can be applied again + kube_cmd = "-n knative-serving delete secret {}".format(K8S_SECRET_NAME) + run_kubectl_command(kube_cmd) - # TODO: more cleanup! + # For Kata and containerd, all configuration is reversible, so we only + # need to sop the container image + docker_cmd = "docker run --rm -f {}".format(REGISTRY_CTR_NAME) + run(docker_cmd, shell=True, check=True) diff --git a/tasks/util/coco.py b/tasks/util/coco.py index 83eb960..710c1ed 100644 --- a/tasks/util/coco.py +++ b/tasks/util/coco.py @@ -1,5 +1,5 @@ from os.path import join -from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url +from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_node_url from tasks.util.toml import read_value_from_toml, update_toml @@ -30,7 +30,7 @@ def guest_attestation(mode="off"): [hypervisor.qemu] guest_pre_attestation_kbs_uri = "{kbs_url}:{kbs_port}" """.format( - kbs_url=get_kbs_url(), kbs_port=KBS_PORT + kbs_url=get_node_url(), kbs_port=KBS_PORT ) update_toml(conf_file_path, updated_toml_str) diff --git a/tasks/util/env.py b/tasks/util/env.py index 75359f8..249647a 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -54,9 +54,11 @@ KBS_PORT = 44444 -def get_kbs_url(): +def get_node_url(): """ - Get the external KBS IP that can be reached from both host and guest + Get the external node IP that can be reached from both host and guest + + This IP is both used for the KBS, and for deploying a local docker registry. If the KBS is deployed using docker compose with host networking and the port is forwarded to the host (i.e. KBS is bound to :${KBS_PORT}, then diff --git a/tasks/util/sev.py b/tasks/util/sev.py index 59904a2..b3892c6 100644 --- a/tasks/util/sev.py +++ b/tasks/util/sev.py @@ -6,7 +6,7 @@ from sevsnpmeasure.vmm_types import VMMType from sevsnpmeasure.vcpu_types import cpu_sig as sev_snp_cpu_sig from subprocess import run -from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url +from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_node_url from tasks.util.toml import read_value_from_toml @@ -34,7 +34,7 @@ def get_kernel_append(): "console=hvc1", "debug" if agent_log else "quiet", "panic=1 nr_cpus=1 selinux=0", - "agent.aa_kbc_params=online_sev_kbc::{}:{}".format(get_kbs_url(), KBS_PORT), + "agent.aa_kbc_params=online_sev_kbc::{}:{}".format(get_node_url(), KBS_PORT), "scsi_mod.scan=none", "agent.log=debug" if agent_log else "", "agent.debug_console agent.debug_console_vport=1026" if debug_console else "",