Skip to content

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
csegarragonz committed Nov 6, 2023
1 parent 550a2e0 commit eacac7a
Show file tree
Hide file tree
Showing 6 changed files with 34 additions and 110 deletions.
83 changes: 0 additions & 83 deletions conf-files/k8s_registry.yaml

This file was deleted.

12 changes: 8 additions & 4 deletions docs/registry.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Using a Local Registry

In order to use a local image registry we need to configure both `containerd`
and `Kata` to like our home-baked registry. In addition, Kata does not seem to
be able to use HTTP registries inside the guest, so we need to go an extra
In order to use a local image registry we need to configure `containerd`,
`Kata`, and `containerd` to like our home-baked registry. In addition, Kata does
not seem to be able to use HTTP registries inside the guest, so we need to go an extra
step and configure HTTPS certificates for our registry too.

To this extent, we first create a self-signed certificate, and give it the
Expand All @@ -13,9 +13,13 @@ registry name.
Second, we need to update the docker config to include our certificates for
this registry, as well as containerd's.

Finally, we need to include both the updated `/etc/hosts` file with the DNS
Third, we need to include both the updated `/etc/hosts` file with the DNS
entries, as well as the certificate, inside the agent's `initrd`.

Finally, we need to configure Knative to accept self-signed certificates. To
do so, we need to update the `controller` deployment by applying a [patch](
./conf-files/knative_controller_custom_certs.yaml.j2).

All this process is automated when we start the local registry with the provided
task:

Expand Down
35 changes: 18 additions & 17 deletions tasks/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,11 @@
from os.path import exists, join
from subprocess import run
from tasks.util.docker import is_ctr_running
from tasks.util.env import CONF_FILES_DIR, K8S_CONFIG_DIR, LOCAL_REGISTRY_URL

# TODO: rename and move this method elsewhere
from tasks.util.env import get_kbs_url
from tasks.util.env import K8S_CONFIG_DIR, LOCAL_REGISTRY_URL
from tasks.util.env import get_node_url
from tasks.util.kata import replace_agent
from tasks.util.knative import configure_self_signed_certs
from tasks.util.kubeadm import run_kubectl_command
from tasks.util.pid import get_pid
from tasks.util.toml import update_toml

HOST_CERT_DIR = join(K8S_CONFIG_DIR, "local-registry")
Expand All @@ -23,13 +20,15 @@

REGISTRY_IMAGE_TAG = "registry:2.7"

K8S_SECRET_NAME = "csg-coco-registry-customca"


@task
def start(ctx):
"""
Configure a local container registry reachable from CoCo guests in K8s
"""
this_ip = get_kbs_url()
this_ip = get_node_url()

# ----------
# DNS Config
Expand Down Expand Up @@ -168,29 +167,31 @@ def start(ctx):
# ----------

# First, create a k8s secret with the credentials
secret_name = "csg-coco-registry-customca"
kube_cmd = (
"-n knative-serving create secret generic {} --from-file=ca.crt={}".format(
secret_name, HOST_CERT_PATH
K8S_SECRET_NAME, HOST_CERT_PATH
)
)
run_kubectl_command(kube_cmd)

# Second, patch the controller deployment
configure_self_signed_certs(HOST_CERT_PATH, secret_name)
configure_self_signed_certs(HOST_CERT_PATH, K8S_SECRET_NAME)


@task
def stop(ctx):
"""
Remove the container registry in the k8s cluster
"""
# First, kill the prot-forward process running in the background
pid = get_pid("kubectl")
run("kill -9 {}".format(pid), shell=True, check=True)
registry_k8s_file = join(CONF_FILES_DIR, "k8s_registry.yaml")
# TODO: is this enough to clean the images?
run_kubectl_command("delete -f {}".format(registry_k8s_file))
We follow the steps in start in reverse order, paying particular interest
to the steps that are not idempotent (e.g. creating a k8s secret).
"""
# For Knative, we only need to delete the secret, as the other bit is a
# patch to the controller deployment that can be applied again
kube_cmd = "-n knative-serving delete secret {}".format(K8S_SECRET_NAME)
run_kubectl_command(kube_cmd)

# TODO: more cleanup!
# For Kata and containerd, all configuration is reversible, so we only
# need to sop the container image
docker_cmd = "docker run --rm -f {}".format(REGISTRY_CTR_NAME)
run(docker_cmd, shell=True, check=True)
4 changes: 2 additions & 2 deletions tasks/util/coco.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from os.path import join
from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url
from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_node_url
from tasks.util.toml import read_value_from_toml, update_toml


Expand Down Expand Up @@ -30,7 +30,7 @@ def guest_attestation(mode="off"):
[hypervisor.qemu]
guest_pre_attestation_kbs_uri = "{kbs_url}:{kbs_port}"
""".format(
kbs_url=get_kbs_url(), kbs_port=KBS_PORT
kbs_url=get_node_url(), kbs_port=KBS_PORT
)
update_toml(conf_file_path, updated_toml_str)

Expand Down
6 changes: 4 additions & 2 deletions tasks/util/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,11 @@
KBS_PORT = 44444


def get_kbs_url():
def get_node_url():
"""
Get the external KBS IP that can be reached from both host and guest
Get the external node IP that can be reached from both host and guest
This IP is both used for the KBS, and for deploying a local docker registry.
If the KBS is deployed using docker compose with host networking and the
port is forwarded to the host (i.e. KBS is bound to :${KBS_PORT}, then
Expand Down
4 changes: 2 additions & 2 deletions tasks/util/sev.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from sevsnpmeasure.vmm_types import VMMType
from sevsnpmeasure.vcpu_types import cpu_sig as sev_snp_cpu_sig
from subprocess import run
from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url
from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_node_url
from tasks.util.toml import read_value_from_toml


Expand Down Expand Up @@ -34,7 +34,7 @@ def get_kernel_append():
"console=hvc1",
"debug" if agent_log else "quiet",
"panic=1 nr_cpus=1 selinux=0",
"agent.aa_kbc_params=online_sev_kbc::{}:{}".format(get_kbs_url(), KBS_PORT),
"agent.aa_kbc_params=online_sev_kbc::{}:{}".format(get_node_url(), KBS_PORT),
"scsi_mod.scan=none",
"agent.log=debug" if agent_log else "",
"agent.debug_console agent.debug_console_vport=1026" if debug_console else "",
Expand Down

0 comments on commit eacac7a

Please sign in to comment.