diff --git a/apps/helloworld-knative-encrypted/service.yaml b/apps/helloworld-knative-encrypted/service.yaml new file mode 100644 index 0000000..40518d1 --- /dev/null +++ b/apps/helloworld-knative-encrypted/service.yaml @@ -0,0 +1,24 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: helloworld-knative + annotations: + "features.knative.dev/podspec-runtimeclassname": "enabled" +spec: + template: + metadata: + labels: + apps.coco-serverless/name: helloworld-py + io.katacontainers.config.pre_attestation.enabled: "false" + spec: + runtimeClassName: kata-qemu-sev + # coco-knative: need to run user container as root + securityContext: + runAsUser: 1000 + containers: + - image: csegarragonz/coco-helloworld-py:encrypted + ports: + - containerPort: 8080 + env: + - name: TARGET + value: "World" diff --git a/conf-files/ocicrypt.conf b/conf-files/ocicrypt.conf new file mode 100644 index 0000000..477e6c0 --- /dev/null +++ b/conf-files/ocicrypt.conf @@ -0,0 +1,7 @@ +{ + "key-providers": { + "attestation-agent": { + "grpc": "127.0.0.1:50000" + } + } +} diff --git a/docs/helloworld_knative_attestation.md b/docs/helloworld_knative_attestation.md index 3db6bfa..fec4c24 100644 --- a/docs/helloworld_knative_attestation.md +++ b/docs/helloworld_knative_attestation.md @@ -19,9 +19,15 @@ following subsections (in increasing order of security): with the right firmware, kernel, initrd, and Kata Agent configuration. * [Signed Container Images](#signed-container-images) - Attest that the images used have been signed with a well-known private key. +* [Encrypted Container Images](#encrypted-container-images) - In addition to +signatures, consume encrypted container images. After that, you may jump to [running the application](#run-the-application). +> Note that, if you are running different attestation types one after the other +> you may want to clear the contents of the KBS between runs using: +> `inv kbs.clear-db` + ## Firmware Digest Before running the application, we need to generate the expected launch digest @@ -109,6 +115,31 @@ Now, you may proceed to [running the application](#run-the-application). ## Encrypted Container Images +In this last section, we will configure the system to consume encrypted +container images from public registries. + +To encrypt docker images, we use [`skopeo`](https://github.com/containers/skopeo). +`skopeo` can be used together with image signatures, just make sure to pass the +`--sign` flag: + +```bash +inv skopeo.encrypt-container-image "docker.io/csegarragonz/coco-helloworld-py:unencrypted" --sign +``` + +> To check that the image is actually encrypted, you may try to run it: +> `docker run docker.io/csegarragonz/coco-helloworld-py:encrypted` + +Then, as in the previous section, update the KBS to set the right signature +verification policy: + +```bash +# IMPORTANT: do not use the `--clean` flag here, as the encrypt-container-image +# command will provision a secret to the KBS +inv kbs.provision-launch-digest --signature-policy verify +``` + +Finally, you may proceed to [running the application](#run-the-application). + ## Run the application Once the KBS has been populated with the right measurements and secrets, we can @@ -117,3 +148,9 @@ deploy the workload just like with the `Hello world! (Knative)` app: ```bash kubectl apply -f ./apps/helloworld-knative ``` + +note that if you are using encrypted images you will have to do: + +```bash +kubectl apply -f ./apps/helloworld-knative-encrypted +``` diff --git a/tasks/__init__.py b/tasks/__init__.py index 8851c4b..dc24039 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -13,6 +13,7 @@ from . import kubeadm from . import operator from . import sev +from . import skopeo ns = Collection( apps, @@ -28,4 +29,5 @@ kubeadm, operator, sev, + skopeo, ) diff --git a/tasks/coco.py b/tasks/coco.py index 44f5577..4a0d991 100644 --- a/tasks/coco.py +++ b/tasks/coco.py @@ -1,7 +1,6 @@ from invoke import task from os.path import join -from tasks.util.env import KATA_CONFIG_DIR -from tasks.util.kbs import KBS_PORT, get_kbs_url +from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url from tasks.util.toml import read_value_from_toml, update_toml diff --git a/tasks/kbs.py b/tasks/kbs.py index 2245cf9..ccfb06c 100644 --- a/tasks/kbs.py +++ b/tasks/kbs.py @@ -1,19 +1,16 @@ -from base64 import b64encode from invoke import task -from os.path import exists, join +from os.path import exists from subprocess import run from tasks.util.cosign import COSIGN_PUB_KEY from tasks.util.kbs import ( - NO_SIGNATURE_POLICY, SIMPLE_KBS_DIR, - SIMPLE_KBS_RESOURCE_PATH, SIGNATURE_POLICY_NONE, create_kbs_resource, connect_to_kbs_db, populate_signature_verification_policy, + set_launch_measurement_policy, validate_signature_verification_policy, ) -from tasks.util.sev import get_launch_digest SIGNATURE_POLICY_STRING_ID = "default/security-policy/test" @@ -74,7 +71,7 @@ def clear_db(ctx): @task -def provision_launch_digest(ctx, signature_policy=NO_SIGNATURE_POLICY, clean=False): +def provision_launch_digest(ctx, signature_policy=SIGNATURE_POLICY_NONE, clean=False): """ Provision the KBS with the launch digest for the current node @@ -101,85 +98,52 @@ def provision_launch_digest(ctx, signature_policy=NO_SIGNATURE_POLICY, clean=Fal if clean: clear_db(ctx) - # First, add our launch digest to the KBS policy - ld = get_launch_digest("sev") - ld_b64 = b64encode(ld).decode() - - # Create a new record - connection = connect_to_kbs_db() - with connection: - with connection.cursor() as cursor: - policy_id = 10 - - # When enabling signature verification, we need to provide a - # signature policy. This policy has a constant string identifier - # that the kata agent will ask for (default/security-policy/test), - # which points to a config file that specifies how to validate - # signatures - if signature_policy != NO_SIGNATURE_POLICY: - resource_path = "signature_policy_{}.json".format(signature_policy) - - if signature_policy == SIGNATURE_POLICY_NONE: - # If we set a `none` signature policy, it means that we don't - # check any signatures on the pulled container images - policy_json_str = populate_signature_verification_policy( - signature_policy - ) - else: - # The verify policy, checks that the image has been signed - # with a given key. As everything in the KBS, the key - # we give in the policy is an ID for another resource. - # Note that the following resource prefix is NOT required - # (i.e. we could change it to keys/cosign/1 as long as the - # corresponding resource exists) - signing_key_resource_id = "default/cosign-key/1" - policy_json_str = populate_signature_verification_policy( - signature_policy, - [ - [ - "docker.io/csegarragonz/coco-helloworld-py", - signing_key_resource_id, - ], - [ - "docker.io/csegarragonz/coco-knative-sidecar", - signing_key_resource_id, - ], - ], - ) - - # Create a resource for the signing key - signing_key_kbs_path = "cosign.pub" - signing_key_resource_path = join( - SIMPLE_KBS_RESOURCE_PATH, signing_key_kbs_path - ) - sql = "INSERT INTO resources VALUES(NULL, NULL, " - sql += "'{}', '{}', {})".format( - signing_key_resource_id, signing_key_kbs_path, policy_id - ) - cursor.execute(sql) - - # Lastly, we copy the public signing key to the resource - # path annotated in the policy - cp_cmd = "cp {} {}".format( - COSIGN_PUB_KEY, signing_key_resource_path - ) - run(cp_cmd, shell=True, check=True) - - create_kbs_resource(resource_path, policy_json_str) - - # Create the resource (containing the signature policy) in the KBS - sql = "INSERT INTO resources VALUES(NULL, NULL, " - sql += "'{}', '{}', {})".format( - SIGNATURE_POLICY_STRING_ID, resource_path, policy_id - ) - cursor.execute(sql) - - # We associate the signature policy to a digest policy, meaning - # that irrespective of what signature policy are we using (even if - # we are not checking any signatures) we will always check the FW - # digest against the measured one - sql = "INSERT INTO policy VALUES ({}, ".format(policy_id) - sql += "'[\"{}\"]', '[]', 0, 0, '[]', now(), NULL, 1)".format(ld_b64) - cursor.execute(sql) - - connection.commit() + # First, we provision a launch digest policy that only allows to + # boot confidential VMs with the launch measurement that we have + # just calculated. We will associate signature verification and + # image encryption policies to this launch digest policy. + set_launch_measurement_policy() + + # To make sure the launch policy is enforced, we must enable + # signature verification. This means that we also need to provide a + # signature policy. This policy has a constant string identifier + # that the kata agent will ask for (default/security-policy/test), + # which points to a config file that specifies how to validate + # signatures + resource_path = "signature_policy_{}.json".format(signature_policy) + + if signature_policy == SIGNATURE_POLICY_NONE: + # If we set a `none` signature policy, it means that we don't + # check any signatures on the pulled container images (still + # necessary to set the policy to check the launch measurment) + policy_json_str = populate_signature_verification_policy(signature_policy) + else: + # The verify policy, checks that the image has been signed + # with a given key. As everything in the KBS, the key + # we give in the policy is an ID for another resource. + # Note that the following resource prefix is NOT required + # (i.e. we could change it to keys/cosign/1 as long as the + # corresponding resource exists) + signing_key_resource_id = "default/cosign-key/1" + policy_json_str = populate_signature_verification_policy( + signature_policy, + [ + [ + "docker.io/csegarragonz/coco-helloworld-py", + signing_key_resource_id, + ], + [ + "docker.io/csegarragonz/coco-knative-sidecar", + signing_key_resource_id, + ], + ], + ) + + # Create a resource for the signing key + with open(COSIGN_PUB_KEY) as fh: + create_kbs_resource(signing_key_resource_id, "cosign.pub", fh.read()) + + # Finally, create a resource for the image signing policy. Note that the + # resource ID for the image signing policy is hardcoded in the kata agent + # (particularly in the attestation agent) + create_kbs_resource(SIGNATURE_POLICY_STRING_ID, resource_path, policy_json_str) diff --git a/tasks/skopeo.py b/tasks/skopeo.py new file mode 100644 index 0000000..571580a --- /dev/null +++ b/tasks/skopeo.py @@ -0,0 +1,103 @@ +from base64 import b64encode +from invoke import task +from json import loads as json_loads +from os.path import exists, join +from subprocess import run +from tasks.util.cosign import sign_container_image +from tasks.util.env import CONF_FILES_DIR, K8S_CONFIG_DIR +from tasks.util.guest_components import ( + start_coco_keyprovider, + stop_coco_keyprovider, +) +from tasks.util.kbs import create_kbs_secret + +SKOPEO_VERSION = "1.13.0" +SKOPEO_IMAGE = "quay.io/skopeo/stable:v{}".format(SKOPEO_VERSION) +SKOPEO_ENCRYPTION_KEY = join(K8S_CONFIG_DIR, "image_enc.key") +AA_CTR_ENCRYPTION_KEY = "/tmp/image_enc.key" + + +def run_skopeo_cmd(cmd, capture_stdout=False): + ocicrypt_conf_host = join(CONF_FILES_DIR, "ocicrypt.conf") + ocicrypt_conf_guest = "/ocicrypt.conf" + skopeo_cmd = [ + "docker run --rm", + "--net host", + "-e OCICRYPT_KEYPROVIDER_CONFIG={}".format(ocicrypt_conf_guest), + "-v {}:{}".format(ocicrypt_conf_host, ocicrypt_conf_guest), + "-v ~/.docker/config.json:/config.json", + SKOPEO_IMAGE, + cmd, + ] + skopeo_cmd = " ".join(skopeo_cmd) + if capture_stdout: + return ( + run(skopeo_cmd, shell=True, capture_output=True) + .stdout.decode("utf-8") + .strip() + ) + else: + run(skopeo_cmd, shell=True, check=True) + + +def create_encryption_key(): + cmd = "head -c32 < /dev/random > {}".format(SKOPEO_ENCRYPTION_KEY) + run(cmd, shell=True, check=True) + + +@task +def encrypt_container_image(ctx, image_tag, sign=False): + """ + Encrypt an OCI container image using Skopeo + + The image tag must be provided in the format: docker.io//:tag + """ + encryption_key_resource_id = "default/image-encryption-key/1" + if not exists(SKOPEO_ENCRYPTION_KEY): + create_encryption_key() + + # We use CoCo's keyprovider server (that implements the ocicrypt protocol) + # to encrypt the OCI image. To that extent, we need to mount the encryption + # key somewhere that the attestation agent (in the keyprovider) can find + # it + start_coco_keyprovider(SKOPEO_ENCRYPTION_KEY, AA_CTR_ENCRYPTION_KEY) + + encrypted_image_tag = image_tag.split(":")[0] + ":encrypted" + skopeo_cmd = [ + "copy --insecure-policy", + "--authfile /config.json", + "--encryption-key", + "provider:attestation-agent:keyid=kbs:///{}::keypath={}".format( + encryption_key_resource_id, AA_CTR_ENCRYPTION_KEY + ), + "docker://{}".format(image_tag), + "docker://{}".format(encrypted_image_tag), + ] + skopeo_cmd = " ".join(skopeo_cmd) + run_skopeo_cmd(skopeo_cmd) + + # Sanity check that the image is actually encrypted + inspect_jsonstr = run_skopeo_cmd( + "inspect docker://{}".format(encrypted_image_tag), capture_stdout=True + ) + inspect_json = json_loads(inspect_jsonstr) + layers = [ + layer["MIMEType"].endswith("tar+gzip+encrypted") + for layer in inspect_json["LayersData"] + ] + if not all(layers): + print("Some layers in image {} are not encrypted!".format(encrypted_image_tag)) + stop_coco_keyprovider() + raise RuntimeError("Image encryption failed!") + + # Create a secret in KBS with the encryption key. Skopeo needs it as raw + # bytes, whereas KBS wants it base64 encoded, so we do the conversion first + with open(SKOPEO_ENCRYPTION_KEY, "rb") as fh: + key_b64 = b64encode(fh.read()).decode() + + create_kbs_secret(encryption_key_resource_id, key_b64) + + if sign: + sign_container_image(encrypted_image_tag) + + stop_coco_keyprovider() diff --git a/tasks/util/env.py b/tasks/util/env.py index f0639ed..00abdc3 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -1,4 +1,5 @@ from os.path import dirname, realpath, join +from subprocess import run PROJ_ROOT = dirname(dirname(dirname(realpath(__file__)))) @@ -42,3 +43,28 @@ # Apps config APPS_SOURCE_DIR = join(PROJ_ROOT, "apps") + +# KBS Config + +KBS_PORT = 44444 + + +def get_kbs_url(): + """ + Get the external KBS IP that can be reached from both host and guest + + If the KBS is deployed using docker compose with host networking and the + port is forwarded to the host (i.e. KBS is bound to :${KBS_PORT}, then + we can use this method to figure out the "public-facing" IP that can be + reached both from the host and the guest + """ + ip_cmd = "ip -o route get to 8.8.8.8" + ip_cmd_out = ( + run(ip_cmd, shell=True, capture_output=True) + .stdout.decode("utf-8") + .strip() + .split(" ") + ) + idx = ip_cmd_out.index("src") + 1 + kbs_url = ip_cmd_out[idx] + return kbs_url diff --git a/tasks/util/guest_components.py b/tasks/util/guest_components.py new file mode 100644 index 0000000..2b868c9 --- /dev/null +++ b/tasks/util/guest_components.py @@ -0,0 +1,53 @@ +from os.path import join +from subprocess import run +from tasks.util.env import PROJ_ROOT +from time import sleep + +GUEST_COMPONENTS_DIR = join(PROJ_ROOT, "..", "guest-components") +COCO_KEYPROVIDER_DIR = join( + GUEST_COMPONENTS_DIR, "attestation-agent", "coco_keyprovider" +) + +COCO_KEYPROVIDER_CTR_NAME = "coco-keyprovider" +COCO_KEYPROVIDER_CTR_PORT = 50000 + + +def start_coco_keyprovider(host_key_path, guest_key_path): + """ + Start the CoCo key-provider to encrypt a docker image using Skopeo + """ + docker_cmd = [ + "docker run -d", + "--net host", + "--name {}".format(COCO_KEYPROVIDER_CTR_NAME), + "-v {}:/usr/src/guest-components".format(GUEST_COMPONENTS_DIR), + "-v {}:{}".format(host_key_path, guest_key_path), + "-w /usr/src/guest-components/attestation-agent/coco-keyprovider", + "rust:1.72", + "bash -c 'rustup component add rustfmt && cargo run --release --bin", + "coco_keyprovider -- --socket 127.0.0.1:{}'".format(COCO_KEYPROVIDER_CTR_PORT), + ] + docker_cmd = " ".join(docker_cmd) + print(docker_cmd) + run(docker_cmd, shell=True, check=True) + + # Wait for the gRPC server to be ready + poll_period = 2 + string_to_check = "listening to socket addr" + while True: + sleep(poll_period) + logs_cmd = "docker logs {}".format(COCO_KEYPROVIDER_CTR_NAME) + ctr_logs = run(logs_cmd, shell=True, capture_output=True).stderr.decode("utf-8") + if string_to_check in ctr_logs: + print("gRPC server ready!") + break + + print("Waiting for keyprovider's gRPC server to be ready...") + + +def stop_coco_keyprovider(): + """ + Stop the CoCo key-provider + """ + docker_cmd = "docker rm -f {}".format(COCO_KEYPROVIDER_CTR_NAME) + run(docker_cmd, shell=True, check=True) diff --git a/tasks/util/kbs.py b/tasks/util/kbs.py index 4534a89..7fe6839 100644 --- a/tasks/util/kbs.py +++ b/tasks/util/kbs.py @@ -1,3 +1,4 @@ +from base64 import b64encode from json import dumps as json_dumps from os import makedirs from os.path import join @@ -5,8 +6,8 @@ from pymysql.cursors import DictCursor from subprocess import run from tasks.util.env import PROJ_ROOT +from tasks.util.sev import get_launch_digest -KBS_PORT = 44444 SIMPLE_KBS_DIR = join(PROJ_ROOT, "..", "simple-kbs") # WARNING: this resource path depends on the KBS' `server` service working # directory. The server expects the `resources` directory to be in: @@ -14,6 +15,8 @@ SIMPLE_KBS_RESOURCE_PATH = join(SIMPLE_KBS_DIR, "resources") SIMPLE_KBS_KEYS_RESOURCE_PATH = join(SIMPLE_KBS_RESOURCE_PATH, "keys") +DEFAULT_LAUNCH_POLICY_ID = 10 + # -------- # Signature Verification Policy # -------- @@ -60,31 +63,39 @@ def connect_to_kbs_db(): return connection -def get_kbs_url(): +def set_launch_measurement_policy(): """ - Get the external KBS IP that can be reached from both host and guest - - If the KBS is deployed using docker compose with host networking and the - port is forwarded to the host (i.e. KBS is bound to :${KBS_PORT}, then - we can use this method to figure out the "public-facing" IP that can be - reached both from the host and the guest + This method configures and sets the launch measurement policy """ - ip_cmd = "ip -o route get to 8.8.8.8" - ip_cmd_out = ( - run(ip_cmd, shell=True, capture_output=True) - .stdout.decode("utf-8") - .strip() - .split(" ") - ) - idx = ip_cmd_out.index("src") + 1 - kbs_url = ip_cmd_out[idx] - return kbs_url - - -def create_kbs_resource(resource_path, resource_contents): + # Get the launch measurement + ld = get_launch_digest("sev") + ld_b64 = b64encode(ld).decode() + + # Create a policy associated to this measurement in the KBS DB + connection = connect_to_kbs_db() + with connection: + with connection.cursor() as cursor: + sql = "INSERT INTO policy VALUES ({}, ".format(DEFAULT_LAUNCH_POLICY_ID) + sql += "'[\"{}\"]', '[]', 0, 0, '[]', now(), NULL, 1)".format(ld_b64) + cursor.execute(sql) + + connection.commit() + + +def create_kbs_resource( + resource_id, + resource_kbs_path, + resource_contents, + resource_launch_policy_id=DEFAULT_LAUNCH_POLICY_ID, +): """ Create a KBS resource for the kata-agent to consume + Each KBS resource is identified by a resource ID. Each KBS resource has + a resource path, where the actual resource lives. In addition, each + each resource is associated to a launch policy, that checks that the FW + digest is as expected. + KBS resources are stored in a `resources` directory in the same **working directory** from which we call the KBS binary. This value can be checked in the simple KBS' docker-compose.yml file. The `resource_path` argument is @@ -92,10 +103,42 @@ def create_kbs_resource(resource_path, resource_contents): """ makedirs(SIMPLE_KBS_RESOURCE_PATH, exist_ok=True) - with open(join(SIMPLE_KBS_RESOURCE_PATH, resource_path), "w") as fh: + # First, insert the resource in the SQL database + connection = connect_to_kbs_db() + with connection: + with connection.cursor() as cursor: + sql = "INSERT INTO resources VALUES(NULL, NULL, " + sql += "'{}', '{}', {})".format( + resource_id, resource_kbs_path, resource_launch_policy_id + ) + cursor.execute(sql) + + connection.commit() + + # Second, dump the resource contents in the specified resource path + with open(join(SIMPLE_KBS_RESOURCE_PATH, resource_kbs_path), "w") as fh: fh.write(resource_contents) +def create_kbs_secret( + secret_id, secret_contents, resource_launch_policy_id=DEFAULT_LAUNCH_POLICY_ID +): + """ + Create a KBS secret for the kata-agent to consume + """ + # First, insert the resource in the SQL database + connection = connect_to_kbs_db() + with connection: + with connection.cursor() as cursor: + sql = "INSERT INTO secrets VALUES(NULL, " + sql += "'{}', '{}', {})".format( + secret_id, secret_contents, resource_launch_policy_id + ) + cursor.execute(sql) + + connection.commit() + + def validate_signature_verification_policy(signature_policy): """ Validate that a given signature policy is supported diff --git a/tasks/util/sev.py b/tasks/util/sev.py index 3fc449a..59904a2 100644 --- a/tasks/util/sev.py +++ b/tasks/util/sev.py @@ -6,8 +6,7 @@ from sevsnpmeasure.vmm_types import VMMType from sevsnpmeasure.vcpu_types import cpu_sig as sev_snp_cpu_sig from subprocess import run -from tasks.util.env import KATA_CONFIG_DIR -from tasks.util.kbs import KBS_PORT, get_kbs_url +from tasks.util.env import KATA_CONFIG_DIR, KBS_PORT, get_kbs_url from tasks.util.toml import read_value_from_toml