eval(vm-detail): add further detail for ovmf plots

bin/qemu_wrapper_remove_sev_blob.py

@@ -0,0 +1,41 @@
+#!/bin/python3
+from os import environ, posix_spawn
+from sys import argv
+
+# ------------------------------
+# This script is meant to be used as a replacement for the hypervisor path
+# in the Kata config file. It takes the QEMU command prepared to boot an SEV
+# guest, and it removes the `sev-guest` blob.
+#
+# We use this script to boot a non-SEV guest with OVMF.
+# ------------------------------
+
+
+def launch_qemu(argv):
+    # Remove the SEV blob
+    sev_idx = -1
+    for ind, arg in enumerate(argv):
+        if "sev-guest" in arg:
+            sev_idx = ind - 1
+    new_argv = argv[:sev_idx] + argv[sev_idx + 2 :]
+
+    # Change the machine type
+    m_idx = new_argv.index("-machine")
+    new_argv[m_idx + 1] = "q35,accel=kvm,kernel_irqchip=split"
+
+    qemu_binary = "/opt/confidential-containers/bin/qemu-system-x86_64"
+    qemu_cmdline = (
+        [qemu_binary]
+        + new_argv
+        + [
+            "--serial",
+            "file:/tmp/qemu-serial.log",
+        ]
+    )
+    # Use posix_spawn instead of the higher-level run, as the latter does
+    # some `fd` re-direction that breaks the underlying QEMU command
+    posix_spawn(qemu_binary, qemu_cmdline, environ)
+
+
+if __name__ == "__main__":
+    launch_qemu(argv)

eval/README.md

@@ -171,6 +171,24 @@ which generates a plot in [`./plots/vm-detail/vm_detail.png`](
 
 ![plot](./plots/vm-detail/vm_detail.png)
 
+We also generate additional plots where we compare two flame graphs of the
+start-up times of VMs with different configurations.
+
+We have a plot comparing SEV vs no-SEV in [`./plots/vm-detail/vm_detail_multisev.png`](
+./plots/vm-detail/vm_detail_multisev.png):
+
+![plot](./plots/vm-detail/vm_detail_multisev.png)
+
+another one where we compare the impact of the inital VM memory size
+[`./plots/vm-detail/vm_detail_multimem.png`](./plots/vm-detail/vm_detail_multimem.png):
+
+![plot](./plots/vm-detail/vm_detail_multimem.png)
+
+and another one where we compare the impact of the firmware size
+[`./plots/vm-detail/vm_detail_multiovmf.png`](./plots/vm-detail/vm_detail_multiovmf.png):
+
+![plot](./plots/vm-detail/vm_detail_multiovmf.png)
+
 ### Image Pull
 
 This experiment breaks down the costs associated to pulling a container image

eval/results/vm-detail/coco-fw-sig-enc_1_cold.csv

@@ -0,0 +1,17 @@
+Run,Event,TimeStampMs
+0,StartRunPodSandbox,1699562856.858289
+0,StartVMPreparation,1699562856.981932
+0,StartVMStarted,1699562857.058932
+0,StartPreAtt,1699562858.153284
+0,EndPreAtt,1699562858.346251
+0,EndVMStarted,1699562858.381809
+0,StartOVMFBoot,1699562858.429145
+0,StartOVMFDxeMain,1699562858.429145
+0,StartOVMFVerify,1699562861.8513165
+0,EndOVMFVerify,1699562862.113079
+0,EndOVMFDxeMain,1699562862.2181027
+0,StartGuestKernelBoot,1699562862.37501
+0,EndOVMFBoot,1699562862.37501
+0,EndGuestKernelBoot,1699562863.090781
+0,AgentStarted,1699562863.101919
+0,EndRunPodSandbox,1699562863.160194

eval/results/vm-detail/coco-fw-sig-enc_64_cold.csv

@@ -0,0 +1,17 @@
+Run,Event,TimeStampMs
+0,StartRunPodSandbox,1699391028.711745
+0,StartVMPreparation,1699391028.819409
+0,StartVMStarted,1699391028.924948
+0,StartPreAtt,1699391058.570334
+0,EndPreAtt,1699391058.7891
+0,EndVMStarted,1699391058.824617
+0,StartOVMFBoot,1699391058.890586
+0,StartOVMFDxeMain,1699391058.890586
+0,StartOVMFVerify,1699391062.4319131
+0,EndOVMFVerify,1699391062.7007267
+0,EndOVMFDxeMain,1699391062.817648
+0,StartGuestKernelBoot,1699391062.989718
+0,EndOVMFBoot,1699391062.989718
+0,EndGuestKernelBoot,1699391064.40825
+0,AgentStarted,1699391064.420966
+0,EndRunPodSandbox,1699391064.498001

eval/results/vm-detail/coco-nosev-ovmf_1_cold.csv

@@ -0,0 +1,11 @@
+Run,Event,TimeStampMs
+0,StartRunPodSandbox,1699893884.748306
+0,StartVMPreparation,1699893884.869838
+0,StartVMStarted,1699893884.919469
+0,StartPreAtt,1699893884.919469
+0,EndPreAtt,1699893884.919469
+0,EndVMStarted,1699893884.964831
+0,StartGuestKernelBoot,1699893886.092737
+0,EndGuestKernelBoot,1699893886.81893
+0,AgentStarted,1699893886.83003
+0,EndRunPodSandbox,1699893886.880283

eval/results/vm-detail/coco-nosev_1_cold.csv

@@ -0,0 +1,11 @@
+Run,Event,TimeStampMs
+0,StartRunPodSandbox,1699654942.724576
+0,StartVMPreparation,1699654942.828931
+0,StartVMStarted,1699654942.884835
+0,StartPreAtt,1699654942.884835
+0,EndPreAtt,1699654942.884835
+0,EndVMStarted,1699654942.960531
+0,StartGuestKernelBoot,1699654943.051211
+0,EndGuestKernelBoot,1699654944.067048
+0,AgentStarted,1699654944.078369
+0,EndRunPodSandbox,1699654944.141366

tasks/eval/mem_size.py

@@ -18,38 +18,12 @@
 )
 from tasks.eval.util.pod import wait_for_pod_ready_and_get_ts
 from tasks.eval.util.setup import setup_baseline
-from tasks.util.env import KATA_CONFIG_DIR
 from tasks.util.k8s import template_k8s_file
+from tasks.util.kata import get_default_vm_mem_size, update_vm_mem_size
 from tasks.util.kubeadm import get_pod_names_in_ns, run_kubectl_command
-from tasks.util.toml import read_value_from_toml, update_toml
 from time import sleep, time
 
 
-def get_default_vm_mem_size():
-    """
-    Get the default memory assigned to each new VM from the Kata config file.
-    This value is expressed in MB. We also take by default, accross baselines,
-    the value used for the qemu-sev runtime class.
-    """
-    toml_path = join(KATA_CONFIG_DIR, "configuration-qemu-sev.toml")
-    mem = int(read_value_from_toml(toml_path, "hypervisor.qemu.default_memory"))
-    assert mem > 0, "Read non-positive default memory size: {}".format(mem)
-    return mem
-
-
-def update_vm_mem_size(toml_path, new_mem_size):
-    """
-    Update the default VM memory size in the Kata config file
-    """
-    updated_toml_str = """
-    [hypervisor.qemu]
-    default_memory = {mem_size}
-    """.format(
-        mem_size=new_mem_size
-    )
-    update_toml(toml_path, updated_toml_str)
-
-
 def do_run(result_file, baseline, num_run, num_par_inst):
     start_ts = time()
 

tasks/eval/util/env.py

@@ -1,4 +1,4 @@
-from tasks.util.env import KATA_CONFIG_DIR, PROJ_ROOT
+from tasks.util.env import BIN_DIR, COCO_ROOT, KATA_CONFIG_DIR, PROJ_ROOT
 from os.path import join
 
 EVAL_ROOT = join(PROJ_ROOT, "eval")
@@ -29,14 +29,25 @@
         "runtime_class": "kata",
         "cri_handler": "",
         "image_tag": "unencrypted",
+        "firmware": "",
     },
     # This baseline uses plain Knative on CoCo, but without SEV-enabled VMs
-    # (so all CoCo machinery, but no runtime memory encryption)
     "coco-nosev": {
         "conf_file": join(KATA_CONFIG_DIR, "configuration-qemu.toml"),
         "runtime_class": "kata-qemu",
         "cri_handler": "cc",
         "image_tag": "unencrypted",
+        "firmware": "",
+    },
+    # This baseline is the same one as before, but makes sure we use OVMF as
+    # firware (Kata may use SeaBIOS by default)
+    "coco-nosev-ovmf": {
+        "conf_file": join(KATA_CONFIG_DIR, "configuration-qemu-sev.toml"),
+        "runtime_class": "kata-qemu-sev",
+        "cri_handler": "cc",
+        "image_tag": "unencrypted",
+        "firmware": join(COCO_ROOT, "share", "ovmf", "OVMF_CSG.fd"),
+        "hypervisor": join(BIN_DIR, "qemu_wrapper_replace_bios_sev.py"),
     },
     # This baseline uses Knative on confidential VMs with Kata, but does not
     # have any kind of attestation feature. This is an _insecure_ baseline,

tasks/eval/util/setup.py

@@ -1,10 +1,47 @@
-from os.path import join
+from os import makedirs
+from os.path import basename, exists, join
+from subprocess import run
 from tasks.eval.util.env import BASELINES, EXPERIMENT_IMAGE_REPO
-from tasks.util.coco import guest_attestation, signature_verification
+from tasks.util.coco import (
+    guest_attestation,
+    signature_verification,
+    set_firmware,
+    set_hypervisor,
+)
 from tasks.util.containerd import set_cri_handler
 from tasks.util.kbs import clear_kbs_db, provision_launch_digest
 
 
+def get_backup_file_path_from_conf_file(conf_file):
+    backup_dir = "/tmp/coco-serverless-back-up"
+    if not exists(backup_dir):
+        makedirs(backup_dir)
+
+    backup_file = join(backup_dir, basename(conf_file))
+    return backup_file
+
+
+def backup_kata_config_file(conf_file):
+    backup_file = get_backup_file_path_from_conf_file(conf_file)
+    run("cp {} {}".format(conf_file, backup_file), shell=True, check=True)
+
+
+def restore_kata_config_file(conf_file):
+    backup_file = get_backup_file_path_from_conf_file(conf_file)
+    run("cp {} {}".format(backup_file, conf_file), shell=True, check=True)
+
+
+def cleanup_baseline(baseline):
+    """
+    Clean-up the system after executing a baseline
+
+    This method reverts the Kata configuration file to the default one after
+    a baseline has executed
+    """
+    baseline_traits = BASELINES[baseline]
+    restore_kata_config_file(baseline_traits["conf_file"])
+
+
 def setup_baseline(baseline, used_images, image_repo=EXPERIMENT_IMAGE_REPO):
     """
     Configure the system for a specific baseline
@@ -15,12 +52,24 @@ def setup_baseline(baseline, used_images, image_repo=EXPERIMENT_IMAGE_REPO):
     """
     baseline_traits = BASELINES[baseline]
 
+    # First, save a copy of the current config file so that we can reset it
+    # after we are done
+    backup_kata_config_file(baseline_traits["conf_file"])
+
+    # Change the path to the used hypervisor
+    if "hypervisor" in baseline_traits:
+        set_hypervisor(baseline_traits["conf_file"], baseline_traits["hypervisor"])
+
     # Change the CRI handler
     if "cri_handler" in baseline_traits:
         set_cri_handler(
             baseline_traits["runtime_class"], baseline_traits["cri_handler"]
         )
 
+    # Change the firmware
+    if "firmware" in baseline_traits:
+        set_firmware(baseline_traits["conf_file"], baseline_traits["firmware"])
+
     # Turn guest pre-attestation on/off (connect KBS to PSP)
     if "guest_attestation" in baseline_traits:
         guest_attestation(