diff --git a/apps/helloworld-py/deployment.yaml b/apps/helloworld-py/deployment.yaml index 8fec5c3..d1abebb 100644 --- a/apps/helloworld-py/deployment.yaml +++ b/apps/helloworld-py/deployment.yaml @@ -15,7 +15,7 @@ spec: apps.coco-serverless/name: helloworld-py io.katacontainers.config.pre_attestation.enabled: "false" spec: - runtimeClassName: kata-qemu + runtimeClassName: kata-qemu-sev containers: - name: helloworld-py image: ghcr.io/csegarragonz/coco-helloworld-py:unencrypted diff --git a/eval/plots/startup/startup.pdf b/eval/plots/startup/startup.pdf index 78b433d..f00c65e 100644 Binary files a/eval/plots/startup/startup.pdf and b/eval/plots/startup/startup.pdf differ diff --git a/eval/plots/startup/startup.png b/eval/plots/startup/startup.png index 3f67aa4..715dc1b 100644 Binary files a/eval/plots/startup/startup.png and b/eval/plots/startup/startup.png differ diff --git a/eval/results/startup/coco-fw-sig-enc_cold.csv b/eval/results/startup/coco-fw-sig-enc_cold.csv index 2674c6c..3f75046 100644 --- a/eval/results/startup/coco-fw-sig-enc_cold.csv +++ b/eval/results/startup/coco-fw-sig-enc_cold.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697221341.961345 -0,Initialized,1697221342.0 -0,PodScheduled,1697221342.0 -0,Ready,1697221358.0 -0,ContainersReady,1697221358.0 -1,Initialized,1697221365.0 -1,PodScheduled,1697221365.0 -1,Start,1697221365.0416353 -1,Ready,1697221382.0 -1,ContainersReady,1697221382.0 -2,Initialized,1697221388.0 -2,PodScheduled,1697221388.0 -2,Start,1697221388.1206732 -2,Ready,1697221405.0 -2,ContainersReady,1697221405.0 +0,Start,1697471005.8187528 +0,Initialized,1697471006.0 +0,PodScheduled,1697471006.0 +0,StartImagePull,1697471012.417832 +0,EndImagePull,1697471019.097273 +0,Ready,1697471022.0 +0,ContainersReady,1697471022.0 +1,Start,1697471028.9048746 +1,Initialized,1697471029.0 +1,PodScheduled,1697471029.0 +1,StartImagePull,1697471035.480519 +1,EndImagePull,1697471042.345242 +1,Ready,1697471045.0 +1,ContainersReady,1697471045.0 +2,Start,1697471051.9837024 +2,Initialized,1697471052.0 +2,PodScheduled,1697471052.0 +2,StartImagePull,1697471058.526666 +2,EndImagePull,1697471065.106732 +2,Ready,1697471069.0 +2,ContainersReady,1697471069.0 diff --git a/eval/results/startup/coco-fw-sig-enc_warm.csv b/eval/results/startup/coco-fw-sig-enc_warm.csv index 2f1ce6e..4591506 100644 --- a/eval/results/startup/coco-fw-sig-enc_warm.csv +++ b/eval/results/startup/coco-fw-sig-enc_warm.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697221255.6528904 -0,Initialized,1697221256.0 -0,PodScheduled,1697221256.0 -0,Ready,1697221272.0 -0,ContainersReady,1697221272.0 -1,Start,1697221278.7323177 -1,Initialized,1697221279.0 -1,PodScheduled,1697221279.0 -1,Ready,1697221295.0 -1,ContainersReady,1697221295.0 -2,Start,1697221301.8187425 -2,Initialized,1697221302.0 -2,PodScheduled,1697221302.0 -2,Ready,1697221336.0 -2,ContainersReady,1697221336.0 +0,Start,1697470936.555282 +0,Initialized,1697470937.0 +0,PodScheduled,1697470937.0 +0,StartImagePull,1697470943.266319 +0,EndImagePull,1697470950.055939 +0,Ready,1697470953.0 +0,ContainersReady,1697470953.0 +1,Start,1697470959.6539378 +1,Initialized,1697470960.0 +1,PodScheduled,1697470960.0 +1,StartImagePull,1697470966.313114 +1,EndImagePull,1697470972.984553 +1,Ready,1697470976.0 +1,ContainersReady,1697470976.0 +2,Start,1697470982.73579 +2,Initialized,1697470983.0 +2,PodScheduled,1697470983.0 +2,StartImagePull,1697470989.371295 +2,EndImagePull,1697470996.04208 +2,Ready,1697470999.0 +2,ContainersReady,1697470999.0 diff --git a/eval/results/startup/coco-fw-sig_cold.csv b/eval/results/startup/coco-fw-sig_cold.csv index 94e6adf..6429751 100644 --- a/eval/results/startup/coco-fw-sig_cold.csv +++ b/eval/results/startup/coco-fw-sig_cold.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Initialized,1697221114.0 -0,PodScheduled,1697221114.0 -0,Start,1697221114.1425405 -0,Ready,1697221131.0 -0,ContainersReady,1697221131.0 -1,Initialized,1697221137.0 -1,PodScheduled,1697221137.0 -1,Start,1697221137.219481 -1,Ready,1697221154.0 -1,ContainersReady,1697221154.0 -2,Initialized,1697221160.0 -2,PodScheduled,1697221160.0 -2,Start,1697221160.308928 -2,Ready,1697221177.0 -2,ContainersReady,1697221177.0 +0,Initialized,1697470810.0 +0,PodScheduled,1697470810.0 +0,Start,1697470810.1108723 +0,StartImagePull,1697470816.715057 +0,EndImagePull,1697470823.870758 +0,Ready,1697470828.0 +0,ContainersReady,1697470828.0 +1,Initialized,1697470834.0 +1,PodScheduled,1697470834.0 +1,Start,1697470834.1904464 +1,StartImagePull,1697470840.79858 +1,EndImagePull,1697470847.525106 +1,Ready,1697470851.0 +1,ContainersReady,1697470851.0 +2,Initialized,1697470857.0 +2,PodScheduled,1697470857.0 +2,Start,1697470857.2810328 +2,StartImagePull,1697470863.906405 +2,EndImagePull,1697470870.359324 +2,Ready,1697470874.0 +2,ContainersReady,1697470874.0 diff --git a/eval/results/startup/coco-fw-sig_warm.csv b/eval/results/startup/coco-fw-sig_warm.csv index 12786f0..70ff854 100644 --- a/eval/results/startup/coco-fw-sig_warm.csv +++ b/eval/results/startup/coco-fw-sig_warm.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697221046.878591 -0,Initialized,1697221047.0 -0,PodScheduled,1697221047.0 -0,Ready,1697221063.0 -0,ContainersReady,1697221063.0 -1,Start,1697221069.967245 -1,Initialized,1697221070.0 -1,PodScheduled,1697221070.0 -1,Ready,1697221085.0 -1,ContainersReady,1697221085.0 -2,Initialized,1697221091.0 -2,PodScheduled,1697221091.0 -2,Start,1697221091.0533175 -2,Ready,1697221108.0 -2,ContainersReady,1697221108.0 +0,Start,1697470739.8409154 +0,Initialized,1697470740.0 +0,PodScheduled,1697470740.0 +0,StartImagePull,1697470746.378967 +0,EndImagePull,1697470753.072175 +0,Ready,1697470756.0 +0,ContainersReady,1697470756.0 +1,Start,1697470762.9284194 +1,Initialized,1697470763.0 +1,PodScheduled,1697470763.0 +1,StartImagePull,1697470769.530279 +1,EndImagePull,1697470775.931434 +1,Ready,1697470779.0 +1,ContainersReady,1697470779.0 +2,Initialized,1697470786.0 +2,PodScheduled,1697470786.0 +2,Start,1697470786.0162864 +2,StartImagePull,1697470792.610916 +2,EndImagePull,1697470799.336783 +2,Ready,1697470804.0 +2,ContainersReady,1697470804.0 diff --git a/eval/results/startup/coco-fw_cold.csv b/eval/results/startup/coco-fw_cold.csv index dd8ffb1..54688e3 100644 --- a/eval/results/startup/coco-fw_cold.csv +++ b/eval/results/startup/coco-fw_cold.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697220961.5506494 -0,Initialized,1697220962.0 -0,PodScheduled,1697220962.0 -0,Ready,1697220975.0 -0,ContainersReady,1697220975.0 -1,Start,1697220981.6300936 -1,Initialized,1697220982.0 -1,PodScheduled,1697220982.0 -1,Ready,1697220996.0 -1,ContainersReady,1697220996.0 -2,Start,1697221002.709317 -2,Initialized,1697221003.0 -2,PodScheduled,1697221003.0 -2,Ready,1697221017.0 -2,ContainersReady,1697221017.0 +0,Start,1697470371.6252952 +0,Initialized,1697470372.0 +0,PodScheduled,1697470372.0 +0,StartImagePull,1697470378.218857 +0,EndImagePull,1697470383.633569 +0,Ready,1697470386.0 +0,ContainersReady,1697470386.0 +1,Start,1697470392.7114177 +1,Initialized,1697470393.0 +1,PodScheduled,1697470393.0 +1,StartImagePull,1697470399.253206 +1,EndImagePull,1697470404.577172 +1,Ready,1697470407.0 +1,ContainersReady,1697470407.0 +2,Start,1697470413.7974222 +2,Initialized,1697470414.0 +2,PodScheduled,1697470414.0 +2,StartImagePull,1697470420.490807 +2,EndImagePull,1697470425.785166 +2,Ready,1697470428.0 +2,ContainersReady,1697470428.0 diff --git a/eval/results/startup/coco-fw_warm.csv b/eval/results/startup/coco-fw_warm.csv index 645a4e8..b929ed2 100644 --- a/eval/results/startup/coco-fw_warm.csv +++ b/eval/results/startup/coco-fw_warm.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Initialized,1697220855.0 -0,PodScheduled,1697220855.0 -0,Start,1697220855.17601 -0,Ready,1697220869.0 -0,ContainersReady,1697220869.0 -1,Initialized,1697220875.0 -1,PodScheduled,1697220875.0 -1,Start,1697220875.2426844 -1,Ready,1697220935.0 -1,ContainersReady,1697220935.0 -2,Initialized,1697220940.0 -2,PodScheduled,1697220940.0 -2,Start,1697220940.4730253 -2,Ready,1697220955.0 -2,ContainersReady,1697220955.0 +0,Initialized,1697470310.0 +0,PodScheduled,1697470310.0 +0,Start,1697470310.3903742 +0,StartImagePull,1697470316.993853 +0,EndImagePull,1697470322.42001 +0,Ready,1697470325.0 +0,ContainersReady,1697470325.0 +1,Start,1697470331.4710257 +1,Initialized,1697470332.0 +1,PodScheduled,1697470332.0 +1,StartImagePull,1697470338.146335 +1,EndImagePull,1697470343.454944 +1,Ready,1697470346.0 +1,ContainersReady,1697470346.0 +2,Start,1697470351.548689 +2,Initialized,1697470352.0 +2,PodScheduled,1697470352.0 +2,StartImagePull,1697470358.193838 +2,EndImagePull,1697470363.794951 +2,Ready,1697470366.0 +2,ContainersReady,1697470366.0 diff --git a/eval/results/startup/coco_cold.csv b/eval/results/startup/coco_cold.csv index 3413ab3..d203d54 100644 --- a/eval/results/startup/coco_cold.csv +++ b/eval/results/startup/coco_cold.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697220756.8188488 -0,Initialized,1697220757.0 -0,PodScheduled,1697220757.0 -0,Ready,1697220770.0 -0,ContainersReady,1697220770.0 -1,Start,1697220776.8993905 -1,Initialized,1697220777.0 -1,PodScheduled,1697220777.0 -1,Ready,1697220791.0 -1,ContainersReady,1697220791.0 -2,Start,1697220796.9742057 -2,Initialized,1697220797.0 -2,PodScheduled,1697220797.0 -2,Ready,1697220811.0 -2,ContainersReady,1697220811.0 +0,Initialized,1697472420.0 +0,PodScheduled,1697472420.0 +0,Start,1697472420.4138386 +0,StartImagePull,1697472426.742439 +0,EndImagePull,1697472432.313472 +0,Ready,1697472435.0 +0,ContainersReady,1697472435.0 +1,Start,1697472441.492506 +1,Initialized,1697472442.0 +1,PodScheduled,1697472442.0 +1,StartImagePull,1697472447.854151 +1,EndImagePull,1697472453.553883 +1,Ready,1697472456.0 +1,ContainersReady,1697472456.0 +2,Start,1697472462.5733995 +2,Initialized,1697472463.0 +2,PodScheduled,1697472463.0 +2,StartImagePull,1697472469.0991 +2,EndImagePull,1697472474.478074 +2,Ready,1697472477.0 +2,ContainersReady,1697472477.0 diff --git a/eval/results/startup/coco_warm.csv b/eval/results/startup/coco_warm.csv index d0fe9cd..225f470 100644 --- a/eval/results/startup/coco_warm.csv +++ b/eval/results/startup/coco_warm.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697220694.552954 -0,Initialized,1697220695.0 -0,PodScheduled,1697220695.0 -0,Ready,1697220708.0 -0,ContainersReady,1697220708.0 -1,Start,1697220714.630086 -1,Initialized,1697220715.0 -1,PodScheduled,1697220715.0 -1,Ready,1697220729.0 -1,ContainersReady,1697220729.0 -2,Start,1697220735.7185054 -2,Initialized,1697220736.0 -2,PodScheduled,1697220736.0 -2,Ready,1697220750.0 -2,ContainersReady,1697220750.0 +0,Initialized,1697472358.0 +0,PodScheduled,1697472358.0 +0,Start,1697472358.171579 +0,StartImagePull,1697472364.526325 +0,EndImagePull,1697472370.138844 +0,Ready,1697472373.0 +0,ContainersReady,1697472373.0 +1,Initialized,1697472378.0 +1,PodScheduled,1697472378.0 +1,Start,1697472378.2509236 +1,StartImagePull,1697472384.621931 +1,EndImagePull,1697472390.294493 +1,Ready,1697472393.0 +1,ContainersReady,1697472393.0 +2,Start,1697472399.336547 +2,Initialized,1697472400.0 +2,PodScheduled,1697472400.0 +2,StartImagePull,1697472405.92968 +2,EndImagePull,1697472411.515635 +2,Ready,1697472414.0 +2,ContainersReady,1697472414.0 diff --git a/eval/results/startup/docker_cold.csv b/eval/results/startup/docker_cold.csv index dd0d133..904910e 100644 --- a/eval/results/startup/docker_cold.csv +++ b/eval/results/startup/docker_cold.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Initialized,1697220557.0 -0,PodScheduled,1697220557.0 -0,Start,1697220557.0543413 -0,Ready,1697220559.0 -0,ContainersReady,1697220559.0 -1,Initialized,1697220593.0 -1,PodScheduled,1697220593.0 -1,Start,1697220593.3947294 -1,Ready,1697220599.0 -1,ContainersReady,1697220599.0 -2,Start,1697220633.5336018 -2,Initialized,1697220634.0 -2,PodScheduled,1697220634.0 -2,Ready,1697220640.0 -2,ContainersReady,1697220640.0 +0,Start,1697476334.9183407 +0,Initialized,1697476335.0 +0,PodScheduled,1697476335.0 +0,StartImagePull,1697476335.985474 +0,EndImagePull,1697476338.508711 +0,Ready,1697476341.0 +0,ContainersReady,1697476341.0 +1,Initialized,1697476376.0 +1,PodScheduled,1697476376.0 +1,Start,1697476376.060256 +1,StartImagePull,1697476377.095043 +1,EndImagePull,1697476379.547167 +1,Ready,1697476382.0 +1,ContainersReady,1697476382.0 +2,Initialized,1697476417.0 +2,PodScheduled,1697476417.0 +2,Start,1697476417.1830556 +2,StartImagePull,1697476418.365998 +2,EndImagePull,1697476420.864521 +2,Ready,1697476423.0 +2,ContainersReady,1697476423.0 diff --git a/eval/results/startup/docker_warm.csv b/eval/results/startup/docker_warm.csv index ffa83fd..b91e441 100644 --- a/eval/results/startup/docker_warm.csv +++ b/eval/results/startup/docker_warm.csv @@ -1,16 +1,22 @@ Run,Event,TimeStampMs -0,Start,1697220448.6815393 -0,Initialized,1697220449.0 -0,PodScheduled,1697220449.0 -0,Ready,1697220450.0 -0,ContainersReady,1697220450.0 -1,Start,1697220484.8129041 -1,Initialized,1697220485.0 -1,PodScheduled,1697220485.0 -1,Ready,1697220486.0 -1,ContainersReady,1697220486.0 -2,Start,1697220520.9298782 -2,Initialized,1697220521.0 -2,PodScheduled,1697220521.0 -2,Ready,1697220522.0 -2,ContainersReady,1697220522.0 +0,StartImagePull,1697476186.999791 +0,EndImagePull,1697476189.388161 +0,Initialized,1697476226.0 +0,PodScheduled,1697476226.0 +0,Start,1697476226.3291554 +0,Ready,1697476228.0 +0,ContainersReady,1697476228.0 +1,StartImagePull,1697476186.999791 +1,EndImagePull,1697476189.388161 +1,Start,1697476262.4502141 +1,Initialized,1697476263.0 +1,PodScheduled,1697476263.0 +1,Ready,1697476264.0 +1,ContainersReady,1697476264.0 +2,StartImagePull,1697476186.999791 +2,EndImagePull,1697476189.388161 +2,Start,1697476298.58044 +2,Initialized,1697476299.0 +2,PodScheduled,1697476299.0 +2,Ready,1697476300.0 +2,ContainersReady,1697476300.0 diff --git a/tasks/eval/startup.py b/tasks/eval/startup.py index 2928af2..92357d5 100644 --- a/tasks/eval/startup.py +++ b/tasks/eval/startup.py @@ -2,6 +2,7 @@ from glob import glob from invoke import task from json import loads as json_loads +from matplotlib.patches import Patch from matplotlib.pyplot import subplots from os import makedirs from os.path import basename, exists, join @@ -16,6 +17,7 @@ PLOTS_DIR, ) from tasks.util.coco import guest_attestation, signature_verification +from tasks.util.containerd import get_time_pulling_image from tasks.util.k8s import template_k8s_file from tasks.util.kbs import clear_kbs_db, provision_launch_digest from tasks.util.kubeadm import get_pod_names_in_ns, run_kubectl_command @@ -128,6 +130,19 @@ def do_run(result_file, num_run, service_file, warmup=False): sleep(2) if not warmup: + # Work-out the time spent pulling container images + start_ts_service, end_ts_service = get_time_pulling_image("coco-helloworld-py") + start_ts_sidecar, end_ts_sidecar = get_time_pulling_image( + "coco-knative-sidecar" + ) + # We combine both image pulling times by taking the extreme-most + # timestamps. This is not necessarily correct (?) + start_ts = min(start_ts_service, start_ts_sidecar) + end_ts = max(end_ts_service, end_ts_sidecar) + events_ts.append(("StartImagePull", start_ts)) + events_ts.append(("EndImagePull", end_ts)) + + # Sort the events by timestamp and write them to a file events_ts = sorted(events_ts, key=lambda x: x[1]) for event in events_ts: write_csv_line(result_file, num_run, event[0], event[1]) @@ -196,6 +211,11 @@ def run(ctx, baseline=None): print("Executing baseline {} warmup run...".format(bline)) do_run(result_file, -1, service_file, warmup=True) + if flavour == "cold": + # `cold` happens after `warm`, so we want to clean-up after + # all the `warm` runs + cleanup_after_run(bline, used_images) + for nr in range(num_runs): print( "Executing baseline {} ({}) run {}/{}...".format( @@ -246,6 +266,8 @@ def plot(ctx): else: bar_width = (1 - space_between_baselines) / (num_flavours + 1) + pattern_for_flavour = {"warm": "//", "cold": "."} + color_for_event = {"pre-pull": "green", "image-pull": "blue", "post-pull": "orange"} for ind, flavour in enumerate(BASELINE_FLAVOURS): if num_flavours % 2 == 0: x_offset = bar_width * (ind + 1 / 2 - num_flavours / 2) @@ -253,23 +275,79 @@ def plot(ctx): x_offset = bar_width * (ind - num_flavours / 2) this_xs = [x + x_offset for x in xs] - print("bar width:", bar_width) - print("x offset:", x_offset) - print("xs:", this_xs) - ys = [] - for b in xlabels: - ys.append( - results_dict[b][flavour]["Ready"]["mean"] - - results_dict[b][flavour]["Start"]["mean"] - ) - ax.bar(this_xs, ys, label=flavour, width=bar_width) + ordered_events = { + "pre-pull": ("Start", "StartImagePull"), + "image-pull": ("StartImagePull", "EndImagePull"), + "post-pull": ("EndImagePull", "Ready"), + } + stacked_ys = [] + for ev in ordered_events: + start_ev = ordered_events[ev][0] + end_ev = ordered_events[ev][1] + ys = [] + for b in xlabels: + # Given the way we parse the containerd logs, the Start/End + # ImagePull events for baselines that cache docker images + # (in their 'warm' flavour) are going to refer to ImagePull + # events in the past, and botch any timestamp differences. + # Thus, we need to make a special-case for them here + if b == "docker" and flavour == "warm": + # For docker warm starts, we attribute all time spent in + # a post-pull stage + if ev == "post-pull": + event_duration = ( + results_dict[b][flavour]["Ready"]["mean"] + - results_dict[b][flavour]["Start"]["mean"] + ) + else: + event_duration = 0 + else: + event_duration = ( + results_dict[b][flavour][end_ev]["mean"] + - results_dict[b][flavour][start_ev]["mean"] + ) + + assert event_duration >= 0 + ys.append(event_duration) + + stacked_ys.append(ys) + + for num, ys in enumerate(stacked_ys): + label = list(ordered_events.keys())[num] + if num == 0: + ax.bar( + this_xs, + ys, + width=bar_width, + color=color_for_event[label], + edgecolor="black", + hatch=pattern_for_flavour[flavour], + ) + acc_ys = ys + else: + ax.bar( + this_xs, + ys, + bottom=acc_ys, # stacked_ys[num - 1], + width=bar_width, + color=color_for_event[label], + edgecolor="black", + hatch=pattern_for_flavour[flavour], + ) + for i in range(len(acc_ys)): + acc_ys[i] += ys[i] # Misc ax.set_xticks(xs, xlabels) ax.set_xlabel("Baseline") ax.set_ylabel("Time [s]") - ax.set_title("End-to-end latency to start a pod") - ax.legend() + ax.set_title("End-to-end latency to start a pod\n(cold start='/' - warm start='.')") + + # Manually craft the legend + legend_handles = [] + for ev in color_for_event: + legend_handles.append(Patch(color=color_for_event[ev], label=ev)) + ax.legend(handles=legend_handles) for plot_format in ["pdf", "png"]: plot_file = join(plots_dir, "startup.{}".format(plot_format)) diff --git a/tasks/util/containerd.py b/tasks/util/containerd.py new file mode 100644 index 0000000..a45219e --- /dev/null +++ b/tasks/util/containerd.py @@ -0,0 +1,29 @@ +from json import loads as json_loads +from subprocess import run + + +def get_time_pulling_image(image_name): + """ + Retrieve the start and end timestamp (in epoch floating seconds) for the + PullImage event from containerd + """ + journalctl_cmd = 'sudo journalctl -xeu containerd --since "10 min ago" -o json' + out = ( + run(journalctl_cmd, shell=True, capture_output=True) + .stdout.decode("utf-8") + .strip() + .split("\n") + ) + pull_image_json = [] + for o in out: + o_json = json_loads(o) + image_name = "coco-helloworld-py" + if "PullImage" in o_json["MESSAGE"] and image_name in o_json["MESSAGE"]: + pull_image_json.append(o_json) + + assert len(pull_image_json) >= 2 + + start_ts = int(pull_image_json[-2]["__REALTIME_TIMESTAMP"]) / 1e6 + end_ts = int(pull_image_json[-1]["__REALTIME_TIMESTAMP"]) / 1e6 + + return start_ts, end_ts