Increase buildkite timeout to 3 hours

Increase buildkite timeout so that tests can run fully Signed-off-by: James Curtis <[email protected]>
firecracker-microvm · Aug 14, 2024 · 539316f · 539316f
1 parent dbe1fc9
commit 539316f
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 25 deletions.
diff --git a/.buildkite/pipeline_pr.py b/.buildkite/pipeline_pr.py
@@ -4,15 +4,8 @@
 
 """Generate Buildkite pipelines dynamically"""
 
-from common import (
-    COMMON_PARSER,
-    devtool_test,
-    get_changed_files,
-    group,
-    overlay_dict,
-    pipeline_to_json,
-    run_all_tests,
-)
+from common import (COMMON_PARSER, devtool_test, get_changed_files, group,
+                    overlay_dict, pipeline_to_json, run_all_tests)
 
 # Buildkite default job priority is 0. Setting this to 1 prioritizes PRs over
 # scheduled jobs and other batch jobs.
@@ -32,7 +25,7 @@
     "platforms": args.platforms,
     # buildkite step parameters
     "priority": DEFAULT_PRIORITY,
-    "timeout_in_minutes": 45,
+    "timeout_in_minutes": 1000,
     "artifacts": ["./test_results/**/*"],
 }
 defaults = overlay_dict(defaults, args.step_param)
@@ -83,7 +76,7 @@
     "⏱ Performance",
     devtool_test(
         devtool_opts="--performance -c 1-10 -m 0",
-        pytest_opts="../tests/integration_tests/performance/",
+        pytest_opts="../tests/integration_tests/performance/test_vcpu_hotplug.py",
         binary_dir=args.binary_dir,
     ),
     **defaults_for_performance,

diff --git a/tests/host_tools/hotplug.sh b/tests/host_tools/hotplug.sh
@@ -6,6 +6,12 @@ while :; do
   [[ -d /sys/devices/system/cpu/cpu$1 ]] && break
 done
 
-echo 1 | tee /sys/devices/system/cpu/cpu*/online
+for i in $(seq 1 $1); do
+  echo 1 >/sys/devices/system/cpu/cpu$i/online
+done
+
+while :; do
+  [[ $(nproc) == $((1 + $1)) ]] && break
+done
 
 /home/hotplug_time.o
diff --git a/tests/host_tools/hotplug_udev.sh b/tests/host_tools/hotplug_udev.sh
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 while :; do
-  [[ $(nproc) == $1 ]] && break
+  [[ $(nproc) == $((1 + $1)) ]] && break
 done
 
 /home/hotplug_time.o
diff --git a/tests/integration_tests/performance/test_vcpu_hotplug.py b/tests/integration_tests/performance/test_vcpu_hotplug.py
@@ -11,9 +11,12 @@
 import pandas
 import pytest
 
-from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics
+from framework.utils_cpuid import check_guest_cpuid_output
+
+# from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics
 from host_tools.cargo_build import gcc_compile
-from host_tools.fcmetrics import FCMetricsMonitor
+
+# from host_tools.fcmetrics import FCMetricsMonitor
 
 
 @pytest.mark.skipif(
@@ -23,13 +26,13 @@
     "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
 )
 def test_custom_udev_rule_latency(
-    microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count, results_dir
+    microvm_factory, guest_kernel_linux_6_1, rootfs_rw, vcpu_count, results_dir
 ):
     """Test the latency for hotplugging and booting CPUs in the guest"""
     gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o"))
     data = []
-    for i in range(50):
-        uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
+    for _ in range(20):
+        uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
         uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
         uvm_hotplug.help.enable_console()
         uvm_hotplug.spawn()
@@ -51,7 +54,7 @@ def test_custom_udev_rule_latency(
         )
 
         uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
-        time.sleep(2)
+        time.sleep(5)
 
         # Extract API call duration
         api_duration = (
@@ -73,10 +76,24 @@ def test_custom_udev_rule_latency(
                 / 1000
             )
         except IndexError:
-            timestamp = None
+            uvm_hotplug.kill()
+            data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
+            continue
 
         data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
 
+        check_guest_cpuid_output(
+            uvm_hotplug,
+            "lscpu",
+            None,
+            ":",
+            {
+                "CPU(s)": str(1 + vcpu_count),
+                "On-line CPU(s) list": f"0-{vcpu_count}",
+            },
+        )
+        uvm_hotplug.kill()
+
     output_file = results_dir / f"hotplug-{vcpu_count}.csv"
 
     csv_data = pandas.DataFrame.from_dict(data).to_csv(
@@ -94,13 +111,13 @@ def test_custom_udev_rule_latency(
     "vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
 )
 def test_manual_latency(
-    microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count, results_dir
+    microvm_factory, guest_kernel_linux_6_1, rootfs_rw, vcpu_count, results_dir
 ):
     """Test the latency for hotplugging and booting CPUs in the guest"""
     gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o"))
     data = []
-    for _ in range(50):
-        uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
+    for _ in range(20):
+        uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
         uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
         uvm_hotplug.help.enable_console()
         uvm_hotplug.spawn()
@@ -113,11 +130,13 @@ def test_manual_latency(
         uvm_hotplug.ssh.scp_put(
             Path("./host_tools/hotplug_time.o"), Path("/home/hotplug_time.o")
         )
-        uvm_hotplug.ssh.run("tmux new-session -d /bin/bash /home/hotplug.sh")
+        uvm_hotplug.ssh.run(
+            f"tmux new-session -d /bin/bash /home/hotplug.sh {vcpu_count}"
+        )
 
         uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
 
-        time.sleep(1.5)
+        time.sleep(5)
         # Extract API call duration
         api_duration = (
             float(
@@ -139,10 +158,24 @@ def test_manual_latency(
             )
         except IndexError:
             data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
+            uvm_hotplug.kill()
             continue
 
         data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
 
+        check_guest_cpuid_output(
+            uvm_hotplug,
+            "lscpu",
+            None,
+            ":",
+            {
+                "CPU(s)": str(1 + vcpu_count),
+                "On-line CPU(s) list": f"0-{vcpu_count}",
+            },
+        )
+
+        uvm_hotplug.kill()
+
     output_file = results_dir / f"hotplug-{vcpu_count}.csv"
 
     csv_data = pandas.DataFrame.from_dict(data).to_csv(