From b361a42e659319d8952a78a99a742f10d1192a70 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 6 Dec 2022 14:44:25 -0500 Subject: [PATCH 1/2] Cirrus: Remove escape codes from log files Signed-off-by: Chris Evich --- contrib/cirrus/prebuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cirrus/prebuild.sh b/contrib/cirrus/prebuild.sh index 0e43f19462..d4306a17ac 100755 --- a/contrib/cirrus/prebuild.sh +++ b/contrib/cirrus/prebuild.sh @@ -33,7 +33,7 @@ showrun $SCRIPT_BASE/cirrus_yaml_test.py if [[ "${DISTRO_NV}" =~ fedora ]]; then msg "Checking shell scripts" showrun ooe.sh dnf install -y ShellCheck # small/quick addition - showrun shellcheck --color=always --format=tty \ + showrun shellcheck --format=tty \ --shell=bash --external-sources \ --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \ --exclude SC2046,SC2034,SC2090,SC2064 \ From 7c6873b23d2bf66de6f8587d2f41319e16340b6e Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 8 Nov 2022 14:00:00 -0500 Subject: [PATCH 2/2] Cirrus: Collect benchmarks on machine instances The hardware used for podman-machine testing is fairly stable/predictable because it's bare-metal. This is a nearly ideal environment for collection of benchmarking data. Arrange for that to happen, and the resulting data to be collected. Also keep track of the benchmark-basis details in a machine-readable `env` file along side the raw and parsed benchmarks. Signed-off-by: Chris Evich --- .cirrus.yml | 13 +++++++++++-- Makefile | 4 ++-- contrib/cirrus/lib.sh | 31 ++++++++++++++++++++++++++++++- contrib/cirrus/runner.sh | 3 +++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index fe70da4201..108bbcdf83 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -666,7 +666,10 @@ podman_machine_task: clone_script: *get_gosrc setup_script: *setup main_script: *main - always: *int_logs_artifacts + always: &machine_logs_benchmarks + <<: *int_logs_artifacts + benchmark_artifacts: + path: ./data/* podman_machine_aarch64_task: @@ -692,7 +695,7 @@ podman_machine_aarch64_task: clone_script: *get_gosrc_aarch64 setup_script: *setup main_script: *main - always: *int_logs_artifacts + always: *machine_logs_benchmarks # Always run subsequent to integration tests. While parallelism is lost @@ -1087,6 +1090,12 @@ artifacts_task: - tar xjf repo.tbz - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/ - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/ + benchmarks_script: + - mkdir -p /tmp/benchmarks + - cd /tmp/benchmarks + - $ARTCURL/podman_machine/benchmark.zip + - unzip benchmark.zip + - mv ./data/benchmarks.{env,csv} $CIRRUS_WORKING_DIR/ always: contents_script: ls -la $CIRRUS_WORKING_DIR # Produce downloadable files and an automatic zip-file accessible diff --git a/Makefile b/Makefile index d06ecbd361..5bfeea06e4 100644 --- a/Makefile +++ b/Makefile @@ -577,11 +577,11 @@ localintegration: test-binaries ginkgo remoteintegration: test-binaries ginkgo-remote .PHONY: localmachine -localmachine: test-binaries +localmachine: test-binaries .install.ginkgo $(MAKE) ginkgo-run GINKGONODES=1 GINKGOWHAT=pkg/machine/e2e/. HACK= .PHONY: localbenchmarks -localbenchmarks: test-binaries +localbenchmarks: install.tools test-binaries PATH=$(PATH):$(shell pwd)/hack ACK_GINKGO_RC=true $(GINKGO) \ -focus "Podman Benchmark Suite" \ -tags "$(BUILDTAGS) benchmarks" -noColor \ diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 7a5364886b..8d7e384315 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -93,7 +93,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA" # contexts, such as host->container or root->rootless user # # List of envariables which must be EXACT matches -PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS' +PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC|NETWORK_BACKEND|OCI_RUNTIME|ROOTLESS_USER|SCRIPT_BASE|SKIP_USERNS|EC2_INST_TYPE' # List of envariable patterns which must match AT THE BEGINNING of the name. PASSTHROUGH_ENV_ATSTART='CI|TEST' @@ -290,3 +290,32 @@ remove_packaged_podman_files() { # Be super extra sure and careful vs performant and completely safe sync && echo 3 > /proc/sys/vm/drop_caches || true } + +# Execute make localbenchmarks in $CIRRUS_WORKING_DIR/data +# for preserving as a task artifact. +localbenchmarks() { + local datadir + req_env_vars DISTRO_NV PODBIN_NAME PRIV_NAME TEST_ENVIRON TEST_FLAVOR + req_env_vars VM_IMAGE_NAME EC2_INST_TYPE + + datadir=$CIRRUS_WORKING_DIR/data + mkdir -p $datadir + + ( + echo "# Env. var basis for benchmarks benchmarks." + printenv | grep -Ev "$SECRET_ENV_RE" | sort + + echo "# Machine details for data-comparison sake, not actual env. vars." + # Checked above in req_env_vars + # shellcheck disable=SC2154 + echo "\ +CPUTOTAL=$(grep -ce '^processor' /proc/cpuinfo) +INST_TYPE=$EC2_INST_TYPE # one day may include other cloud's VM types. +MEMTOTAL=$(awk -F: '$1 == "MemTotal" { print $2 }' $datadir/benchmarks.env + make localbenchmarks | tee $datadir/benchmarks.raw + msg "Processing raw benchmarks output" + hack/parse-localbenchmarks < $datadir/benchmarks.raw | tee $datadir/benchmarks.csv +} diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index c57f14b4c5..9160966b32 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -394,6 +394,9 @@ dotest() { } _run_machine() { + # This environment is convenient for executing some benchmarking + localbenchmarks + # N/B: Can't use _bail_if_test_can_be_skipped here b/c content isn't under test/ make localmachine |& logformatter }