From 33891e8e967ab1b677dbf074a1b6a0fb371534e2 Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Thu, 20 Jul 2023 15:01:13 -0600
Subject: [PATCH] System tests: add test tags

BATS 1.8.0 introduces tags: metadata that can be applied to
a single test or one entire file, then used for filtering
in a test run.

Issue #19299 introduces the possibility of using OpenQA
for podman reverse dependency testing: continuous CI on
all packages that can affect podman, so we don't go two
months with no bodhi builds then get caught by surprise
when systemd or kernel or crun change in ways that break us.

This PR introduces one bats tag, "distro-integration".
The intention is for OpenQA (or other) tests to install
the podman-tests package and run:

    bats --filter-tags distro-integration /usr/share/podman/test/system

Goal is to keep the test list short and sweet: we do not
need to test command-line option parsing. We *DO* need to
test interactions with systemd, kernel, nethack, and other
critical components.

Signed-off-by: Ed Santiago <santiago@redhat.com>
---
 hack/bats                                                | 4 ++++
 test/system/001-basic.bats                               | 3 +++
 test/system/030-run.bats                                 | 3 +++
 test/system/032-sig-proxy.bats                           | 1 +
 test/system/035-logs.bats                                | 2 ++
 test/system/070-build.bats                               | 2 ++
 test/system/075-exec.bats                                | 2 ++
 test/system/080-pause.bats                               | 2 ++
 test/system/090-events.bats                              | 3 +++
 test/system/130-kill.bats                                | 1 +
 test/system/160-volumes.bats                             | 1 +
 test/system/170-run-userns.bats                          | 2 ++
 test/system/180-blkio.bats                               | 2 ++
 test/system/190-run-ipcns.bats                           | 2 ++
 .../{195-run-namesapces.bats => 195-run-namespaces.bats} | 1 +
 test/system/260-sdnotify.bats                            | 3 +++
 test/system/410-selinux.bats                             | 9 +++++++++
 test/system/500-networking.bats                          | 1 +
 18 files changed, 44 insertions(+)
 rename test/system/{195-run-namesapces.bats => 195-run-namespaces.bats} (97%)

diff --git a/hack/bats b/hack/bats
index fb99181f52..ba26e0444a 100755
--- a/hack/bats
+++ b/hack/bats
@@ -23,6 +23,9 @@ $0 is a wrapper for invoking podman system tests.
                     runs only subtests within FILENAME-PATTERH whose names
                     match that string.
 
+   --tag=TAG      Passed on to bats as '--filter-tags TAG'
+                  As of 2023-07-26 the only tag used is 'distro-integration'
+
    -T             Passed on to bats, which will then show timing data
 
    --help         display usage message
@@ -83,6 +86,7 @@ for i;do
         --rootless) TEST_ROOT= ;;
         --remote)   REMOTE=remote ;;
         --ts|-T)    bats_opts+=("-T") ;;
+        --tag=*)    bats_filter=("--filter-tags" "$value") ;;
         */*.bats)   TESTS=$i ;;
         *)
             if [[ $i =~ : ]]; then
diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats
index b6a88495c1..5109bd20a9 100644
--- a/test/system/001-basic.bats
+++ b/test/system/001-basic.bats
@@ -13,6 +13,7 @@ function setup() {
 
 #### DO NOT ADD ANY TESTS HERE! ADD NEW TESTS AT BOTTOM!
 
+# bats test_tags=distro-integration
 @test "podman version emits reasonable output" {
     run_podman version
 
@@ -38,6 +39,7 @@ function setup() {
     is "$output" ".*The --config flag is ignored by Podman. Exists for Docker compatibility\+"		  "verify warning for --config option"
 }
 
+# bats test_tags=distro-integration
 @test "podman info" {
     # These will be displayed on the test output stream, offering an
     # at-a-glance overview of important system configuration details
@@ -71,6 +73,7 @@ function setup() {
        "--context=swarm should fail"
 }
 
+# bats test_tags=distro-integration
 @test "podman can pull an image" {
     run_podman rmi -a -f
 
diff --git a/test/system/030-run.bats b/test/system/030-run.bats
index 2619137074..681f00a4c6 100644
--- a/test/system/030-run.bats
+++ b/test/system/030-run.bats
@@ -3,6 +3,7 @@
 load helpers
 load helpers.network
 
+# bats test_tags=distro-integration
 @test "podman run - basic tests" {
     rand=$(random_string 30)
 
@@ -296,6 +297,7 @@ echo $rand        |   0 | $rand
 }
 
 # #6829 : add username to /etc/passwd inside container if --userns=keep-id
+# bats test_tags=distro-integration
 @test "podman run : add username to /etc/passwd if --userns=keep-id" {
     skip_if_not_rootless "--userns=keep-id only works in rootless mode"
     # Default: always run as root
@@ -695,6 +697,7 @@ json-file | f
 # https://github.com/containers/podman/issues/9096
 # podman exec may truncate stdout/stderr; actually a bug in conmon:
 # https://github.com/containers/conmon/issues/236
+# bats test_tags=distro-integration
 @test "podman run - does not truncate or hang with big output" {
     # Size, in bytes, to dd and to expect in return
     char_count=700000
diff --git a/test/system/032-sig-proxy.bats b/test/system/032-sig-proxy.bats
index fca2d44d17..d34645c7a2 100644
--- a/test/system/032-sig-proxy.bats
+++ b/test/system/032-sig-proxy.bats
@@ -5,6 +5,7 @@ load helpers.sig-proxy
 
 # Each of the tests below does some setup, then invokes the helper from helpers.sig-proxy.bash.
 
+# bats test_tags=distro-integration
 @test "podman sigproxy test: run" {
     # We're forced to use $PODMAN because run_podman cannot be backgrounded
     $PODMAN run -i --name c_run $IMAGE sh -c "$SLEEPLOOP" &
diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats
index aa33d5cc04..b401d0339a 100644
--- a/test/system/035-logs.bats
+++ b/test/system/035-logs.bats
@@ -337,6 +337,7 @@ timeout: sending signal TERM to command.*" "logs --since -f on running container
     _log_test_follow_since k8s-file
 }
 
+# bats test_tags=distro-integration
 @test "podman logs - --since --follow journald" {
     # We can't use journald on RHEL as rootless: rhbz#1895105
     skip_if_journald_unavailable
@@ -379,6 +380,7 @@ $content--2.*" "logs --until -f on running container works"
     _log_test_follow_until k8s-file
 }
 
+# bats test_tags=distro-integration
 @test "podman logs - --until --follow journald" {
     # We can't use journald on RHEL as rootless: rhbz#1895105
     skip_if_journald_unavailable
diff --git a/test/system/070-build.bats b/test/system/070-build.bats
index b3571105fa..76e56023a5 100644
--- a/test/system/070-build.bats
+++ b/test/system/070-build.bats
@@ -6,6 +6,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "podman build - basic test" {
     rand_filename=$(random_string 20)
     rand_content=$(random_string 50)
@@ -267,6 +268,7 @@ EOF
 }
 
 
+# bats test_tags=distro-integration
 @test "podman build - workdir, cmd, env, label" {
     tmpdir=$PODMAN_TMPDIR/build-test
     mkdir -p $tmpdir
diff --git a/test/system/075-exec.bats b/test/system/075-exec.bats
index 6c7126fb7a..93f034f8f1 100644
--- a/test/system/075-exec.bats
+++ b/test/system/075-exec.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "podman exec - basic test" {
     rand_filename=$(random_string 20)
     rand_content=$(random_string 50)
@@ -43,6 +44,7 @@ load helpers
     run_podman rm $cid
 }
 
+# bats test_tags=distro-integration
 @test "podman exec - leak check" {
     skip_if_remote "test is meaningless over remote"
 
diff --git a/test/system/080-pause.bats b/test/system/080-pause.bats
index deaf8d8318..f9e632c42c 100644
--- a/test/system/080-pause.bats
+++ b/test/system/080-pause.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "podman pause/unpause" {
     if is_rootless && ! is_cgroupsv2; then
         skip "'podman pause' (rootless) only works with cgroups v2"
@@ -58,6 +59,7 @@ load helpers
     run_podman 125 unpause $cname
 }
 
+# bats test_tags=distro-integration
 @test "podman unpause --all" {
     if is_rootless && ! is_cgroupsv2; then
         skip "'podman pause' (rootless) only works with cgroups v2"
diff --git a/test/system/090-events.bats b/test/system/090-events.bats
index 7577d2d54f..18593e17ba 100644
--- a/test/system/090-events.bats
+++ b/test/system/090-events.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "events with a filter by label" {
     cname=test-$(random_string 30 | tr A-Z a-z)
     labelname=$(random_string 10)
@@ -141,6 +142,7 @@ function _events_disjunctive_filters() {
     _events_disjunctive_filters ""
 }
 
+# bats test_tags=distro-integration
 @test "events with events_logfile_path in containers.conf" {
     skip_if_remote "remote does not support --events-backend"
     events_file=$PODMAN_TMPDIR/events.log
@@ -162,6 +164,7 @@ function _populate_events_file() {
     done
 }
 
+# bats test_tags=distro-integration
 @test "events log-file rotation" {
     skip_if_remote "setting CONTAINERS_CONF_OVERRIDE logger options does not affect remote client"
 
diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats
index e4ec105d78..ad06bbade3 100644
--- a/test/system/130-kill.bats
+++ b/test/system/130-kill.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "podman kill - test signal handling in containers" {
 
     # Prepare for 'logs -f'
diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats
index 369e95447b..650ab5a5d7 100644
--- a/test/system/160-volumes.bats
+++ b/test/system/160-volumes.bats
@@ -395,6 +395,7 @@ EOF
     is "$output"  "" "no more volumes to prune"
 }
 
+# bats test_tags=distro-integration
 @test "podman volume type=bind" {
     myvoldir=${PODMAN_TMPDIR}/volume_$(random_string)
     mkdir $myvoldir
diff --git a/test/system/170-run-userns.bats b/test/system/170-run-userns.bats
index ca53f8fb6e..b1f18cd073 100644
--- a/test/system/170-run-userns.bats
+++ b/test/system/170-run-userns.bats
@@ -3,6 +3,8 @@
 #
 # Tests for podman build
 #
+# bats file_tags=distro-integration
+#
 
 load helpers
 
diff --git a/test/system/180-blkio.bats b/test/system/180-blkio.bats
index 7b7fb0201a..a156773943 100644
--- a/test/system/180-blkio.bats
+++ b/test/system/180-blkio.bats
@@ -2,6 +2,8 @@
 #
 # podman blkio-related tests
 #
+# bats file_tags=distro-integration
+#
 
 load helpers
 
diff --git a/test/system/190-run-ipcns.bats b/test/system/190-run-ipcns.bats
index a764f9a7b4..b20871abb3 100644
--- a/test/system/190-run-ipcns.bats
+++ b/test/system/190-run-ipcns.bats
@@ -3,6 +3,8 @@
 #
 # Tests for podman build
 #
+# bats file_tags=distro-integration
+#
 
 load helpers
 
diff --git a/test/system/195-run-namesapces.bats b/test/system/195-run-namespaces.bats
similarity index 97%
rename from test/system/195-run-namesapces.bats
rename to test/system/195-run-namespaces.bats
index 17c30cc818..2bc67475c4 100644
--- a/test/system/195-run-namesapces.bats
+++ b/test/system/195-run-namespaces.bats
@@ -5,6 +5,7 @@
 
 load helpers
 
+# bats test_tags=distro-integration
 @test "podman test all namespaces" {
     # format is nsname | option name
     tests="
diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats
index ae1f828afb..97041a9446 100644
--- a/test/system/260-sdnotify.bats
+++ b/test/system/260-sdnotify.bats
@@ -103,6 +103,7 @@ function _assert_mainpid_is_conmon() {
     _stop_socat
 }
 
+# bats test_tags=distro-integration
 @test "sdnotify : conmon" {
     export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
     _start_socat
@@ -141,6 +142,7 @@ READY=1" "sdnotify sent MAINPID and READY"
 
 # These tests can fail in dev. environment because of SELinux.
 # quick fix: chcon -t container_runtime_exec_t ./bin/podman
+# bats test_tags=distro-integration
 @test "sdnotify : container" {
     _prefetch $SYSTEMD_IMAGE
 
@@ -452,6 +454,7 @@ spec:
 " > $fname
 }
 
+# bats test_tags=distro-integration
 @test "podman kube play - exit-code propagation" {
     fname=$PODMAN_TMPDIR/$(random_string).yaml
 
diff --git a/test/system/410-selinux.bats b/test/system/410-selinux.bats
index d104c4aaf3..4b60f0c0cf 100644
--- a/test/system/410-selinux.bats
+++ b/test/system/410-selinux.bats
@@ -34,26 +34,32 @@ function check_label() {
 }
 
 
+# bats test_tags=distro-integration
 @test "podman selinux: confined container" {
     check_label "" "container_t"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=distro-integration
 @test "podman selinux: container with label=disable" {
     check_label "--security-opt label=disable" "spc_t"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=distro-integration
 @test "podman selinux: privileged container" {
     check_label "--privileged --userns=host" "spc_t"
 }
 
+# bats test_tags=distro-integration
 @test "podman selinux: init container" {
     check_label "--systemd=always" "container_init_t"
 }
 
+# bats test_tags=distro-integration
 @test "podman selinux: init container with --security-opt type" {
     check_label "--systemd=always --security-opt=label=type:spc_t" "spc_t"
 }
 
+# bats test_tags=distro-integration
 @test "podman selinux: init container with --security-opt level&type" {
     check_label "--systemd=always --security-opt=label=level:s0:c1,c2 --security-opt=label=type:spc_t" "spc_t" "s0:c1,c2"
 }
@@ -62,6 +68,7 @@ function check_label() {
     check_label "--systemd=always --security-opt=label=level:s0:c1,c2" "container_init_t"  "s0:c1,c2"
 }
 
+# FIXME #19376 - container-selinux broken -- bats test_tags=distro-integration
 @test "podman selinux: pid=host" {
     # FIXME this test fails when run rootless with runc:
     #   Error: container_linux.go:367: starting container process caused: process_linux.go:495: container init caused: readonly path /proc/asound: operation not permitted: OCI permission denied
@@ -153,6 +160,7 @@ function check_label() {
 }
 
 # pr #7902 - containers in pods should all run under same context
+# bats test_tags=distro-integration
 @test "podman selinux: containers in pods share full context" {
     skip_if_no_selinux
 
@@ -226,6 +234,7 @@ function check_label() {
     is "$output" "Error.*: $expect" "podman emits useful diagnostic on failure"
 }
 
+# bats test_tags=distro-integration
 @test "podman selinux: check relabel" {
     skip_if_no_selinux
 
diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats
index 5caea14fe6..fdca1af440 100644
--- a/test/system/500-networking.bats
+++ b/test/system/500-networking.bats
@@ -449,6 +449,7 @@ load helpers.network
 }
 
 # Test for https://github.com/containers/podman/issues/10052
+# bats test_tags=distro-integration
 @test "podman network connect/disconnect with port forwarding" {
     random_1=$(random_string 30)
     HOST_PORT=$(random_free_port)