test/system: Decouple image caching from Zuul

Since the rewrite of the system test suite[0] we've relied on the Zuul
playbooks for taking care of caching images using Skopeo for increasing
the reliability of the tests (in the past the instability of the Fedora
registry caused problems). This state is problematic if we want to use
the tests in other environments than the Zuul CI. This moves the caching
from Zuul into the system tests.

Currently, Bats does not support officially suite-wide setup and
teardown functions. The solution I chose was to add two new test files
that are executed before and after all tests. This may complicate the
execution of cherry-picked tests but that is not a very common use case
anyway.

The tests are now to some extent capable of adjusting to the host
environment. This is meant in the sense of: I'm running on RHEL, the
"default image" is UBI; I'm running on Fedora, the "default image" is
fedora-toolbox. This mechanism relies on os-release, which is the same
as what Toolbox itself uses.

Along the road I chose to update the f29/fedora-toolbox:29 image to the
fedora-toolbox:32 image because I did not want to maintain two versions
of images (the name changed some time ago[1]). I also introduced or
modified a number of helper functions to make better use of the caching
and default system recognition.

[0] #517
[1] #615

playbooks/setup-env.yaml

@@ -46,30 +46,3 @@
 
     - name: Show podman debug information
       command: podman info --debug
-
-    - name: Pull the default image from registry.fedoraproject.org
-      command: "podman pull registry.fedoraproject.org/fedora-toolbox:{{ ansible_distribution_version }}"
-      register: _podman
-      until: _podman.rc == 0
-      retries: 5
-      delay: 10
-
-    - name: Copy the default image to a directory
-      command:
-        cmd: "skopeo copy containers-storage:registry.fedoraproject.org/fedora-toolbox:{{ ansible_distribution_version }} dir:{{ zuul.project.src_dir }}/fedora-toolbox-{{ ansible_distribution_version }}"
-        creates: "{{ zuul.project.src_dir }}/fedora-toolbox-{{ ansible_distribution_version }}/manifest.json"
-
-    - name: Pull registry.fedoraproject.org/f29/fedora-toolbox:29
-      command: podman pull registry.fedoraproject.org/f29/fedora-toolbox:29
-      register: _podman
-      until: _podman.rc == 0
-      retries: 5
-      delay: 10
-
-    - name: Copy registry.fedoraproject.org/f29/fedora-toolbox:29 to a directory
-      command:
-        cmd: "skopeo copy containers-storage:registry.fedoraproject.org/f29/fedora-toolbox:29 dir:{{ zuul.project.src_dir }}/fedora-toolbox-29"
-        creates: "{{ zuul.project.src_dir }}/fedora-toolbox-29/manifest.json"
-
-    - name: Clean up the local containers storage
-      command: podman system reset --force

test/system/000-setup.bats

@@ -0,0 +1,11 @@
+#!/usr/bin/env bats
+
+load 'libs/helpers'
+
+@test "test suite: Setup" {
+    # Cache the default image for the system
+    _pull_and_cache_distro_image $(get_system_id) $(get_system_version) || die
+    # Cache all images that will be needed during the tests
+    _pull_and_cache_distro_image fedora 32 || die
+    _pull_and_cache_distro_image busybox || die
+}

test/system/101-create.bats

@@ -22,15 +22,17 @@ teardown() {
 }
 
 @test "create: Create a container with a valid custom name ('custom-containerName')" {
+  pull_default_image
+
   run $TOOLBOX -y create -c "custom-containerName"
 
   assert_success
 }
 
-@test "create: Create a container with a custom image and name ('fedora29'; f29)" {
-  pull_image_old 29
+@test "create: Create a container with a custom image and name ('fedora32'; f32)" {
+  pull_distro_image fedora 32
 
-  run $TOOLBOX -y create -c "fedora29" -i fedora-toolbox:29
+  run $TOOLBOX -y create -c "fedora32" -i fedora-toolbox:32
 
   assert_success
 }
@@ -53,17 +55,17 @@ teardown() {
   assert_line --index 2 "Run 'toolbox --help' for usage."
 }
 
-@test "create: Create a container with a distro and release options ('fedora'; f29)" {
-  pull_image 29
+@test "create: Create a container with a distro and release options ('fedora'; f32)" {
+  pull_distro_image fedora 32
 
-  run $TOOLBOX -y create -d "fedora" -r f29
+  run $TOOLBOX -y create -d "fedora" -r f32
 
   assert_success
-  assert_output --partial "Created container: fedora-toolbox-29"
-  assert_output --partial "Enter with: toolbox enter --release 29"
+  assert_output --partial "Created container: fedora-toolbox-32"
+  assert_output --partial "Enter with: toolbox enter --release 32"
 
   # Make sure the container has actually been created
   run podman ps -a
 
-  assert_output --regexp "Created[[:blank:]]+fedora-toolbox-29"
+  assert_output --regexp "Created[[:blank:]]+fedora-toolbox-32"
 }

test/system/102-list.bats

@@ -35,7 +35,7 @@ teardown() {
 }
 
 @test "list: Run 'list' with zero toolbox's containers and images, but other image (the list should be empty)" {
-  get_busybox_image
+  pull_distro_image busybox
 
   run podman images
 
@@ -50,8 +50,9 @@ teardown() {
 @test "list: Try to list images and containers (no flag) with 3 containers and 2 images (the list should have 3 images and 2 containers)" {
   # Pull the two images
   pull_default_image
-  pull_image_old 29
-  # Create tree containers
+  pull_distro_image fedora 32
+
+  # Create three containers
   create_default_container
   create_container non-default-one
   create_container non-default-two
@@ -60,24 +61,24 @@ teardown() {
   run $TOOLBOX list --images
 
   assert_success
-  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
-  assert_output --partial "fedora-toolbox:29"
+  assert_output --partial "$(get_system_id)-toolbox:$(get_system_version)"
+  assert_output --partial "fedora-toolbox:32"
 
   # Check containers
   run $TOOLBOX list --containers
 
   assert_success
-  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox-$(get_system_version)"
   assert_output --partial "non-default-one"
   assert_output --partial "non-default-two"
 
   # Check all together
   run $TOOLBOX list
 
   assert_success
-  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
-  assert_output --partial "fedora-toolbox:29"
-  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "$(get_system_id)-toolbox:$(get_system_version)"
+  assert_output --partial "fedora-toolbox:32"
+  assert_output --partial "$(get_system_id)-toolbox-$(get_system_version)"
   assert_output --partial "non-default-one"
   assert_output --partial "non-default-two"
 }

test/system/999-teardown.bats

@@ -0,0 +1,7 @@
+#!/usr/bin/env bats
+
+load 'libs/helpers'
+
+@test "test suite: Teardown" {
+  _clean_cached_images
+}