From bf107bbfe58443c92f97917b7c4db75047254de1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Harry=20M=C3=ADchal?= <harrymichal@seznam.cz>
Date: Mon, 10 Feb 2020 16:16:36 +0100
Subject: [PATCH] Rework the system tests

The tests introduced by #250 have proven to be rather unstable due to
mistakes in their design. The tests very quite chaotically structured.
Because of that images were deleted and pulled too often which caused
several false positives (#374, #372).

This changes the strucutre of the tests in a major way. The tests (resp.
commands) are now ran in a manner to kinda simulate the way Toolbox is
used. From clean state, through creating containers, using them and in
the end deleting them. This should reduce the strain on the bandwidth
and possibly even speed up the tests themselves.

More information in the README.md in the directory with the tests.
---
 .../{001-basics.bats => 001-version.bats}     | 12 ----
 test/system/002-help.bats                     |  8 +++
 test/system/101-create.bats                   | 20 -------
 test/system/101-list.bats                     | 24 ++++++++
 test/system/102-create.bats                   | 20 +++++++
 test/system/102-list.bats                     | 44 ---------------
 test/system/103-list.bats                     | 15 +++++
 test/system/103-remove.bats                   | 56 -------------------
 test/system/104-run.bats                      | 12 ----
 test/system/201-run.bats                      | 23 ++++++++
 test/system/301-rm.bats                       | 28 ++++++++++
 test/system/302-rmi.bats                      |  8 +++
 test/system/README.md                         | 45 +++++----------
 test/system/helpers.bash                      | 34 -----------
 14 files changed, 139 insertions(+), 210 deletions(-)
 rename test/system/{001-basics.bats => 001-version.bats} (55%)
 create mode 100644 test/system/002-help.bats
 delete mode 100644 test/system/101-create.bats
 create mode 100644 test/system/101-list.bats
 create mode 100644 test/system/102-create.bats
 delete mode 100644 test/system/102-list.bats
 create mode 100644 test/system/103-list.bats
 delete mode 100644 test/system/103-remove.bats
 delete mode 100644 test/system/104-run.bats
 create mode 100644 test/system/201-run.bats
 create mode 100644 test/system/301-rm.bats
 create mode 100644 test/system/302-rmi.bats

diff --git a/test/system/001-basics.bats b/test/system/001-version.bats
similarity index 55%
rename from test/system/001-basics.bats
rename to test/system/001-version.bats
index 8c3417d77..86c54434f 100644
--- a/test/system/001-basics.bats
+++ b/test/system/001-version.bats
@@ -2,14 +2,6 @@
 
 load helpers
 
-function setup() {
-  :
-}
-
-function teardown() {
-  :
-}
-
 @test "Output version number using full flag" {
   skip "Not implemented"
   run_toolbox --version
@@ -20,7 +12,3 @@ function teardown() {
   run_toolbox version
 }
 
-@test "Show usage screen when no command is given" {
-  run_toolbox 1
-  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
-}
diff --git a/test/system/002-help.bats b/test/system/002-help.bats
new file mode 100644
index 000000000..8620a36b0
--- /dev/null
+++ b/test/system/002-help.bats
@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Show usage screen when no command is given" {
+  run_toolbox 1
+  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
+}
\ No newline at end of file
diff --git a/test/system/101-create.bats b/test/system/101-create.bats
deleted file mode 100644
index bced2303b..000000000
--- a/test/system/101-create.bats
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bats
-
-load helpers
-
-@test "Create the default container." {
-  run_toolbox -y create
-}
-
-@test "Create a container with a valid custom name (whole word)" {
-  run_toolbox -y create -c "customname"
-}
-
-@test "Try to create a container with a bad custom name (with special characters)" {
-  run_toolbox 1 -y create -c "ßpeci@l.Nam€"
-  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
-}
-
-@test "Create a container with a custom image (f29)" {
-  run_toolbox -y create -i fedora-toolbox:29
-}
diff --git a/test/system/101-list.bats b/test/system/101-list.bats
new file mode 100644
index 000000000..198071811
--- /dev/null
+++ b/test/system/101-list.bats
@@ -0,0 +1,24 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with zero containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3 (Img: 3 + Spc: 0 + Cont: 0)"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}
+
+@test "Run list with zero containers (-c flag)" {
+  run_toolbox list -c
+  is "$output" "" "Output of list should be blank"
+}
+
+@test "Run list with zero images (-i flag)" {
+  run_toolbox list -i
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}
\ No newline at end of file
diff --git a/test/system/102-create.bats b/test/system/102-create.bats
new file mode 100644
index 000000000..edd5ad01b
--- /dev/null
+++ b/test/system/102-create.bats
@@ -0,0 +1,20 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Create the default container" {
+  run_toolbox -y create
+}
+
+@test "Create a container with a valid custom name ('not-running')" {
+  run_toolbox -y create -c "not-running"
+}
+
+@test "Create a container with a custom image and name ('running';f29)" {
+  run_toolbox -y create -c "running" -i fedora-toolbox:29
+}
+
+@test "Try to create a container with invalid custom name" {
+  run_toolbox 1 -y create -c "ßpeci@l.Nam€"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+}
\ No newline at end of file
diff --git a/test/system/102-list.bats b/test/system/102-list.bats
deleted file mode 100644
index c587039b9..000000000
--- a/test/system/102-list.bats
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bats
-
-load helpers
-
-@test "Run list with zero containers and zero images" {
-  remove_all_images
-  remove_all_containers
-  run_toolbox list
-  is "$output" "" "Output of list should be blank"
-}
-
-@test "Run list with zero containers (-c flag)" {
-  remove_all_containers
-  run_toolbox list -c
-  is "$output" "" "Output of list should be blank"
-}
-
-@test "Run list with zero images (-i flag)" {
-  remove_all_images
-  run_toolbox list -i
-  is "$output" "" "Output of list should be blank"
-}
-
-@test "Run list with 1 default container and 1 default image" {
-  create_toolbox
-  run_toolbox list
-  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "Default image"
-  is "${lines[3]}" ".*fedora-toolbox-.*" "Default container"
-  is "${#lines[@]}" "4" "Expected length of output is 4"
-}
-
-@test "Run list with 3 containers (-c flag)" {
-  create_toolbox 3 fedora
-  run_toolbox list -c
-  for i in $(seq 1 3); do
-    is "${lines[$i]}" ".*fedora-$((i)) \+" "One of the containers"
-  done
-}
-
-@test "Run list with 3 images (-i flag)" {
-  get_images 3
-  run_toolbox list -i
-  is "${#lines[@]}" "4" "Expected length of output is 4"
-}
diff --git a/test/system/103-list.bats b/test/system/103-list.bats
new file mode 100644
index 000000000..eb23bff0b
--- /dev/null
+++ b/test/system/103-list.bats
@@ -0,0 +1,15 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with three containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "9" "Expected number of lines of the output is 8 (Img: 3 + Spc: 1 + Cont: 5 (duplication expected))"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "The first of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "The second of the two images"
+
+  is "${lines[5]}" ".*fedora-toolbox-.*" "The default container should be first in the list"
+  is "${lines[6]}" ".*not-running.*" "The container 'not-running' should be second"
+  is "${lines[7]}" ".*running.*" "The container 'running' should be third (last)"
+}
\ No newline at end of file
diff --git a/test/system/103-remove.bats b/test/system/103-remove.bats
deleted file mode 100644
index c31b2a209..000000000
--- a/test/system/103-remove.bats
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bats
-
-load helpers
-
-@test "Remove a specific container (called fedora-2)" {
-  create_toolbox 2 fedora
-  run_toolbox rm fedora-2
-  is "$output" "" "Successfull removal shouldn't print anything"
-}
-
-@test "Remove a specific image (default image called by name)" {
-  get_images 1
-  run_toolbox rmi "$TOOLBOX_DEFAULT_IMAGE"
-}
-
-@test "Try to remove a nonexistent container" {
-  local todelete="nonexistentcontainer"
-  run_toolbox 1 rm "$todelete"
-  is "$output" "toolbox: failed to inspect $todelete" "Toolbox should fail with: no such container"
-}
-
-@test "Try to remove a nonexistent image" {
-  local todelete="nonexistentimage"
-  run_toolbox 1 rmi "$todelete"
-}
-
-@test "Try to remove a running container (called fedora-1)" {
-  create_toolbox 1 fedora
-  run_toolbox run -c fedora-1 echo "WAKE UP"
-  run_toolbox 1 rm fedora-1
-  is "$output" "toolbox: failed to remove container fedora-1" "Toolbox should fail to remove the container"
-}
-
-@test "Remove all containers (2 present)" {
-  create_toolbox 2 fedora
-  run_toolbox rm --all
-  is "$output" "" ""
-}
-
-@test "Remove all images" {
-  get_images 2
-  run_toolbox rmi --all
-}
-
-@test "Try to remove all containers (running containers)" {
-  create_toolbox 2 fedora
-  run_toolbox run -c fedora-1 echo "WAKE UP"
-  run_toolbox run -c fedora-2 echo "WAKE UP"
-  run_toolbox 1 rm --all
-}
-
-@test "Try to remove all images with present containers" {
-  get_images 2
-  create_toolbox 2 fedora
-  run_toolbox 1 rmi --all
-}
diff --git a/test/system/104-run.bats b/test/system/104-run.bats
deleted file mode 100644
index ba5dad390..000000000
--- a/test/system/104-run.bats
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bats
-
-load helpers
-
-function setup() {
-  setup_with_one_container
-}
-
-@test "Echo 'Hello World' inside of an container" {
-  run_toolbox run echo "Hello World"
-  is "$output" "Hello World" "Should say 'Hello World'"
-}
diff --git a/test/system/201-run.bats b/test/system/201-run.bats
new file mode 100644
index 000000000..ffcaaa841
--- /dev/null
+++ b/test/system/201-run.bats
@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Echo 'Hello World' inside of the default container" {
+  run_toolbox run echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Echo 'Hello World' inside of the 'running' container" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Stop the 'running' container using 'podman stop'" {
+  run_podman stop running
+  is "${#lines[@]}" "1" "Expected number of lines of the output is 1 (with the id of the container)"
+}
+
+@test "Echo 'hello World' again in the 'running' container after being stopped and exit" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
\ No newline at end of file
diff --git a/test/system/301-rm.bats b/test/system/301-rm.bats
new file mode 100644
index 000000000..05c165c29
--- /dev/null
+++ b/test/system/301-rm.bats
@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Try to remove a nonexistent container" {
+  run_toolbox 1 rm nonexistentcontainer
+  is "$output" "toolbox: failed to inspect $todelete" "Toolbox should fail with: no such container"
+}
+
+@test "Try to remove the running container 'running'" {
+  run_toolbox 1 rm running
+  is "$output" "toolbox: failed to remove container running" "Toolbox should fail to remove the running container"
+}
+
+@test "Remove the not running container 'not-running'" {
+  run_toolbox rm not-running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove the running container 'running'" {
+  run_toolbox rm --force running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove all remaining containers (only 1 should be left)" {
+  run_toolbox rm --force --all
+  is "$output" "" "The output should be empty"
+}
\ No newline at end of file
diff --git a/test/system/302-rmi.bats b/test/system/302-rmi.bats
new file mode 100644
index 000000000..d28cf35ea
--- /dev/null
+++ b/test/system/302-rmi.bats
@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Remove all images (2 should be present; --force should not be necessary)" {
+  run_toolbox rmi --all
+  is "$output" "" "The output should be empty"
+}
\ No newline at end of file
diff --git a/test/system/README.md b/test/system/README.md
index dd05ca635..cce00b171 100644
--- a/test/system/README.md
+++ b/test/system/README.md
@@ -1,43 +1,24 @@
 # System tests
 
 These tests are built with BATS (Bash Automated Testing System). They are
-strongly influenced by libpod project.
+strongly influenced by the [libpod](https://github.com/containers/libpod) project.
 
-The tests are meant to ensure that toolbox's functionality remains stable
+The tests are meant to ensure that Toolbox's functionality remains stable
 throughout updates of both Toolbox and Podman/libpod.
 
 ## Structure
 
-- **Basic Tests**
-  - [] output version number (Toolbox + Podman)
-  - [x] show help screen when no command is given
-  - [x] create the default container
-  - [x] create a container with a custom name
-  - [x] create a container from a custom image
-  - [x] list containers (no present)
-  - [x] list default container and default image
-  - [x] list containers (some present; different name patterns)
-  - [x] list images (no present)
-  - [x] list images (some present; different name patterns)
-  - [x] remove a specific container
-  - [x] try to remove nonexistent container
-  - [x] try to remove a running container
-  - [x] remove all containers
-  - [x] try to remove all containers (running)
-  - [x] remove a specific image
-  - [x] remove all images
-  - [x] run a command inside of an existing container
-
-- **Advanced Tests**
-  - [ ] create several containers with various configuration and then list them
-  - [ ] create several containers and hop between them (series of enter/exit)
-  - [ ] create a container, enter it, run a series of basic commands (id,
-        whoami, dnf, top, systemctl,..)
-  - [ ] enter a container and test basic set of networking tools (ping,
-        traceroute,..)
-
-The list of tests is stil rather basic. We **welcome** PRs with test
-suggestions or even their implementation.
+- **0xx (Info)**
+  - In this section are ran commands that are not dependent on the presence/number of containers/images (eg. getting version, help,..).
+- **1xx (Initialization)**
+  - In this sections are ran some commands (list, create) when Toolbox has not really been used, yet.
+  - It tries to list an empty list, creates several containers (default one + several with custom names/images).
+- **2xx (Usage)**
+  - In this section the prepared containers are used for the first time testing the initialization (CMD of the container).
+  - Not all containers will be used because in the *Cleanup* phase we want to try removing containers in both running/not running state.
+- **3xx (Cleanup)**
+  - In this section the containers and images from the previous *phases* are removed
+  - There is a difference between removing running/not running containers. We need to check the right behaviour.
 
 ## Convention
 
diff --git a/test/system/helpers.bash b/test/system/helpers.bash
index 175b1c6a8..127b749bd 100644
--- a/test/system/helpers.bash
+++ b/test/system/helpers.bash
@@ -17,40 +17,6 @@ LGC='\033[1;32m' # Light Green Color
 LBC='\033[1;34m' # Light Blue Color
 NC='\033[0m' # No Color
 
-# Basic setup
-function basic_setup() {
-    echo "# [basic_setup]" >&2
-    # Make sure desired images are present
-    if [ -z "$found_needed_image" ]; then
-        run_podman pull "$TOOLBOX_DEFAULT_IMAGE"
-    fi
-}
-
-function setup_with_one_container() {
-    echo "# [setup_with_one_container]" >&2
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-    # Create a new (default) container if no other are present
-    run_toolbox -y create
-}
-
-function basic_teardown() {
-    echo "# [basic_teardown]" >&2
-    # Clean up all containers
-    remove_all_containers
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-}
-
-# Set the default setup function
-function setup() {
-    basic_setup
-}
-
-function teardown() {
-    basic_teardown
-}
-
 
 ################
 #  run_podman  #  Invoke $PODMAN, with timeout, using BATS 'run'