Rework the system tests

The tests introduced by containers#250 have proven to be rather unstable due to
mistakes in their design. The tests very quite chaotically structured.
Because of that images were deleted and pulled too often which caused
several false positives (containers#374, containers#372).

This changes the strucutre of the tests in a major way. The tests (resp.
commands) are now ran in a manner to kinda simulate the way Toolbox is
used. From clean state, through creating containers, using them and in
the end deleting them. This should reduce the strain on the bandwidth
and possibly even speed up the tests themselves.

More information in the README.md in the directory with the tests.

test/system/001-basics.bats → test/system/001-version.bats

@@ -2,14 +2,6 @@
 
 load helpers
 
-function setup() {
-  :
-}
-
-function teardown() {
-  :
-}
-
 @test "Output version number using full flag" {
   skip "Not implemented"
   run_toolbox --version
@@ -20,7 +12,3 @@ function teardown() {
   run_toolbox version
 }
 
-@test "Show usage screen when no command is given" {
-  run_toolbox 1
-  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
-}

test/system/002-help.bats

@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Show usage screen when no command is given" {
+  run_toolbox 1
+  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
+}

test/system/101-list.bats

@@ -0,0 +1,24 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with zero containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3 (Img: 3 + Spc: 0 + Cont: 0)"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}
+
+@test "Run list with zero containers (-c flag)" {
+  run_toolbox list -c
+  is "$output" "" "Output of list should be blank"
+}
+
+@test "Run list with zero images (-i flag)" {
+  run_toolbox list -i
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}

test/system/102-create.bats

@@ -0,0 +1,20 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Create the default container" {
+  run_toolbox -y create
+}
+
+@test "Create a container with a valid custom name ('not-running')" {
+  run_toolbox -y create -c "not-running"
+}
+
+@test "Create a container with a custom image and name ('running';f29)" {
+  run_toolbox -y create -c "running" -i fedora-toolbox:29
+}
+
+@test "Try to create a container with invalid custom name" {
+  run_toolbox 1 -y create -c "ß[email protected]€"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+}

test/system/103-list.bats

@@ -0,0 +1,15 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with three containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "8" "Expected number of lines of the output is 8 (Img: 3 + Cont: 5 (duplication expected))"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "The first of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "The second of the two images"
+
+  is "${lines[4]}" ".*fedora-toolbox-.*" "The default container should be first in the list"
+  is "${lines[5]}" ".*not-running.*" "The container 'not-running' should be second"
+  is "${lines[6]}" ".*running.*" "The container 'running' should be third (last)"
+}

test/system/201-run.bats

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Echo 'Hello World' inside of the default container" {
+  run_toolbox run echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Echo 'Hello World' inside of the 'running' container" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Stop the 'running' container using 'podman stop'" {
+  run_podman stop running
+  is "${#lines[@]}" "1" "Expected number of lines of the output is 1 (with the id of the container)"
+}
+
+@test "Echo 'hello World' again in the 'running' container after being stopped and exit" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}

test/system/301-rm.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Try to remove a nonexistent container" {
+  run_toolbox 1 rm nonexistentcontainer
+  is "$output" "toolbox: failed to inspect $todelete" "Toolbox should fail with: no such container"
+}
+
+@test "Try to remove the running container 'running'" {
+  run_toolbox 1 rm running
+  is "$output" "toolbox: failed to remove container running" "Toolbox should fail to remove the running container"
+}
+
+@test "Remove the not running container 'not-running'" {
+  run_toolbox rm not-running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove the running container 'running'" {
+  run_toolbox rm --force running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove all remaining containers (only 1 should be left)" {
+  run_toolbox rm --force --all
+  is "$output" "" "The output should be empty"
+}

test/system/302-rmi.bats

@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Remove all images (2 should be present; --force should not be necessary)" {
+  run_toolbox rmi --all
+  is "$output" "" "The output should be empty"
+}

test/system/README.md

@@ -1,43 +1,24 @@
 # System tests
 
 These tests are built with BATS (Bash Automated Testing System). They are
-strongly influenced by libpod project.
+strongly influenced by the [libpod](https://github.com/containers/libpod) project.
 
-The tests are meant to ensure that toolbox's functionality remains stable
+The tests are meant to ensure that Toolbox's functionality remains stable
 throughout updates of both Toolbox and Podman/libpod.
 
 ## Structure
 
-- **Basic Tests**
-  - [] output version number (Toolbox + Podman)
-  - [x] show help screen when no command is given
-  - [x] create the default container
-  - [x] create a container with a custom name
-  - [x] create a container from a custom image
-  - [x] list containers (no present)
-  - [x] list default container and default image
-  - [x] list containers (some present; different name patterns)
-  - [x] list images (no present)
-  - [x] list images (some present; different name patterns)
-  - [x] remove a specific container
-  - [x] try to remove nonexistent container
-  - [x] try to remove a running container
-  - [x] remove all containers
-  - [x] try to remove all containers (running)
-  - [x] remove a specific image
-  - [x] remove all images
-  - [x] run a command inside of an existing container
-
-- **Advanced Tests**
-  - [ ] create several containers with various configuration and then list them
-  - [ ] create several containers and hop between them (series of enter/exit)
-  - [ ] create a container, enter it, run a series of basic commands (id,
-        whoami, dnf, top, systemctl,..)
-  - [ ] enter a container and test basic set of networking tools (ping,
-        traceroute,..)
-
-The list of tests is stil rather basic. We **welcome** PRs with test
-suggestions or even their implementation.
+- **0xx (Info)**
+  - In this section are ran commands that are not dependent on the presence/number of containers/images (eg. getting version, help,..).
+- **1xx (Initialization)**
+  - In this sections are ran some commands (list, create) when Toolbox has not really been used, yet.
+  - It tries to list an empty list, creates several containers (default one + several with custom names/images).
+- **2xx (Usage)**
+  - In this section the prepared containers are used for the first time testing the initialization (CMD of the container).
+  - Not all containers will be used because in the *Cleanup* phase we want to try removing containers in both running/not running state.
+- **3xx (Cleanup)**
+  - In this section the containers and images from the previous *phases* are removed
+  - There is a difference between removing running/not running containers. We need to check the right behaviour.
 
 ## Convention
 

test/system/helpers.bash

@@ -17,40 +17,6 @@ LGC='\033[1;32m' # Light Green Color
 LBC='\033[1;34m' # Light Blue Color
 NC='\033[0m' # No Color
 
-# Basic setup
-function basic_setup() {
-    echo "# [basic_setup]" >&2
-    # Make sure desired images are present
-    if [ -z "$found_needed_image" ]; then
-        run_podman pull "$TOOLBOX_DEFAULT_IMAGE"
-    fi
-}
-
-function setup_with_one_container() {
-    echo "# [setup_with_one_container]" >&2
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-    # Create a new (default) container if no other are present
-    run_toolbox -y create
-}
-
-function basic_teardown() {
-    echo "# [basic_teardown]" >&2
-    # Clean up all containers
-    remove_all_containers
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-}
-
-# Set the default setup function
-function setup() {
-    basic_setup
-}
-
-function teardown() {
-    basic_teardown
-}
-
 
 ################
 #  run_podman  #  Invoke $PODMAN, with timeout, using BATS 'run'