Add a set of tests

playbooks/toolbox-test-git.yaml

@@ -13,7 +13,6 @@
         chdir: '{{ zuul.project.src_dir }}'
 
     - name: Run system tests
-      ignore_errors: yes
       command: bats ./test/system
       environment:
         PODMAN: './podman/usr/bin/podman'

test/system/001-basics.bats

@@ -0,0 +1,22 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  :
+}
+
+@test "Output version number using full flag" {
+  skip "Not implemented"
+  run_toolbox --version
+}
+
+@test "Output version number using command" {
+  skip "Not implemented"
+  run_toolbox version
+}
+
+@test "Show usage screen when no command is given" {
+  run_toolbox 1
+  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
+}

test/system/101-create.bats

@@ -0,0 +1,49 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Create the default container." {
+  run_toolbox -y create
+}
+
+@test "Create a container with a valid custom name (whole word)" {
+  run_toolbox -y create -c "customname"
+}
+
+@test "Try to create a container with a bad custom name (with special characters)" {
+  run_toolbox 1 -y create -c "ßpeci@lNam€"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+  #is "${lines[1]}" "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'." "Toolbox shows required pattern for naming of containers"
+}
+
+@test "Try to create a container with a bad custom name (with a dot)" {
+  run_toolbox 1 -y create -c ".custom.name."
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+  #is "${lines[1]}" "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'." "Toolbox shows required pattern for naming of containers"
+}
+
+@test "Try to create a container with a bad custom name (with a hyphen)" {
+  run_toolbox 1 -y create -c "-custom-name-"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+  #is "${lines[1]}" "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'." "Toolbox shows required pattern for naming of containers"
+}
+
+@test "Try to create a container with a bad custom name (with an underline)" {
+  run_toolbox 1 -y create -c "_custom_name_"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+  #is "${lines[1]}" "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'." "Toolbox shows required pattern for naming of containers"
+}
+
+@test "Try to create a container with no name (just a single space)" {
+  run_toolbox 1 -y create -c " "
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+  #is "${lines[1]}" "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'." "Toolbox shows required pattern for naming of containers"
+}
+
+@test "Create a container with a custom image (f28)" {
+  run_toolbox -y create -i $(get_image_name fedora 28)
+}
+
+@test "Create a container with a custom name and image (f29)" {
+  run_toolbox -y create -c "customname" -i $(get_image_name fedora 29)
+}

test/system/README.md

@@ -0,0 +1,48 @@
+# System tests
+
+These tests are built on BATS (Bash Automated Testing System). They are strongly influenced by libpod project.
+
+The tests are meant to ensure that toolbox's functionality remains stable throughout updates of both Toolbox and Podman/libpod.
+
+## Structure
+
+The tests are divided to sections according to the number of required steps to achieve the goal and the overall difficulty to get the task done.
+
+- **Basic Tests**
+  - [x] output version number (Toolbox + Podman)
+  - [x] show help screen when no command is given
+  - [x] show help screen when full flag is given
+  - [x] show help screen when short flag is given
+  - [x] show help screen when called
+  - [x] list containers (no present)
+  - [x] list default container and default image
+  - [x] list containers (some present; different name patterns)
+  - [x] list images (no present)
+  - [x] list images (some present; different name patterns)
+  - [x] create the default container
+  - [ ] execute a full cycle of toolbox with default settings (create, list, run, enter, exit, delete)
+
+- **Advanced Tests**
+  - [x] create a container with a custom name
+  - [x] create a container from a custom image
+  - [x] create a container from a custom image with a custom name
+  - [x] remove a specific container
+  - [x] try to remove nonexistent container
+  - [x] try to remove a running container
+  - [x] remove all containers
+  - [x] try to remove all containers (running)
+  - [x] remove a specific image
+  - [x] remove all images
+  - [ ] run a command inside of an existing image
+
+- **Complex Tests**
+  - [ ] create several containers with various configuration and then list them
+  - [ ] create several containers and hop between them (series of enter/exit)
+  - [ ] create a container, enter it, run a series of basic commands (id, whoami, dnf, top, systemctl,..)
+  - [ ] enter a container and test basic set of networking tools (ping, traceroute,..)
+
+The list of tests is stil rather basic. We **welcome** PRs with test suggestions or even their implementation.
+
+## Convention
+
+- All tests that start with *Try to..* expect non-zero return value.