test/system: Refactor tests using bats-support and bats-assert

The bats-support[0] and bats-assert[1] libraries extend the
capabilities of bats[2]. Mainly, bats-assert is very useful for clean
checking of values/outputs/return codes.

Apart from updating the cases to use the libraries, the test cases have
been restructured in a way that they don't depend on each other anymore.
This required major changes in the helpers.bats file.

Overall, the tests are cleaner to read and easier to extend due to the
test cases being independent.

Some slight changes were made to the test cases themselves. Should not
alter their final behaviour.

There will be a follow up commit that will take care of downloading of
the tested images locally and caching them using Skopeo to speedup the
tests and try to resolve network problems when pulling the images that
we experienced in the past.

[0] https://github.com/bats-core/bats-support
[1] https://github.com/bats-core/bats-assert
[2] https://github.com/bats-core/bats-core

.gitignore

@@ -1 +1,3 @@
 src/toolbox
+test/system/libs/bats-assert
+test/system/libs/bats-support

playbooks/fedora-31/setup-env.yaml

@@ -4,6 +4,7 @@
     - name: Install requirements
       become: yes
       package:
+        use: dnf
         name:
           - golang
           - golang-github-cpuguy83-md2man
@@ -16,6 +17,18 @@
           - udisks2
           - podman
 
+    - name: Install bats-support library
+      git:
+        repo: https://github.com/bats-core/bats-support
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support'
+        depth: 1
+
+    - name: Install bats-assert library
+      git:
+        repo: https://github.com/bats-core/bats-assert
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert'
+        depth: 1
+
     - name: Setup environment
       become: yes
       command:

playbooks/fedora-32/setup-env.yaml

@@ -17,6 +17,18 @@
           - udisks2
           - podman
 
+    - name: Install bats-support library
+      git:
+        repo: https://github.com/bats-core/bats-support
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support'
+        depth: 1
+
+    - name: Install bats-assert library
+      git:
+        repo: https://github.com/bats-core/bats-assert
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert'
+        depth: 1
+
     - name: Setup environment
       become: yes
       command:

playbooks/fedora-33/setup-env.yaml

@@ -17,6 +17,18 @@
           - udisks2
           - podman
 
+    - name: Install bats-support library
+      git:
+        repo: https://github.com/bats-core/bats-support
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support'
+        depth: 1
+
+    - name: Install bats-assert library
+      git:
+        repo: https://github.com/bats-core/bats-assert
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert'
+        depth: 1
+
     - name: Setup environment
       become: yes
       command:

playbooks/fedora-rawhide/setup-env.yaml

@@ -17,6 +17,18 @@
           - udisks2
           - podman
 
+    - name: Install bats-support library
+      git:
+        repo: https://github.com/bats-core/bats-support
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-support'
+        depth: 1
+
+    - name: Install bats-assert library
+      git:
+        repo: https://github.com/bats-core/bats-assert
+        dest: '{{ zuul.project.src_dir }}/test/system/libs/bats-assert'
+        depth: 1
+
     - name: Setup environment
       become: yes
       command:

test/system/001-version.bats

@@ -1,7 +1,10 @@
 #!/usr/bin/env bats
 
-load helpers
+load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
 
-@test "Output version number using full flag" {
-  run_toolbox --version
+@test "version: Check version using option --version" {
+  run toolbox --version
+
+  assert_output --regexp '^toolbox version [0-9]+\.[0-9]+\.[0-9]+$'
 }

test/system/002-help.bats

@@ -1,8 +1,34 @@
 #!/usr/bin/env bats
 
-load helpers
+load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
 
-@test "Show usage screen when no command is given" {
-  run_toolbox 1
-  is "${lines[0]}" "Error: missing command" "Usage line 1"
+@test "help: Try to run toolbox with no command (shows usage screen)" {
+  run toolbox
+
+  assert_failure
+  assert_line --index 0 "Error: missing command"
+  assert_output --partial "Run 'toolbox --help' for usage."
+}
+
+@test "help: Run command 'help'" {
+  run toolbox help
+
+  assert_success
+  assert_output --partial "toolbox - Unprivileged development environment"
+}
+
+@test "help: Use flag '--help' (it should show usage screen)" {
+  run toolbox --help
+
+  assert_success
+  assert_output --partial "toolbox - Unprivileged development environment"
+}
+
+@test "help: Try to run toolbox with non-existent command (shows usage screen)" {
+  run toolbox foo
+
+  assert_failure
+  assert_line --index 0 "Error: unknown command \"foo\" for \"toolbox\""
+  assert_line --index 1 "Run 'toolbox --help' for usage."
 }

test/system/101-create.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
+load 'libs/helpers'
+
+setup() {
+  cleanup_containers
+}
+
+teardown() {
+  cleanup_containers
+}
+
+
+@test "create: Create the default container" {
+  pull_default_image
+
+  run toolbox -y create
+
+  assert_success
+}
+
+@test "create: Create a container with a valid custom name ('custom-containerName')" {
+  run toolbox -y create -c "custom-containerName"
+
+  assert_success
+}
+
+@test "create: Create a container with a custom image and name ('fedora29'; f29)" {
+  pull_image 29
+
+  run toolbox -y create -c "fedora29" -i fedora-toolbox:29
+
+  assert_success
+}
+
+@test "create: Try to create a container with invalid custom name ('ß[email protected]€'; using positional argument)" {
+  run toolbox -y create "ß[email protected]€"
+
+  assert_failure
+  assert_line --index 0 "Error: invalid argument for 'CONTAINER'"
+  assert_line --index 1 "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'"
+  assert_line --index 2 "Run 'toolbox --help' for usage."
+}
+
+@test "create: Try to create a container with invalid custom name ('ß[email protected]€'; using option --container)" {
+  run toolbox -y create -c "ß[email protected]€"
+
+  assert_failure
+  assert_line --index 0 "Error: invalid argument for '--container'"
+  assert_line --index 1 "Container names must match '[a-zA-Z0-9][a-zA-Z0-9_.-]*'"
+  assert_line --index 2 "Run 'toolbox --help' for usage."
+}

test/system/102-list.bats

@@ -0,0 +1,83 @@
+#!/usr/bin/env bats
+
+load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
+load 'libs/helpers'
+
+setup() {
+  cleanup_all
+}
+
+teardown() {
+  cleanup_all
+}
+
+
+@test "list: Run 'list' with zero containers and zero images (the list should be empty)" {
+  run toolbox list
+
+  assert_success
+  assert_output ""
+}
+
+@test "list: Run 'list -c' with zero containers (the list should be empty)" {
+  run toolbox list -c
+
+  assert_success
+  assert_output ""
+}
+
+@test "list: Run 'list -i' with zero images (the list should be empty)" {
+  run toolbox list -c
+
+  assert_success
+  assert_output ""
+}
+
+@test "list: Run 'list' with zero toolbox's containers and images, but other image (the list should be empty)" {
+  get_busybox_image
+
+  run podman images
+
+  assert_output --partial "$BUSYBOX_IMAGE"
+
+  run toolbox list
+
+  assert_success
+  assert_output ""
+}
+
+@test "list: Try to list images and containers (no flag) with 3 containers and 2 images (the list should have 3 images and 2 containers)" {
+  # Pull the two images
+  pull_default_image
+  pull_image 29
+  # Create tree containers
+  create_default_container
+  create_container non-default-one
+  create_container non-default-two
+
+  # Check images
+  run toolbox list --images
+
+  assert_success
+  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "fedora-toolbox:29"
+
+  # Check containers
+  run toolbox list --containers
+
+  assert_success
+  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "non-default-one"
+  assert_output --partial "non-default-two"
+
+  # Check all together
+  run toolbox list
+
+  assert_success
+  assert_output --partial "fedora-toolbox:${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "fedora-toolbox:29"
+  assert_output --partial "fedora-toolbox-${DEFAULT_FEDORA_VERSION}"
+  assert_output --partial "non-default-one"
+  assert_output --partial "non-default-two"
+}

test/system/103-run.bats

@@ -0,0 +1,52 @@
+#!/usr/bin/env bats
+
+load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
+load 'libs/helpers'
+
+# It seems like 'toolbox run' (or 'enter') doesn't work fine when
+# the workdir is outside the $HOME.
+# This hack is to make the tests work from outside the $HOME.
+readonly CURDIR=$PWD
+
+setup() {
+  cd "$HOME" || return 1
+  cleanup_containers
+}
+
+teardown() {
+  cleanup_containers
+  cd "$CURDIR" || return 1
+}
+
+
+@test "run: Try to run echo 'Hello World' with no containers created" {
+  run toolbox run echo "Hello World"
+
+  assert_failure
+  assert_line --index 0 --regexp 'Error: container .* not found'
+  assert_output --partial "Run 'toolbox --help' for usage."
+}
+
+#TODO: This should work without --partial
+# The issue here is that toolbox output add the CRLF character at the end
+@test "run: Run echo 'Hello World' inside of the default container" {
+  create_default_container
+
+  run toolbox --verbose run echo "Hello World"
+
+  assert_success
+  assert_output --partial "Hello World"
+}
+
+@test "run: Run echo 'Hello World' inside a container after being stopped" {
+  create_container running
+
+  start_container running
+  stop_container running
+
+  run toolbox --verbose run --container running echo -n "Hello World"
+
+  assert_success
+  assert_output --partial "Hello World"
+}