cmd/create: Add option --authfile

The option accepts a path to a file that is passed to an internal call
to 'podman pull' via the '--authfile' option. This will make it easier
to pull images from registries with authentication in-place.

Slightly adjusts the format of system tests README.

Fixes containers#689

containers#935

doc/toolbox-create.1.md

@@ -4,7 +4,8 @@
 toolbox\-create - Create a new toolbox container
 
 ## SYNOPSIS
-**toolbox create** [*--distro DISTRO* | *-d DISTRO*]
+**toolbox create** [*--authfile AUTHFILE*]
+               [*--distro DISTRO* | *-d DISTRO*]
                [*--image NAME* | *-i NAME*]
                [*--release RELEASE* | *-r RELEASE*]
                [*CONTAINER*]
@@ -79,6 +80,14 @@ confusion.
 
 ## OPTIONS ##
 
+**--authfile** FILE
+
+Path to a FILE with credentials for authenticating to the registry for private
+images. The FILE is usually set using `podman login`, and will be used by
+`podman pull` to get the image.
+
+The default location for FILE is `$XDG_RUNTIME_DIR/containers/auth.json`.
+
 **--distro** DISTRO, **-d** DISTRO
 
 Create a toolbox container for a different operating system DISTRO than the
@@ -120,6 +129,12 @@ $ toolbox create --distro fedora --release f36
 $ toolbox create --image bar foo
 ```
 
+### Create a toolbox container from a custom image needing authentication
+
+```
+$ toolbox create --authfile ~/auth.json --image registry.example.com/bar
+```
+
 ## SEE ALSO
 
-`toolbox(1)`, `toolbox-init-container(1)`, `podman(1)`, `podman-create(1)`
+`toolbox(1)`, `toolbox-init-container(1)`, `podman(1)`, `podman-create(1)`, `podman-login(1)`, `podman-pull(1)`

playbooks/setup-env.yaml

@@ -12,8 +12,10 @@
           - flatpak-session-helper
           - golang
           - golang-github-cpuguy83-md2man
+          - httpd-tools
           - meson
           - ninja-build
+          - openssl
           - podman
           - skopeo
           - systemd

src/cmd/create.go

@@ -42,6 +42,7 @@ const (
 
 var (
 	createFlags struct {
+		authFile  string
 		container string
 		distro    string
 		image     string
@@ -67,6 +68,11 @@ var createCmd = &cobra.Command{
 func init() {
 	flags := createCmd.Flags()
 
+	flags.StringVar(&createFlags.authFile,
+		"authfile",
+		"",
+		"Path to a file with credentials for authenticating to the registry for private images")
+
 	flags.StringVarP(&createFlags.container,
 		"container",
 		"c",
@@ -124,6 +130,12 @@ func create(cmd *cobra.Command, args []string) error {
 		return errors.New("options --image and --release cannot be used together")
 	}
 
+	if cmd.Flag("authfile").Changed {
+		if !utils.PathExists(createFlags.authFile) {
+			return fmt.Errorf("file %s not found", createFlags.authFile)
+		}
+	}
+
 	var container string
 	var containerArg string
 
@@ -717,7 +729,7 @@ func pullImage(image, release string) (bool, error) {
 		defer s.Stop()
 	}
 
-	if err := podman.Pull(imageFull); err != nil {
+	if err := podman.Pull(imageFull, createFlags.authFile); err != nil {
 		var builder strings.Builder
 		fmt.Fprintf(&builder, "failed to pull image %s\n", imageFull)
 		fmt.Fprintf(&builder, "If it was a private image, log in with: podman login %s\n", domain)

src/pkg/podman/podman.go

@@ -227,9 +227,18 @@ func IsToolboxImage(image string) (bool, error) {
 }
 
 // Pull pulls an image
-func Pull(imageName string) error {
+//
+// authfile is a path to a JSON authentication file and is internally used only
+// if it is not an empty string.
+func Pull(imageName string, authfile string) error {
 	logLevelString := LogLevel.String()
-	args := []string{"--log-level", logLevelString, "pull", imageName}
+	args := []string{"--log-level", logLevelString, "pull"}
+
+	if authfile != "" {
+		args = append(args, []string{"--authfile", authfile}...)
+	}
+
+	args = append(args, imageName)
 
 	if err := shell.Run("podman", nil, nil, nil, args...); err != nil {
 		return err

test/system/000-setup.bats

@@ -19,4 +19,6 @@ load 'libs/helpers'
         _pull_and_cache_distro_image fedora "$((system_version-1))" || false
         _pull_and_cache_distro_image fedora "$((system_version-2))" || false
     fi
+
+    _setup_docker_registry
 }

test/system/101-create.bats

@@ -117,3 +117,35 @@ teardown() {
   assert_line --index 0 "Error: release not found for non-default distribution $distro"
   assert [ ${#lines[@]} -eq 1 ]
 }
+
+@test "create: Try to create a container and pass a non-existent file to the --authfile option" {
+  local file="$BATS_RUN_TMPDIR/non-existent-file"
+
+  run $TOOLBOX create --authfile "$file"
+
+  assert_failure
+  assert_output "Error: file $file not found"
+}
+
+@test "create: Create a container based on an image from locked registry using an authentication file" {
+  local authfile="$BATS_RUN_TMPDIR/authfile"
+  local image="fedora-toolbox:32"
+
+  run $PODMAN login --authfile "$authfile" --username user --password user "$DOCKER_REG_URI"
+  assert_success
+
+  run $TOOLBOX --assumeyes create --image "$DOCKER_REG_URI/$image"
+
+  assert_failure
+  assert_line --index 0 "Error: failed to pull image $DOCKER_REG_URI/$image"
+  assert_line --index 1 "If it was a private image, log in with: podman login $DOCKER_REG_URI"
+  assert_line --index 2 "Use 'toolbox --verbose ...' for further details."
+
+  run $TOOLBOX --assumeyes create --authfile "$authfile" --image "$DOCKER_REG_URI/$image"
+
+  assert_success
+  assert_output "Created container: fedora-toolbox-32"
+  assert_output "Enter with: toolbox enter fedora-toolbox-32"
+
+  rm "$authfile"
+}

test/system/999-teardown.bats

@@ -4,5 +4,6 @@ load 'libs/helpers'
 
 @test "test suite: Teardown" {
   _clean_cached_images
+  _clean_docker_registry
   _clean_temporary_storage
 }

test/system/README.md

@@ -13,6 +13,8 @@ Running them won't remove any existing containers or images.
 - `awk`
 - `bats`
 - `GNU coreutils`
+- `httpd-tools`
+- `openssl`
 - `podman`
 - `skopeo`
 - `toolbox`
@@ -22,23 +24,6 @@ and consistency. The libraries are [bats-support](https://github.com/bats-core/b
 and [bats-assert](https://github.com/bats-core/bats-assert). These libraries are
 provided as git submodules in the `libs` directory. Make sure both are present.
 
-## Convention
-
-- All tests should follow the nomenclature: `[command]: <test description>...`
-- When the test is expected to fail, start the test description with "Try
-  to..."
-- When the test is to give a non obvious output, it should be put in parenthesis
-  at the end of the title
-
-Examples:
-
-* `@test "create: Create the default container"`
-* `@test "rm: Try to remove a non-existent container"`
-
-- All the tests start with a clean system (no images or containers) to make sure
-  that there are no dependencies between tests and they are really isolated. Use
-  the `setup()` and `teardown()` functions for that purpose.
-
 ## How to run the tests
 
 First, make sure you have all the dependencies installed.
@@ -60,3 +45,38 @@ suite could look something like this: `PODMAN=/usr/libexec/podman TOOLBOX=./tool
 When running the tests, make sure the `test suite: [job]` jobs are successful.
 These jobs set up the whole environment and are a strict requirement for other
 jobs to run correctly.
+
+## Writing tests
+
+### Naming convention
+
+- All tests should follow the nomenclature: `[command]: <test description>...`
+- When the test is expected to fail, start the test description with "Try
+  to..."
+- When the test is to give a non obvious output, it should be put in parenthesis
+  at the end of the title
+
+Examples:
+
+* `@test "create: Create the default container"`
+* `@test "rm: Try to remove a non-existent container"`
+
+### Test case environment
+
+- All the tests start with a clean system (no images or containers) to make sure
+  that there are no dependencies between tests and they are really isolated. Use
+  the `setup()` and `teardown()` functions for that purpose.
+
+### Image registry
+
+- The system tests set up an OCI image registry for testing purposes. The registry
+  requires authentication. There is one account present: user (password: user)
+
+- The registry contains by default only one image: `fedora-toolbox:32`
+
+Example pull of the `fedora-toolbox:32` image:
+
+```bash
+$PODMAN login --username user --password user "$DOCKER_REG_URI"
+$PODMAN pull "$DOCKER_REG_URI/fedora-toolbox:32"
+```

test/system/libs/helpers.bash

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 load 'libs/bats-support/load'
+load 'libs/bats-assert/load'
 
 # Helpful globals
 readonly TEMP_BASE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/toolbox"
@@ -9,6 +10,12 @@ readonly TEMP_STORAGE_DIR="${TEMP_BASE_DIR}/system-test-storage"
 readonly IMAGE_CACHE_DIR="${BATS_RUN_TMPDIR}/image-cache"
 readonly ROOTLESS_PODMAN_STORE_DIR="${TEMP_STORAGE_DIR}/storage"
 readonly PODMAN_STORE_CONFIG_FILE="${TEMP_STORAGE_DIR}/store.conf"
+readonly DOCKER_REG_ROOT="${TEMP_STORAGE_DIR}/podman-registry-root"
+readonly DOCKER_REG_DIR="${TEMP_STORAGE_DIR}/docker-registry"
+readonly DOCKER_REG_CERTS_DIR="${BATS_RUN_TMPDIR}/certs"
+readonly DOCKER_REG_AUTH_DIR="${BATS_RUN_TMPDIR}/auth"
+readonly DOCKER_REG_URI="localhost:50000"
+readonly DOCKER_REG_NAME="docker-registry"
 
 # Podman and Toolbox commands to run
 readonly PODMAN=${PODMAN:-podman}
@@ -17,6 +24,7 @@ readonly SKOPEO=$(command -v skopeo)
 
 # Images
 declare -Ag IMAGES=([busybox]="quay.io/toolbox_tests/busybox" \
+                   [docker-reg]="quay.io/toolbox_tests/registry" \
                    [fedora]="registry.fedoraproject.org/fedora-toolbox" \
                    [rhel]="registry.access.redhat.com/ubi8/toolbox")
 
@@ -36,6 +44,7 @@ function _setup_environment() {
   check_xdg_runtime_dir
 }
 
+
 function _setup_containers_store() {
   mkdir -p ${TEMP_STORAGE_DIR}
   # Setup a storage config file for PODMAN
@@ -119,6 +128,86 @@ function _clean_cached_images() {
 }
 
 
+# Prepares a localy hosted image registry
+#
+# The registry is set up with Podman set to an alternative root. It won't
+# affect other containers or images in the default root.
+#
+# Instructions taken from https://docs.docker.com/registry/deploying/
+function _setup_docker_registry() {
+  # Create certificates for HTTPS
+  # This is needed so that Podman does not have to be configured to work with
+  # HTTP-only registries
+  run mkdir -p "${DOCKER_REG_CERTS_DIR}"
+  assert_success
+  run openssl req \
+    -newkey rsa:4096 \
+    -nodes -sha256 \
+    -keyout "${DOCKER_REG_CERTS_DIR}/domain.key" \
+    -addext "subjectAltName= DNS:localhost" \
+    -x509 \
+    -days 365 \
+    -subj '/' \
+    -out "${DOCKER_REG_CERTS_DIR}/domain.crt"
+  assert_success
+
+  # Add certificate to Podman's trusted certificates (rootless)
+  run mkdir -p "~/.config/containers/certs.d/${DOCKER_REG_URI}"
+  assert_success
+  run cp "${DOCKER_REG_CERTS_DIR}/domain.{crt,key}" "~/.config/containers/certs.d/${DOCKER_REG_URI}"
+  assert_success
+
+  # Create a registry user
+  # username: user; password: user
+  run mkdir -p "${DOCKER_REG_AUTH_DIR}"
+  assert_success
+  run htpasswd -Bbc "${DOCKER_REG_AUTH_DIR}/htpasswd" user user
+  assert_success
+
+  # Create separate Podman root
+  run mkdir -p "${DOCKER_REG_ROOT}"
+  assert_success
+
+  # Pull Docker registry image
+  run $PODMAN --root "${DOCKER_REG_ROOT}" pull "${IMAGES[docker-reg]}"
+  assert_success
+
+  # Create a Docker registry
+  run $PODMAN --root "${DOCKER_REG_ROOT}" run -d \
+    --rm \
+    --name "${DOCKER_REG_NAME}" \
+    --privileged \
+    -v "${DOCKER_REG_AUTH_DIR}:/auth" \
+    -e REGISTRY_AUTH=htpasswd \
+    -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" \
+    -e REGISTRY_AUTH_HTPASSWD_PATH="/auth/htpasswd" \
+    -v "${DOCKER_REG_CERTS_DIR}":/certs \
+    -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \
+    -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
+    -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
+    -p 50000:443 \
+    "${IMAGES[docker-reg]}"
+  assert_success
+
+  # Add fedora-toolbox:32 image to the registry
+  run $SKOPEO copy "dir:${IMAGE_CACHE_DIR}/fedora-toolbox-32" "docker://${DOCKER_REG_URI}/fedora-toolbox:32"
+  assert_success
+}
+
+
+# Stop, removes and cleans after a locally hosted Docker registry
+function _clean_docker_registry() {
+  # Stop Docker registry container
+  podman --root "${DOCKER_REG_ROOT}" stop "${DOCKER_REG_NAME}"
+  # Remove Docker registry dirs
+  rm -rf "${DOCKER_REG_ROOT}"
+  rm -rf "${DOCKER_REG_DIR}"
+  # Remove dir with created registry certificates
+  rm -rf "~/.config/containers/certs.d/${DOCKER_REG_URI}"
+
+}
+
+
 # Copies an image from local storage to Podman's image store
 # 
 # Call before creating any container. Network failures are not nice.