From 43e9e44d694ce2d6e194c86718605f29eba6d309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20M=C3=ADchal?= Date: Thu, 25 Nov 2021 01:52:15 +0200 Subject: [PATCH] cmd/create: Add option --authfile The option accepts a path to a file that is passed to an internal call to 'podman pull' via the '--authfile' option. This will make it easier to pull images from registries with authentication in-place. Slightly adjusts the format of system tests README. Fixes https://github.com/containers/toolbox/issues/689 https://github.com/containers/toolbox/pull/935 --- doc/toolbox-create.1.md | 19 +++++++- playbooks/setup-env.yaml | 2 + src/cmd/create.go | 14 +++++- src/pkg/podman/podman.go | 13 ++++- test/system/000-setup.bats | 2 + test/system/101-create.bats | 32 +++++++++++++ test/system/999-teardown.bats | 1 + test/system/README.md | 54 ++++++++++++++------- test/system/libs/helpers.bash | 89 +++++++++++++++++++++++++++++++++++ 9 files changed, 204 insertions(+), 22 deletions(-) diff --git a/doc/toolbox-create.1.md b/doc/toolbox-create.1.md index 0e1861513..77c3787f4 100644 --- a/doc/toolbox-create.1.md +++ b/doc/toolbox-create.1.md @@ -4,7 +4,8 @@ toolbox\-create - Create a new toolbox container ## SYNOPSIS -**toolbox create** [*--distro DISTRO* | *-d DISTRO*] +**toolbox create** [*--authfile AUTHFILE*] + [*--distro DISTRO* | *-d DISTRO*] [*--image NAME* | *-i NAME*] [*--release RELEASE* | *-r RELEASE*] [*CONTAINER*] @@ -79,6 +80,14 @@ confusion. ## OPTIONS ## +**--authfile** FILE + +Path to a FILE with credentials for authenticating to the registry for private +images. The FILE is usually set using `podman login`, and will be used by +`podman pull` to get the image. + +The default location for FILE is `$XDG_RUNTIME_DIR/containers/auth.json`. + **--distro** DISTRO, **-d** DISTRO Create a toolbox container for a different operating system DISTRO than the @@ -120,6 +129,12 @@ $ toolbox create --distro fedora --release f36 $ toolbox create --image bar foo ``` +### Create a toolbox container from a custom image needing authentication + +``` +$ toolbox create --authfile ~/auth.json --image registry.example.com/bar +``` + ## SEE ALSO -`toolbox(1)`, `toolbox-init-container(1)`, `podman(1)`, `podman-create(1)` +`toolbox(1)`, `toolbox-init-container(1)`, `podman(1)`, `podman-create(1)`, `podman-login(1)`, `podman-pull(1)` diff --git a/playbooks/setup-env.yaml b/playbooks/setup-env.yaml index c18f82bbc..07e3aa67f 100644 --- a/playbooks/setup-env.yaml +++ b/playbooks/setup-env.yaml @@ -12,8 +12,10 @@ - flatpak-session-helper - golang - golang-github-cpuguy83-md2man + - httpd-tools - meson - ninja-build + - openssl - podman - skopeo - systemd diff --git a/src/cmd/create.go b/src/cmd/create.go index d819e3754..b4e923651 100644 --- a/src/cmd/create.go +++ b/src/cmd/create.go @@ -42,6 +42,7 @@ const ( var ( createFlags struct { + authFile string container string distro string image string @@ -67,6 +68,11 @@ var createCmd = &cobra.Command{ func init() { flags := createCmd.Flags() + flags.StringVar(&createFlags.authFile, + "authfile", + "", + "Path to a file with credentials for authenticating to the registry for private images") + flags.StringVarP(&createFlags.container, "container", "c", @@ -124,6 +130,12 @@ func create(cmd *cobra.Command, args []string) error { return errors.New("options --image and --release cannot be used together") } + if cmd.Flag("authfile").Changed { + if !utils.PathExists(createFlags.authFile) { + return fmt.Errorf("file %s not found", createFlags.authFile) + } + } + var container string var containerArg string @@ -717,7 +729,7 @@ func pullImage(image, release string) (bool, error) { defer s.Stop() } - if err := podman.Pull(imageFull); err != nil { + if err := podman.Pull(imageFull, createFlags.authFile); err != nil { var builder strings.Builder fmt.Fprintf(&builder, "failed to pull image %s\n", imageFull) fmt.Fprintf(&builder, "If it was a private image, log in with: podman login %s\n", domain) diff --git a/src/pkg/podman/podman.go b/src/pkg/podman/podman.go index 9099df1ea..6c4d9bc18 100644 --- a/src/pkg/podman/podman.go +++ b/src/pkg/podman/podman.go @@ -227,9 +227,18 @@ func IsToolboxImage(image string) (bool, error) { } // Pull pulls an image -func Pull(imageName string) error { +// +// authfile is a path to a JSON authentication file and is internally used only +// if it is not an empty string. +func Pull(imageName string, authfile string) error { logLevelString := LogLevel.String() - args := []string{"--log-level", logLevelString, "pull", imageName} + args := []string{"--log-level", logLevelString, "pull"} + + if authfile != "" { + args = append(args, []string{"--authfile", authfile}...) + } + + args = append(args, imageName) if err := shell.Run("podman", nil, nil, nil, args...); err != nil { return err diff --git a/test/system/000-setup.bats b/test/system/000-setup.bats index f1901bebc..ebd0e494f 100644 --- a/test/system/000-setup.bats +++ b/test/system/000-setup.bats @@ -19,4 +19,6 @@ load 'libs/helpers' _pull_and_cache_distro_image fedora "$((system_version-1))" || false _pull_and_cache_distro_image fedora "$((system_version-2))" || false fi + + _setup_docker_registry } diff --git a/test/system/101-create.bats b/test/system/101-create.bats index 8990fccda..97a50728b 100644 --- a/test/system/101-create.bats +++ b/test/system/101-create.bats @@ -117,3 +117,35 @@ teardown() { assert_line --index 0 "Error: release not found for non-default distribution $distro" assert [ ${#lines[@]} -eq 1 ] } + +@test "create: Try to create a container and pass a non-existent file to the --authfile option" { + local file="$BATS_RUN_TMPDIR/non-existent-file" + + run $TOOLBOX create --authfile "$file" + + assert_failure + assert_output "Error: file $file not found" +} + +@test "create: Create a container based on an image from locked registry using an authentication file" { + local authfile="$BATS_RUN_TMPDIR/authfile" + local image="fedora-toolbox:32" + + run $PODMAN login --authfile "$authfile" --username user --password user "$DOCKER_REG_URI" + assert_success + + run $TOOLBOX --assumeyes create --image "$DOCKER_REG_URI/$image" + + assert_failure + assert_line --index 0 "Error: failed to pull image $DOCKER_REG_URI/$image" + assert_line --index 1 "If it was a private image, log in with: podman login $DOCKER_REG_URI" + assert_line --index 2 "Use 'toolbox --verbose ...' for further details." + + run $TOOLBOX --assumeyes create --authfile "$authfile" --image "$DOCKER_REG_URI/$image" + + assert_success + assert_output "Created container: fedora-toolbox-32" + assert_output "Enter with: toolbox enter fedora-toolbox-32" + + rm "$authfile" +} diff --git a/test/system/999-teardown.bats b/test/system/999-teardown.bats index 5e2aaef11..c4f2742e7 100644 --- a/test/system/999-teardown.bats +++ b/test/system/999-teardown.bats @@ -4,5 +4,6 @@ load 'libs/helpers' @test "test suite: Teardown" { _clean_cached_images + _clean_docker_registry _clean_temporary_storage } diff --git a/test/system/README.md b/test/system/README.md index f7398b089..9423f4c03 100644 --- a/test/system/README.md +++ b/test/system/README.md @@ -13,6 +13,8 @@ Running them won't remove any existing containers or images. - `awk` - `bats` - `GNU coreutils` +- `httpd-tools` +- `openssl` - `podman` - `skopeo` - `toolbox` @@ -22,23 +24,6 @@ and consistency. The libraries are [bats-support](https://github.com/bats-core/b and [bats-assert](https://github.com/bats-core/bats-assert). These libraries are provided as git submodules in the `libs` directory. Make sure both are present. -## Convention - -- All tests should follow the nomenclature: `[command]: ...` -- When the test is expected to fail, start the test description with "Try - to..." -- When the test is to give a non obvious output, it should be put in parenthesis - at the end of the title - -Examples: - -* `@test "create: Create the default container"` -* `@test "rm: Try to remove a non-existent container"` - -- All the tests start with a clean system (no images or containers) to make sure - that there are no dependencies between tests and they are really isolated. Use - the `setup()` and `teardown()` functions for that purpose. - ## How to run the tests First, make sure you have all the dependencies installed. @@ -60,3 +45,38 @@ suite could look something like this: `PODMAN=/usr/libexec/podman TOOLBOX=./tool When running the tests, make sure the `test suite: [job]` jobs are successful. These jobs set up the whole environment and are a strict requirement for other jobs to run correctly. + +## Writing tests + +### Naming convention + +- All tests should follow the nomenclature: `[command]: ...` +- When the test is expected to fail, start the test description with "Try + to..." +- When the test is to give a non obvious output, it should be put in parenthesis + at the end of the title + +Examples: + +* `@test "create: Create the default container"` +* `@test "rm: Try to remove a non-existent container"` + +### Test case environment + +- All the tests start with a clean system (no images or containers) to make sure + that there are no dependencies between tests and they are really isolated. Use + the `setup()` and `teardown()` functions for that purpose. + +### Image registry + +- The system tests set up an OCI image registry for testing purposes. The registry + requires authentication. There is one account present: user (password: user) + +- The registry contains by default only one image: `fedora-toolbox:32` + +Example pull of the `fedora-toolbox:32` image: + +```bash +$PODMAN login --username user --password user "$DOCKER_REG_URI" +$PODMAN pull "$DOCKER_REG_URI/fedora-toolbox:32" +``` diff --git a/test/system/libs/helpers.bash b/test/system/libs/helpers.bash index 41084bfdd..738b123db 100644 --- a/test/system/libs/helpers.bash +++ b/test/system/libs/helpers.bash @@ -1,6 +1,7 @@ #!/usr/bin/env bash load 'libs/bats-support/load' +load 'libs/bats-assert/load' # Helpful globals readonly TEMP_BASE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/toolbox" @@ -9,6 +10,12 @@ readonly TEMP_STORAGE_DIR="${TEMP_BASE_DIR}/system-test-storage" readonly IMAGE_CACHE_DIR="${BATS_RUN_TMPDIR}/image-cache" readonly ROOTLESS_PODMAN_STORE_DIR="${TEMP_STORAGE_DIR}/storage" readonly PODMAN_STORE_CONFIG_FILE="${TEMP_STORAGE_DIR}/store.conf" +readonly DOCKER_REG_ROOT="${TEMP_STORAGE_DIR}/podman-registry-root" +readonly DOCKER_REG_DIR="${TEMP_STORAGE_DIR}/docker-registry" +readonly DOCKER_REG_CERTS_DIR="${BATS_RUN_TMPDIR}/certs" +readonly DOCKER_REG_AUTH_DIR="${BATS_RUN_TMPDIR}/auth" +readonly DOCKER_REG_URI="localhost:50000" +readonly DOCKER_REG_NAME="docker-registry" # Podman and Toolbox commands to run readonly PODMAN=${PODMAN:-podman} @@ -17,6 +24,7 @@ readonly SKOPEO=$(command -v skopeo) # Images declare -Ag IMAGES=([busybox]="quay.io/toolbox_tests/busybox" \ + [docker-reg]="quay.io/toolbox_tests/registry" \ [fedora]="registry.fedoraproject.org/fedora-toolbox" \ [rhel]="registry.access.redhat.com/ubi8/toolbox") @@ -36,6 +44,7 @@ function _setup_environment() { check_xdg_runtime_dir } + function _setup_containers_store() { mkdir -p ${TEMP_STORAGE_DIR} # Setup a storage config file for PODMAN @@ -119,6 +128,86 @@ function _clean_cached_images() { } +# Prepares a localy hosted image registry +# +# The registry is set up with Podman set to an alternative root. It won't +# affect other containers or images in the default root. +# +# Instructions taken from https://docs.docker.com/registry/deploying/ +function _setup_docker_registry() { + # Create certificates for HTTPS + # This is needed so that Podman does not have to be configured to work with + # HTTP-only registries + run mkdir -p "${DOCKER_REG_CERTS_DIR}" + assert_success + run openssl req \ + -newkey rsa:4096 \ + -nodes -sha256 \ + -keyout "${DOCKER_REG_CERTS_DIR}/domain.key" \ + -addext "subjectAltName= DNS:localhost" \ + -x509 \ + -days 365 \ + -subj '/' \ + -out "${DOCKER_REG_CERTS_DIR}/domain.crt" + assert_success + + # Add certificate to Podman's trusted certificates (rootless) + run mkdir -p "~/.config/containers/certs.d/${DOCKER_REG_URI}" + assert_success + run cp "${DOCKER_REG_CERTS_DIR}/domain.{crt,key}" "~/.config/containers/certs.d/${DOCKER_REG_URI}" + assert_success + + # Create a registry user + # username: user; password: user + run mkdir -p "${DOCKER_REG_AUTH_DIR}" + assert_success + run htpasswd -Bbc "${DOCKER_REG_AUTH_DIR}/htpasswd" user user + assert_success + + # Create separate Podman root + run mkdir -p "${DOCKER_REG_ROOT}" + assert_success + + # Pull Docker registry image + run $PODMAN --root "${DOCKER_REG_ROOT}" pull "${IMAGES[docker-reg]}" + assert_success + + # Create a Docker registry + run $PODMAN --root "${DOCKER_REG_ROOT}" run -d \ + --rm \ + --name "${DOCKER_REG_NAME}" \ + --privileged \ + -v "${DOCKER_REG_AUTH_DIR}:/auth" \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH="/auth/htpasswd" \ + -v "${DOCKER_REG_CERTS_DIR}":/certs \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \ + -p 50000:443 \ + "${IMAGES[docker-reg]}" + assert_success + + # Add fedora-toolbox:32 image to the registry + run $SKOPEO copy "dir:${IMAGE_CACHE_DIR}/fedora-toolbox-32" "docker://${DOCKER_REG_URI}/fedora-toolbox:32" + assert_success +} + + +# Stop, removes and cleans after a locally hosted Docker registry +function _clean_docker_registry() { + # Stop Docker registry container + podman --root "${DOCKER_REG_ROOT}" stop "${DOCKER_REG_NAME}" + # Remove Docker registry dirs + rm -rf "${DOCKER_REG_ROOT}" + rm -rf "${DOCKER_REG_DIR}" + # Remove dir with created registry certificates + rm -rf "~/.config/containers/certs.d/${DOCKER_REG_URI}" + +} + + # Copies an image from local storage to Podman's image store # # Call before creating any container. Network failures are not nice.