From ea09cb7b4e952af4bd20177cf4c4c5575567f364 Mon Sep 17 00:00:00 2001 From: Joe Stringer Date: Fri, 5 Jan 2024 13:56:53 -0800 Subject: [PATCH] images: Add docker pull support Add support to pull LVH images directly from the commandline so that users who don't need to customize the image can directly pull the required images directly rather than having to run another script or manually pull & decompress the images locally. Example usage: $ lvh images pull quay.io/lvh-images/kind:5.4-main Signed-off-by: Joe Stringer --- cmd/lvh/images/images.go | 2 +- cmd/lvh/images/pull.go | 45 ++++++++++++ pkg/images/pull.go | 145 +++++++++++++++++++++++++++++++++++++++ scripts/pull_image.sh | 28 -------- 4 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 cmd/lvh/images/pull.go create mode 100644 pkg/images/pull.go delete mode 100755 scripts/pull_image.sh diff --git a/cmd/lvh/images/images.go b/cmd/lvh/images/images.go index ee1f638..56d0d7f 100644 --- a/cmd/lvh/images/images.go +++ b/cmd/lvh/images/images.go @@ -13,6 +13,6 @@ func ImagesCommand() *cobra.Command { Short: "Build VM images", } - ret.AddCommand(BuildCmd(), ExampleCmd()) + ret.AddCommand(BuildCmd(), ExampleCmd(), PullCmd()) return ret } diff --git a/cmd/lvh/images/pull.go b/cmd/lvh/images/pull.go new file mode 100644 index 0000000..56e88a6 --- /dev/null +++ b/cmd/lvh/images/pull.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package images + +import ( + "context" + "fmt" + "os" + + "github.com/cilium/little-vm-helper/pkg/images" + "github.com/spf13/cobra" +) + +var ( + dirName string + cache bool +) + +func PullCmd() *cobra.Command { + + cmd := &cobra.Command{ + Use: "pull ", + Short: "Pull an image from an OCI repository", + Args: cobra.MinimumNArgs(1), + ArgAliases: []string{"imageURL"}, + RunE: func(cmd *cobra.Command, args []string) error { + if err := os.MkdirAll(dirName, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", dirName, err) + } + + _, err := images.PullImage(context.Background(), images.PullConf{ + Image: args[0], + TargetDir: dirName, + Cache: cache, + }) + return err + }, + } + + cmd.Flags().StringVar(&dirName, "dir", "_data", "directory to keep the images (images will be saved in images in /images)") + cmd.Flags().BoolVar(&cache, "cache", false, "cache a compressed version of the image") + + return cmd +} diff --git a/pkg/images/pull.go b/pkg/images/pull.go new file mode 100644 index 0000000..dc65a98 --- /dev/null +++ b/pkg/images/pull.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package images + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" +) + +type PullConf struct { + Image string + TargetDir string + Cache bool +} + +type PullResult struct { + Images []string +} + +// PullImage pulls an OCI image from a remote repository and decompresses it +// into a local directory. +func PullImage(ctx context.Context, conf PullConf) (*PullResult, error) { + result := &PullResult{ + Images: make([]string, 0, 1), + } + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("cannot establish client for image %s: %w", conf.Image, err) + } + defer cli.Close() + + remotePullReader, err := cli.ImagePull(ctx, conf.Image, types.ImagePullOptions{}) + if err != nil { + return nil, fmt.Errorf("cannot pull image %s: %w", conf.Image, err) + } + defer remotePullReader.Close() + + // Complete the image pull + // TODO: Take the output and overwrite each line in stdout for pretty + // status output instead of spamming the terminal with json + io.Copy(os.Stdout, remotePullReader) + + resp, err := cli.ContainerCreate(ctx, &container.Config{ + Image: conf.Image, + Tty: false, + }, nil, nil, nil, "") + if err != nil { + return nil, fmt.Errorf("cannot create container from %s: %w", conf.Image, err) + } + defer func() { + err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ + Force: true, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "could not clean up container %s: %s\n", resp.ID, err) + } + }() + + ctrImagesPath := filepath.Join("/", "data", "images") + imageReader, _, err := cli.CopyFromContainer(ctx, resp.ID, ctrImagesPath) + if err != nil { + return nil, fmt.Errorf("unable to locate images inside %s: %w", conf.Image, err) + } + defer imageReader.Close() + + tarReader := tar.NewReader(imageReader) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to read tar header in %s: %w", conf.Image, err) + } + + image, err := handleTarObject(ctx, tarReader, hdr, conf, resp.ID) + if err != nil { + return nil, err + } + if image != "" { + result.Images = append(result.Images, image) + } + } + + return result, nil +} + +func handleTarObject(ctx context.Context, tr *tar.Reader, hdr *tar.Header, conf PullConf, containerID string) (string, error) { + image := "" + + dstPath := filepath.Join(conf.TargetDir, hdr.Name) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(dstPath, 0755); err != nil { + return image, fmt.Errorf("failed to create directory %s: %w", dstPath, err) + } + case tar.TypeReg: + compressed := strings.HasSuffix(dstPath, ".zst") + if compressed { + image = strings.TrimSuffix(dstPath, ".zst") + cmd := exec.CommandContext(ctx, "zstd", "-d", "-", "-o", image) + cmd.Stdin = tr + + if _, err := cmd.Output(); err != nil { + var e *exec.ExitError + if errors.As(err, &e) { + fmt.Fprintf(os.Stderr, string(e.Stderr)) + } + return image, fmt.Errorf("failed during zst decompression of %s: %w", hdr.Name, err) + } + } + if conf.Cache || !compressed { + dst, err := os.Create(dstPath) + if err != nil { + return image, fmt.Errorf("failed to create file %s: %w", dstPath, err) + } + defer dst.Close() + + n, err := io.CopyN(dst, tr, hdr.Size) + if err != nil { + return image, fmt.Errorf("failed to copy %s from container %s: %w", dstPath, containerID, err) + } + if n != hdr.Size { + return image, fmt.Errorf("tar header reports file %s size %d, but only %d bytes were pulled", hdr.Name, hdr.Size, n) + } + } + default: + return image, fmt.Errorf("unexpected tar header type %d", hdr.Typeflag) + } + + return image, nil +} diff --git a/scripts/pull_image.sh b/scripts/pull_image.sh deleted file mode 100755 index 94acafe..0000000 --- a/scripts/pull_image.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -CTR="" -IMAGES_DIR="${IMAGES_DIR:-.}" - -cleanup() { - docker rm $CTR -} - -# $1 - image -main() { - if [ $# -ne 1 ]; then - >&2 echo "usage: $0 " - exit 1 - fi - - mkdir -p $IMAGES_DIR - CTR=$(docker create $1) - trap cleanup EXIT - docker cp $CTR:/data/images $IMAGES_DIR >/dev/null - - files=($(find $IMAGES_DIR/images/*.zst -type f)) - for f in ${files[@]}; do - zstd --decompress $f - done -} - -main "$@"