diff --git a/CHANGELOG.md b/CHANGELOG.md index ca250f3..f0fdfaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## `master` +- support for platform selection when syncing from multi-platform images (issue #43, *alpha* feature) +- raised default *Docker* API version to `1.41` - remediation of CVEs in dependencies: + [Improper Input Validation in GoGo Protobuf](https://github.com/advisories/GHSA-c3h9-896r-86jm) + [containerd CRI plugin: Insecure handling of image volumes](https://github.com/advisories/GHSA-crp2-qrr5-8pq7) diff --git a/Makefile b/Makefile index 33ffaaf..20e9cb2 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ endif TEST_ALPINE ?= y TEST_UBUNTU ?= y -TEST_CLEANUP = "127.0.0.1:5000/*/*/*/*" "*/*/*/busybox" \ +TEST_CLEANUP = "127.0.0.1:5000/*/*/*/*" "*/*/*/busybox*" \ "*/cloudrun/container/hello" "registry.hub.docker.com/library/busybox" \ "*/jenkins/jnlp-slave" "*/*/*/hello" diff --git a/README.md b/README.md index f55fb93..a12e767 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ skopeo: docker: # Docker host to use as the relay dockerhost: unix:///var/run/docker.sock - # Docker API version to use, defaults to 1.24 - api-version: 1.24 + # Docker API version to use, defaults to 1.41 + api-version: 1.41 # settings for image matching (see below) lister: @@ -66,18 +66,22 @@ tasks: skip-tls-verify: true # 'mappings' is a list of 'from':'to' pairs that define mappings of image - # paths in the source registry to paths in the destination; 'from' is - # required, while 'to' can be dropped if the path should remain the same as - # 'from'. Regular expressions are supported in both fields (read on below - # for more details). Additionally, the tags being synced for a mapping can - # be limited by providing a 'tags' list. This list may contain semver and - # regular expressions filters (see below). When omitted, all image tags are - # synced. + # paths in the source registry to paths in the destination: + # - 'from' is required, while 'to' can be dropped if the path should remain + # the same as 'from'. + # - Regular expressions are supported in both fields (read on below for + # more details). + # - The tags being synced for a mapping can be limited by providing a 'tags' + # list. This list may contain semver and regular expressions filters + # (see below). When omitted, all image tags are synced. + # - With 'platform', the image to sync from a multi-platform source image + # can be selected (see below). mappings: - from: test/image to: archive/test/image tags: ['0.1.0', '0.1.1'] - from: test/another-image + platform: linux/arm64/v8 ``` @@ -85,9 +89,10 @@ tasks: When syncing via a *Docker* relay, do not use the same *Docker* daemon for building local images (even better: don't use it for anything else but syncing). There is a risk that the reference to a locally built image clashes with the shorthand notation for a reference to an image on `docker.io`. E.g. if you built a local image `busybox`, then this would be indistinguishable from the shorthand `busybox` pointing to `docker.io/library/busybox`. One way to avoid this is to use `registry.hub.docker.com` instead of `docker.io` in references, which would never get shortened. If you're not syncing from/to `docker.io`, then all of this is not a concern. + ### Image Matching -The `mappings` section of a task can employ *Go* regular expressions for describing what images to sync, and how to change the destination path and name of an image. Details about how this works and examples can be found in this [design document](doc/design-image-matching.md). Note however that this is still an *alpha* feature, so things may not quite work as expected. Also keep in mind that regular expressions can be surprising at times, so it would be a good idea to try them out first in a *Go* playground. You may otherwise potentially sync large numbers of images, clogging your target registry, or running into rate limits. Feedback about this feature is encouraged! +The `mappings` section of a task can employ *Go* regular expressions for describing what images to sync, and how to change the destination path and name of an image. Details about how this works and examples can be found in this [design document](doc/design-image-matching.md). Note however that this is still a *beta* feature, so things may not quite work as expected. Also keep in mind that regular expressions can be surprising at times, so it would be a good idea to try them out first in a *Go* playground. You may otherwise potentially sync large numbers of images, clogging your target registry, or running into rate limits. Feedback about this feature is encouraged! ### Tag Filtering @@ -104,11 +109,20 @@ tags: This would sync all tags describing versions equal to or larger than `1.31.0`, but lower than `1.31.9`, via the `semver:` filter. The `regex:` filter additionally syncs any `1.26.`x image with suffix `-glibc`, `-uclibc`, or `-musl`. Finally, the verbatim tags `1.29.4` and `latest` are also synced. -Note that tag filtering is still an *alpha* feature. Also, the tags of an image need to conform to the *semver* specification *2.0.0* in order to be considered during filtering. The implementation uses the [blang/semver](https://github.com/blang/semver) lib. Have a look at their page or [the GoDoc](https://pkg.go.dev/github.com/blang/semver/v4) for more info on how to write *semver* filter expressions. Semver filtering tolerates and handles tags starting with a `v` prefix. Semver filter expressions however must not use a `v` prefix. Regex filters use standard *Go* regular expressions. When the first non-whitespace character after `regex:` is `!`, the filter will use inverted match. Keep in mind that when a regex contains a backslash, you need to place it inside single quotes to keep the YAML valid. +Note that tag filtering is still a *beta* feature. Also, the tags of an image need to conform to the *semver* specification *2.0.0* in order to be considered during filtering. The implementation uses the [blang/semver](https://github.com/blang/semver) lib. Have a look at their page or [the GoDoc](https://pkg.go.dev/github.com/blang/semver/v4) for more info on how to write *semver* filter expressions. Semver filtering tolerates and handles tags starting with a `v` prefix. Semver filter expressions however must not use a `v` prefix. Regex filters use standard *Go* regular expressions. When the first non-whitespace character after `regex:` is `!`, the filter will use inverted match. Keep in mind that when a regex contains a backslash, you need to place it inside single quotes to keep the YAML valid. You can add multiple `semver:` and `regex:` filters under `tags`. Note however that the filters are simply ORed, i.e. a tag is synced if it satisfies at least one of the items under `tags`, be it semver, regex, or verbatim. So this is not a filter chain. Also, no sanity checks are done on the filters, so care must be taken to avoid competing or contradicting filters that select all or nothing at all. +### Platform Selection (*Multi-Platform* Source Images) + +When the source image is a *multi-platform* image, the platform image adequate for the system on which *dregsy* runs is synced by default. Where this is not applicable, the desired platform can be specified via the `platform` setting, separately for each mapping. To sync all available platform images, `platform: all` can be used. Note however that this shorthand is only supported by the *Skopeo* relay. + +To sync a selection of platform images from the same multi-platform source image, several mappings with according `platform` settings can be defined. However, be careful not to map them into the same destination, i.e. use different `to` settings. Otherwise, the synced platform images will "overwrite" each other, with only the last image synced being available from the target repository. + +Note that platform selection is still an *alpha* feature. + + ### Repository Validation & Client Authentication with TLS When connecting to source and target repository servers, TLS validation is performed to verify the identity of a server. If you're using self-signed certificates for a repo server, or a server's certificate cannot be validated with the CA bundle available on your system, you need to provide the required CA certs. The *dregsy* *Docker* image includes the CA bundle that comes with the *Alpine* base image. Also, if a repo server requires client authentication, i.e. mutual TLS, you need to provide an appropriate client key & cert pair. diff --git a/cmd/dregsy/main_test.go b/cmd/dregsy/main_test.go index 68c1ebe..9647eb2 100644 --- a/cmd/dregsy/main_test.go +++ b/cmd/dregsy/main_test.go @@ -32,6 +32,18 @@ import ( "github.com/xelalexv/dregsy/internal/pkg/util" ) +// +var testPlatforms []string = []string{ + "linux/amd64", + "linux/386", + "linux/mips64le", + "linux/ppc64le", + "linux/arm/v5", + "linux/arm/v6", + "linux/arm/v7", + "linux/arm64/v8", +} + // func TestE2EOneoff(t *testing.T) { tryConfig(test.NewTestHelper(t), "e2e/base/oneoff.yaml", @@ -44,6 +56,12 @@ func TestE2EDocker(t *testing.T) { 1, 0, true, nil, test.GetParams()) } +// +func TestE2EDockerPlatform(t *testing.T) { + tryConfig(test.NewTestHelper(t), "e2e/base/docker-platform.yaml", + 1, 0, true, nil, test.GetParams()) +} + // func TestE2EDockerECR(t *testing.T) { registries.SkipIfECRNotConfigured(t) @@ -156,6 +174,18 @@ func TestE2ESkopeo(t *testing.T) { 1, 0, true, nil, test.GetParams()) } +// +func TestE2ESkopeoPlatform(t *testing.T) { + tryConfig(test.NewTestHelper(t), "e2e/base/skopeo-platform.yaml", + 1, 0, true, nil, test.GetParams()) +} + +// +func TestE2ESkopeoAllPlatforms(t *testing.T) { + tryConfig(test.NewTestHelper(t), "e2e/base/skopeo-platform-all.yaml", + 1, 0, true, nil, test.GetParams()) +} + // func TestE2ESkopeoECR(t *testing.T) { registries.SkipIfECRNotConfigured(t) @@ -328,6 +358,54 @@ func validateAgainstTaskMapping(th *test.TestHelper, c *sync.SyncConfig) { "", t.Target.SkipTLSVerify) th.AssertNoError(err) th.AssertEquivalentSlices(m.Tags, tags) + validatePlatforms(th, ref, t, m) + } + } +} + +// +func validatePlatforms(th *test.TestHelper, ref string, task *sync.Task, + mapping *sync.Mapping) { + + if mapping.Platform == "" { + return + } + + plts := make(map[string]bool) + + if mapping.Platform == "all" { + for _, p := range testPlatforms { + plts[p] = true + } + } else { + for _, p := range testPlatforms { + plts[p] = p == mapping.Platform + } + plts[mapping.Platform] = true + } + + for _, t := range mapping.Tags { + + for plt, exp := range plts { + + info, err := skopeo.Inspect( + fmt.Sprintf("%s:%s", ref, t), plt, "{{.Os}}/{{.Architecture}}", + util.DecodeJSONAuth(task.Target.GetAuth()), + "", task.Target.SkipTLSVerify) + th.AssertNoError(err) + + // FIXME: Skopeo inspect only shows OS and architecture, but not + // variant. Also, for platforms that are not present, it + // does not raise an error and instead returns info for the + // "default" platform. When testing syncing of a single + // platform, that's the default. + var os, arch string + if exp { + os, arch, _ = util.SplitPlatform(plt) + } else { + os, arch, _ = util.SplitPlatform(mapping.Platform) + } + th.AssertEqual(fmt.Sprintf("%s/%s", os, arch), info) } } } diff --git a/internal/pkg/relays/docker/docker.go b/internal/pkg/relays/docker/docker.go index 263e84d..18d2569 100644 --- a/internal/pkg/relays/docker/docker.go +++ b/internal/pkg/relays/docker/docker.go @@ -184,23 +184,27 @@ func match(filterRepo, filterPath, filterTag, ref string) (bool, error) { } // -func (dc *dockerClient) pullImage(ref string, allTags bool, auth string, +func (dc *dockerClient) pullImage(ref string, allTags bool, platform, auth string, verbose bool) error { opts := &types.ImagePullOptions{ All: allTags, RegistryAuth: auth, + Platform: platform, } rc, err := dc.client.ImagePull(context.Background(), ref, *opts) return dc.handleLog(rc, err, verbose) } // -func (dc *dockerClient) pushImage(image string, allTags bool, auth string, +func (dc *dockerClient) pushImage(image string, allTags bool, platform, auth string, verbose bool) error { opts := &types.ImagePushOptions{ All: allTags, RegistryAuth: auth, + // NOTE: Platform currently does not seem to be used by + // the Docker client lib + Platform: platform, } rc, err := dc.client.ImagePush(context.Background(), image, *opts) return dc.handleLog(rc, err, verbose) diff --git a/internal/pkg/relays/docker/dockerrelay.go b/internal/pkg/relays/docker/dockerrelay.go index dad892a..2b1d920 100644 --- a/internal/pkg/relays/docker/dockerrelay.go +++ b/internal/pkg/relays/docker/dockerrelay.go @@ -24,8 +24,8 @@ import ( "github.com/docker/docker/client" log "github.com/sirupsen/logrus" + "github.com/xelalexv/dregsy/internal/pkg/relays" "github.com/xelalexv/dregsy/internal/pkg/relays/skopeo" - "github.com/xelalexv/dregsy/internal/pkg/tags" "github.com/xelalexv/dregsy/internal/pkg/util" ) @@ -37,6 +37,18 @@ type RelayConfig struct { APIVersion string `yaml:"api-version"` } +// +type Support struct{} + +// +func (s *Support) Platform(p string) error { + if p == "all" { + return fmt.Errorf( + "relay '%s' does not support mappings with 'platform: all'", RelayID) + } + return nil +} + // type DockerRelay struct { client *dockerClient @@ -48,7 +60,7 @@ func NewDockerRelay(conf *RelayConfig, out io.Writer) (*DockerRelay, error) { relay := &DockerRelay{} dockerHost := client.DefaultDockerHost - apiVersion := "1.24" + apiVersion := "1.41" if conf != nil { if conf.DockerHost != "" { @@ -90,26 +102,31 @@ func (r *DockerRelay) Dispose() error { } // -func (r *DockerRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, - trgtRef, trgtAuth string, trgtSkipTLSVerify bool, ts *tags.TagSet, - verbose bool) error { +func (r *DockerRelay) Sync(opt *relays.SyncOptions) error { - log.WithField("ref", srcRef).Info("pulling source image") + log.WithFields(log.Fields{ + "ref": opt.SrcRef, + "platform": opt.Platform}).Info("pulling source image") + + if opt.Platform == "all" { + return fmt.Errorf("'Platform: all' sync option not supported") + } var tags []string var err error // When no tags are specified, a simple docker pull without a tag will get // all tags. So for Docker relay, we don't need to list tags in this case. - if !ts.IsEmpty() { + if !opt.Tags.IsEmpty() { srcCertDir := "" - repo, _, _ := util.SplitRef(srcRef) + repo, _, _ := util.SplitRef(opt.SrcRef) if repo != "" { srcCertDir = skopeo.CertsDirForRepo(repo) } - tags, err = ts.Expand(func() ([]string, error) { + tags, err = opt.Tags.Expand(func() ([]string, error) { return skopeo.ListAllTags( - srcRef, util.DecodeJSONAuth(srcAuth), srcCertDir, srcSkipTLSVerify) + opt.SrcRef, util.DecodeJSONAuth(opt.SrcAuth), + srcCertDir, opt.SrcSkipTLSVerify) }) if err != nil { @@ -118,15 +135,17 @@ func (r *DockerRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, } if len(tags) == 0 { - if err = r.pull(srcRef, srcAuth, true, verbose); err != nil { + if err = r.pull(opt.SrcRef, opt.Platform, opt.SrcAuth, + true, opt.Verbose); err != nil { return fmt.Errorf( - "error pulling source image '%s': %v", srcRef, err) + "error pulling source image '%s': %v", opt.SrcRef, err) } } else { for _, tag := range tags { - srcRefTagged := fmt.Sprintf("%s:%s", srcRef, tag) - if err = r.pull(srcRefTagged, srcAuth, false, verbose); err != nil { + srcRefTagged := fmt.Sprintf("%s:%s", opt.SrcRef, tag) + if err = r.pull(srcRefTagged, opt.Platform, opt.SrcAuth, + false, opt.Verbose); err != nil { return fmt.Errorf( "error pulling source image '%s': %v", srcRefTagged, err) } @@ -137,21 +156,19 @@ func (r *DockerRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, var srcImages []*image if len(tags) == 0 { - srcImages, err = r.list(srcRef) + srcImages, err = r.list(opt.SrcRef) if err != nil { - log.Error( - fmt.Errorf("error listing all tags of source image '%s': %v", - srcRef, err)) + log.Errorf("error listing all tags of source image '%s': %v", + opt.SrcRef, err) } } else { for _, tag := range tags { - srcRefTagged := fmt.Sprintf("%s:%s", srcRef, tag) + srcRefTagged := fmt.Sprintf("%s:%s", opt.SrcRef, tag) srcImageTagged, err := r.list(srcRefTagged) if err != nil { - log.Error( - fmt.Errorf("error listing source image '%s': %v", - srcRefTagged, err)) + log.Errorf( + "error listing source image '%s': %v", srcRefTagged, err) } srcImages = append(srcImages, srcImageTagged...) } @@ -161,16 +178,19 @@ func (r *DockerRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, log.Infof(" - %s", img.refWithTags()) } - log.WithField("ref", trgtRef).Info("setting tags for target image") + log.WithField("ref", opt.TrgtRef).Info("setting tags for target image") - _, err = r.tag(srcImages, trgtRef) + _, err = r.tag(srcImages, opt.TrgtRef) if err != nil { return fmt.Errorf("error setting tags: %v", err) } - log.WithField("ref", trgtRef).Info("pushing target image") + log.WithFields(log.Fields{ + "ref": opt.TrgtRef, + "platform": opt.Platform}).Info("pushing target image") - if err := r.push(trgtRef, trgtAuth, verbose); err != nil { + if err := r.push( + opt.TrgtRef, opt.Platform, opt.TrgtAuth, opt.Verbose); err != nil { return fmt.Errorf("error pushing target image: %v", err) } @@ -178,8 +198,8 @@ func (r *DockerRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, } // -func (r *DockerRelay) pull(ref, auth string, allTags, verbose bool) error { - return r.client.pullImage(ref, allTags, auth, verbose) +func (r *DockerRelay) pull(ref, platform, auth string, allTags, verbose bool) error { + return r.client.pullImage(ref, allTags, platform, auth, verbose) } // @@ -214,6 +234,6 @@ func (r *DockerRelay) tag(images []*image, targetRef string) ( } // -func (r *DockerRelay) push(ref, auth string, verbose bool) error { - return r.client.pushImage(ref, true, auth, verbose) +func (r *DockerRelay) push(ref, platform, auth string, verbose bool) error { + return r.client.pushImage(ref, true, platform, auth, verbose) } diff --git a/internal/pkg/relays/skopeo/skopeo.go b/internal/pkg/relays/skopeo/skopeo.go index bc3254a..6b86e11 100644 --- a/internal/pkg/relays/skopeo/skopeo.go +++ b/internal/pkg/relays/skopeo/skopeo.go @@ -26,6 +26,8 @@ import ( "strings" log "github.com/sirupsen/logrus" + + "github.com/xelalexv/dregsy/internal/pkg/util" ) const defaultSkopeoBinary = "skopeo" @@ -52,12 +54,41 @@ func CertsDirForRepo(r string) string { } // -func ListAllTags(ref, creds, certDir string, skipTLSVerify bool) ( - []string, error) { +func ListAllTags(ref, creds, certDir string, skipTLSVerify bool) ([]string, error) { + + ret, err := info([]string{"list-tags"}, ref, creds, certDir, skipTLSVerify) + if err != nil { + return nil, + fmt.Errorf("error listing image tags for ref '%s': %v", ref, err) + } + + list, err := decodeTagList(ret) + if err != nil { + return nil, err + } + return list.Tags, nil +} + +// +func Inspect(ref, platform, format, creds, certDir string, skipTLSVerify bool) ( + string, error) { + + cmd := addPlatformOverrides([]string{"inspect"}, platform) + if format != "" { + cmd = append(cmd, fmt.Sprintf("--format=%s", format)) + } - cmd := []string{ - "list-tags", + if insp, err := info(cmd, ref, creds, certDir, skipTLSVerify); err != nil { + return "", fmt.Errorf( + "error inspecting image for ref '%s': %v", ref, err) + } else { + return strings.TrimSpace(string(insp)), nil } +} + +// +func info(cmd []string, ref, creds, certDir string, skipTLSVerify bool) ( + []byte, error) { if skipTLSVerify { cmd = append(cmd, "--tls-verify=false") @@ -77,16 +108,29 @@ func ListAllTags(ref, creds, certDir string, skipTLSVerify bool) ( bufErr := new(bytes.Buffer) if err := runSkopeo(bufOut, bufErr, true, cmd...); err != nil { - return nil, - fmt.Errorf("error listing image tags for ref '%s': %s, %v", - ref, bufErr.String(), err) + return nil, fmt.Errorf("%s, %v", bufErr.String(), err) } - list, err := decodeTagList(bufOut.Bytes()) - if err != nil { - return nil, err + return bufOut.Bytes(), nil +} + +// +func addPlatformOverrides(cmd []string, platform string) []string { + + if platform != "" { + os, arch, variant := util.SplitPlatform(platform) + if os != "" { + cmd = append(cmd, fmt.Sprintf("--override-os=%s", os)) + } + if arch != "" { + cmd = append(cmd, fmt.Sprintf("--override-arch=%s", arch)) + } + if variant != "" { + cmd = append(cmd, fmt.Sprintf("--override-variant=%s", variant)) + } } - return list.Tags, nil + + return cmd } // diff --git a/internal/pkg/relays/skopeo/skopeorelay.go b/internal/pkg/relays/skopeo/skopeorelay.go index f47a3c0..3e71540 100644 --- a/internal/pkg/relays/skopeo/skopeorelay.go +++ b/internal/pkg/relays/skopeo/skopeorelay.go @@ -23,7 +23,7 @@ import ( log "github.com/sirupsen/logrus" - "github.com/xelalexv/dregsy/internal/pkg/tags" + "github.com/xelalexv/dregsy/internal/pkg/relays" "github.com/xelalexv/dregsy/internal/pkg/util" ) @@ -35,6 +35,14 @@ type RelayConfig struct { CertsDir string `yaml:"certs-dir"` } +// +type Support struct{} + +// +func (s *Support) Platform(p string) error { + return nil +} + // type SkopeoRelay struct { wrOut io.Writer @@ -62,12 +70,15 @@ func NewSkopeoRelay(conf *RelayConfig, out io.Writer) *SkopeoRelay { // func (r *SkopeoRelay) Prepare() error { + bufOut := new(bytes.Buffer) if err := runSkopeo(bufOut, nil, true, "--version"); err != nil { return fmt.Errorf("cannot execute skopeo: %v", err) } + log.Info(bufOut.String()) log.WithField("relay", RelayID).Info("relay ready") + return nil } @@ -77,32 +88,30 @@ func (r *SkopeoRelay) Dispose() error { } // -func (r *SkopeoRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, - destRef, destAuth string, destSkipTLSVerify bool, ts *tags.TagSet, - verbose bool) error { +func (r *SkopeoRelay) Sync(opt *relays.SyncOptions) error { - srcCreds := util.DecodeJSONAuth(srcAuth) - destCreds := util.DecodeJSONAuth(destAuth) + srcCreds := util.DecodeJSONAuth(opt.SrcAuth) + destCreds := util.DecodeJSONAuth(opt.TrgtAuth) cmd := []string{ "--insecure-policy", "copy", } - if srcSkipTLSVerify { + if opt.SrcSkipTLSVerify { cmd = append(cmd, "--src-tls-verify=false") } - if destSkipTLSVerify { + if opt.TrgtSkipTLSVerify { cmd = append(cmd, "--dest-tls-verify=false") } srcCertDir := "" - repo, _, _ := util.SplitRef(srcRef) + repo, _, _ := util.SplitRef(opt.SrcRef) if repo != "" { srcCertDir = CertsDirForRepo(repo) cmd = append(cmd, fmt.Sprintf("--src-cert-dir=%s", srcCertDir)) } - repo, _, _ = util.SplitRef(destRef) + repo, _, _ = util.SplitRef(opt.TrgtRef) if repo != "" { cmd = append(cmd, fmt.Sprintf( "--dest-cert-dir=%s/%s", certsBaseDir, withoutPort(repo))) @@ -115,8 +124,8 @@ func (r *SkopeoRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, cmd = append(cmd, fmt.Sprintf("--dest-creds=%s", destCreds)) } - tags, err := ts.Expand(func() ([]string, error) { - return ListAllTags(srcRef, srcCreds, srcCertDir, srcSkipTLSVerify) + tags, err := opt.Tags.Expand(func() ([]string, error) { + return ListAllTags(opt.SrcRef, srcCreds, srcCertDir, opt.SrcSkipTLSVerify) }) if err != nil { @@ -124,11 +133,25 @@ func (r *SkopeoRelay) Sync(srcRef, srcAuth string, srcSkipTLSVerify bool, } errs := false - for _, tag := range tags { - log.WithField("tag", tag).Info("syncing tag") - if err := runSkopeo(r.wrOut, r.wrOut, verbose, append(cmd, - fmt.Sprintf("docker://%s:%s", srcRef, tag), - fmt.Sprintf("docker://%s:%s", destRef, tag))...); err != nil { + + for _, t := range tags { + + log.WithFields( + log.Fields{"tag": t, "platform": opt.Platform}).Info("syncing tag") + + rc := append(cmd, + fmt.Sprintf("docker://%s:%s", opt.SrcRef, t), + fmt.Sprintf("docker://%s:%s", opt.TrgtRef, t)) + + switch opt.Platform { + case "": + case "all": + rc = append(rc, "--all") + default: + rc = addPlatformOverrides(rc, opt.Platform) + } + + if err := runSkopeo(r.wrOut, r.wrOut, opt.Verbose, rc...); err != nil { log.Error(err) errs = true } diff --git a/internal/pkg/relays/types.go b/internal/pkg/relays/types.go new file mode 100644 index 0000000..e1277ef --- /dev/null +++ b/internal/pkg/relays/types.go @@ -0,0 +1,42 @@ +/* + Copyright 2022 Alexander Vollschwitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package relays + +import ( + "github.com/xelalexv/dregsy/internal/pkg/tags" +) + +// +type SyncOptions struct { + // + SrcRef string + SrcAuth string + SrcSkipTLSVerify bool + // + TrgtRef string + TrgtAuth string + TrgtSkipTLSVerify bool + // + Tags *tags.TagSet + Platform string + Verbose bool +} + +// +type Support interface { + Platform(p string) error +} diff --git a/internal/pkg/sync/config.go b/internal/pkg/sync/config.go index 875fd8c..fc063a4 100644 --- a/internal/pkg/sync/config.go +++ b/internal/pkg/sync/config.go @@ -25,6 +25,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/xelalexv/dregsy/internal/pkg/relays" "github.com/xelalexv/dregsy/internal/pkg/relays/docker" "github.com/xelalexv/dregsy/internal/pkg/relays/skopeo" ) @@ -44,6 +45,20 @@ type SyncConfig struct { Tasks []*Task `yaml:"tasks"` } +// +func (c *SyncConfig) ValidateSupport(s relays.Support) error { + + for _, t := range c.Tasks { + for _, m := range t.Mappings { + if err := s.Platform(m.Platform); err != nil { + return err + } + } + } + + return nil +} + // func (c *SyncConfig) validate() error { diff --git a/internal/pkg/sync/mapping.go b/internal/pkg/sync/mapping.go index 9e2fd39..a0f155b 100644 --- a/internal/pkg/sync/mapping.go +++ b/internal/pkg/sync/mapping.go @@ -30,9 +30,10 @@ const RegexpPrefix = "regex:" // type Mapping struct { - From string `yaml:"from"` - To string `yaml:"to"` - Tags []string `yaml:"tags"` + From string `yaml:"from"` + To string `yaml:"to"` + Tags []string `yaml:"tags"` + Platform string `yaml:"platform"` // fromFilter *regexp.Regexp toFilter *regexp.Regexp diff --git a/internal/pkg/sync/sync.go b/internal/pkg/sync/sync.go index c2403aa..967b32a 100644 --- a/internal/pkg/sync/sync.go +++ b/internal/pkg/sync/sync.go @@ -25,18 +25,16 @@ import ( log "github.com/sirupsen/logrus" + "github.com/xelalexv/dregsy/internal/pkg/relays" "github.com/xelalexv/dregsy/internal/pkg/relays/docker" "github.com/xelalexv/dregsy/internal/pkg/relays/skopeo" - "github.com/xelalexv/dregsy/internal/pkg/tags" ) // type Relay interface { Prepare() error Dispose() error - Sync(srcRef, srcAuth string, srcSkiptTLSVerify bool, - trgtRef, trgtAuth string, trgtSkiptTLSVerify bool, - tags *tags.TagSet, verbose bool) error + Sync(opt *relays.SyncOptions) error } // @@ -57,12 +55,16 @@ func New(conf *SyncConfig) (*Sync, error) { switch conf.Relay { case docker.RelayID: - relay, err = docker.NewDockerRelay( - conf.Docker, log.StandardLogger().WriterLevel(log.DebugLevel)) + if err = conf.ValidateSupport(&docker.Support{}); err == nil { + relay, err = docker.NewDockerRelay( + conf.Docker, log.StandardLogger().WriterLevel(log.DebugLevel)) + } case skopeo.RelayID: - relay = skopeo.NewSkopeoRelay( - conf.Skopeo, log.StandardLogger().WriterLevel(log.DebugLevel)) + if err = conf.ValidateSupport(&skopeo.Support{}); err == nil { + relay = skopeo.NewSkopeoRelay( + conf.Skopeo, log.StandardLogger().WriterLevel(log.DebugLevel)) + } default: err = fmt.Errorf("relay type '%s' not supported", conf.Relay) @@ -210,9 +212,16 @@ func (s *Sync) syncTask(t *Task) { break } - if err := s.relay.Sync(src, t.Source.GetAuth(), t.Source.SkipTLSVerify, - trgt, t.Target.GetAuth(), t.Target.SkipTLSVerify, m.tagSet, - t.Verbose); err != nil { + if err := s.relay.Sync(&relays.SyncOptions{ + SrcRef: src, + SrcAuth: t.Source.GetAuth(), + SrcSkipTLSVerify: t.Source.SkipTLSVerify, + TrgtRef: trgt, + TrgtAuth: t.Target.GetAuth(), + TrgtSkipTLSVerify: t.Target.SkipTLSVerify, + Tags: m.tagSet, + Platform: m.Platform, + Verbose: t.Verbose}); err != nil { log.Error(err) t.fail(true) } diff --git a/internal/pkg/sync/sync_test.go b/internal/pkg/sync/sync_test.go new file mode 100644 index 0000000..bfd8168 --- /dev/null +++ b/internal/pkg/sync/sync_test.go @@ -0,0 +1,59 @@ +/* + Copyright 2020 Alexander Vollschwitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sync + +import ( + "testing" + + "github.com/xelalexv/dregsy/internal/pkg/test" +) + +// +func TestInvalidSync(t *testing.T) { + + th := test.NewTestHelper(t) + + // mappings + trySync(th, "config/docker-platform-all.yaml", + "relay 'docker' does not support mappings with 'platform: all'") +} + +// +func trySync(th *test.TestHelper, file, err string) (*Sync, error) { + + test.StackTraceDepth = 2 + defer func() { test.StackTraceDepth = 1 }() + + c, e := LoadConfig(th.GetFixture(file)) + th.AssertNoError(e) + th.AssertNotNil(c) + + s, e := New(c) + if s != nil { + defer func() { s.Dispose() }() + } + + if err != "" { + th.AssertError(e, err) + th.AssertNil(s) + } else { + th.AssertNoError(e) + th.AssertNotNil(s) + } + + return s, e +} diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 09551e2..f3a102d 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -49,6 +49,29 @@ func SplitRef(ref string) (repo, path, tag string) { return } +// +func SplitPlatform(p string) (os, arch, variant string) { + + ix := strings.Index(p, "/") + + if ix == -1 { + os = p + arch = "" + } else { + os = p[:ix] + arch = p[ix+1:] + } + + ix = strings.Index(arch, "/") + + if ix > -1 { + variant = arch[ix+1:] + arch = arch[:ix] + } + + return +} + // func CompileRegex(v string, lineMatch bool) (*regexp.Regexp, error) { if lineMatch { diff --git a/test/fixtures/config/docker-platform-all.yaml b/test/fixtures/config/docker-platform-all.yaml new file mode 100644 index 0000000..54e5bc9 --- /dev/null +++ b/test/fixtures/config/docker-platform-all.yaml @@ -0,0 +1,18 @@ +relay: docker + +docker: + dockerhost: unix:///var/run/docker.sock + +tasks: +- name: test-platform-all + interval: 30 + verbose: true + source: + registry: registry.hub.docker.com + target: + registry: 127.0.0.1:5000 + mappings: + - from: library/busybox + to: docker/library/busybox + tags: ['latest'] + platform: all diff --git a/test/fixtures/config/skopeo-valid.yaml b/test/fixtures/config/skopeo-valid.yaml index 5eaaa57..76bb823 100644 --- a/test/fixtures/config/skopeo-valid.yaml +++ b/test/fixtures/config/skopeo-valid.yaml @@ -18,3 +18,4 @@ tasks: - from: library/busybox to: skopeo/library/busybox tags: ['1.29.2', '1.29.3', 'latest'] + platform: linux/arm/v6 diff --git a/test/fixtures/e2e/base/docker-ecr.yaml b/test/fixtures/e2e/base/docker-ecr.yaml index 79380d9..ab4e13f 100644 --- a/test/fixtures/e2e/base/docker-ecr.yaml +++ b/test/fixtures/e2e/base/docker-ecr.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-ecr diff --git a/test/fixtures/e2e/base/docker-gar-noauth.yaml b/test/fixtures/e2e/base/docker-gar-noauth.yaml index c2738e1..3f1ff5d 100644 --- a/test/fixtures/e2e/base/docker-gar-noauth.yaml +++ b/test/fixtures/e2e/base/docker-gar-noauth.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-gar-noauth diff --git a/test/fixtures/e2e/base/docker-gar.yaml b/test/fixtures/e2e/base/docker-gar.yaml index 9d8a52f..68da90d 100644 --- a/test/fixtures/e2e/base/docker-gar.yaml +++ b/test/fixtures/e2e/base/docker-gar.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-gar diff --git a/test/fixtures/e2e/base/docker-gcr-noauth.yaml b/test/fixtures/e2e/base/docker-gcr-noauth.yaml index 3367694..8164ad3 100644 --- a/test/fixtures/e2e/base/docker-gcr-noauth.yaml +++ b/test/fixtures/e2e/base/docker-gcr-noauth.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-gcr-noauth diff --git a/test/fixtures/e2e/base/docker-gcr.yaml b/test/fixtures/e2e/base/docker-gcr.yaml index cf83050..519d189 100644 --- a/test/fixtures/e2e/base/docker-gcr.yaml +++ b/test/fixtures/e2e/base/docker-gcr.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-gcr diff --git a/test/fixtures/e2e/base/docker-platform.yaml b/test/fixtures/e2e/base/docker-platform.yaml new file mode 100644 index 0000000..5803d10 --- /dev/null +++ b/test/fixtures/e2e/base/docker-platform.yaml @@ -0,0 +1,26 @@ +relay: docker + +docker: + dockerhost: {{ .DockerHost }} + +tasks: +- name: test-platform + interval: 30 + verbose: true + source: + registry: registry.hub.docker.com + auth: {{ .DockerhubAuth }} + target: + registry: 127.0.0.1:5000 + auth: {{ .LocalAuth }} + # not actually supported for Docker relay, but need this for validation + skip-tls-verify: true + mappings: + - from: library/busybox + to: base-docker/library/busybox-arm64 + tags: ['latest'] + platform: linux/arm64/v8 + - from: library/busybox + to: base-docker/library/busybox-amd64 + tags: ['latest'] + platform: linux/amd64 diff --git a/test/fixtures/e2e/base/docker.yaml b/test/fixtures/e2e/base/docker.yaml index 82611fb..9864cca 100644 --- a/test/fixtures/e2e/base/docker.yaml +++ b/test/fixtures/e2e/base/docker.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-docker diff --git a/test/fixtures/e2e/base/oneoff.yaml b/test/fixtures/e2e/base/oneoff.yaml index a739a25..6ffe6f7 100644 --- a/test/fixtures/e2e/base/oneoff.yaml +++ b/test/fixtures/e2e/base/oneoff.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-docker diff --git a/test/fixtures/e2e/base/skopeo-platform-all.yaml b/test/fixtures/e2e/base/skopeo-platform-all.yaml new file mode 100644 index 0000000..4d63b30 --- /dev/null +++ b/test/fixtures/e2e/base/skopeo-platform-all.yaml @@ -0,0 +1,18 @@ +relay: skopeo + +tasks: +- name: test-platform-all + interval: 30 + verbose: true + source: + registry: registry.hub.docker.com + auth: {{ .DockerhubAuth }} + target: + registry: 127.0.0.1:5000 + auth: {{ .LocalAuth }} + skip-tls-verify: true + mappings: + - from: library/busybox + to: base-skopeo/library/busybox-all + tags: ['latest'] + platform: all diff --git a/test/fixtures/e2e/base/skopeo-platform.yaml b/test/fixtures/e2e/base/skopeo-platform.yaml new file mode 100644 index 0000000..8eb82dd --- /dev/null +++ b/test/fixtures/e2e/base/skopeo-platform.yaml @@ -0,0 +1,22 @@ +relay: skopeo + +tasks: +- name: test-platform + interval: 30 + verbose: true + source: + registry: registry.hub.docker.com + auth: {{ .DockerhubAuth }} + target: + registry: 127.0.0.1:5000 + auth: {{ .LocalAuth }} + skip-tls-verify: true + mappings: + - from: library/busybox + to: base-skopeo/library/busybox-arm64 + tags: ['latest'] + platform: linux/arm64/v8 + - from: library/busybox + to: base-skopeo/library/busybox-amd64 + tags: ['latest'] + platform: linux/amd64 diff --git a/test/fixtures/e2e/mapping/docker-dh-search.yaml b/test/fixtures/e2e/mapping/docker-dh-search.yaml index 7fa5fa5..7931315 100644 --- a/test/fixtures/e2e/mapping/docker-dh-search.yaml +++ b/test/fixtures/e2e/mapping/docker-dh-search.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 lister: maxItems: 50 diff --git a/test/fixtures/e2e/mapping/docker-dockerhub.yaml b/test/fixtures/e2e/mapping/docker-dockerhub.yaml index ee830c5..24266a3 100644 --- a/test/fixtures/e2e/mapping/docker-dockerhub.yaml +++ b/test/fixtures/e2e/mapping/docker-dockerhub.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 lister: maxItems: -1 diff --git a/test/fixtures/e2e/mapping/docker-ecr.yaml b/test/fixtures/e2e/mapping/docker-ecr.yaml index 0400a43..907e031 100644 --- a/test/fixtures/e2e/mapping/docker-ecr.yaml +++ b/test/fixtures/e2e/mapping/docker-ecr.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 lister: maxItems: 100 diff --git a/test/fixtures/e2e/mapping/docker-local.yaml b/test/fixtures/e2e/mapping/docker-local.yaml index 5a36e65..293d1e5 100644 --- a/test/fixtures/e2e/mapping/docker-local.yaml +++ b/test/fixtures/e2e/mapping/docker-local.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-local # depends on results of docker-dockerhub test diff --git a/test/fixtures/e2e/tagsets/docker-range.yaml b/test/fixtures/e2e/tagsets/docker-range.yaml index f210646..6f7514a 100644 --- a/test/fixtures/e2e/tagsets/docker-range.yaml +++ b/test/fixtures/e2e/tagsets/docker-range.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: - name: test-docker-range diff --git a/test/fixtures/e2e/tagsets/docker-regex.yaml b/test/fixtures/e2e/tagsets/docker-regex.yaml index 3430ac9..81f06b4 100644 --- a/test/fixtures/e2e/tagsets/docker-regex.yaml +++ b/test/fixtures/e2e/tagsets/docker-regex.yaml @@ -2,7 +2,6 @@ relay: docker docker: dockerhost: {{ .DockerHost }} - api-version: 1.24 tasks: