#22: tags filtering with semver and regex

Makefile

@@ -110,6 +110,9 @@ endif
 TEST_ALPINE ?= y
 TEST_UBUNTU ?= y
 
+TEST_CLEANUP = "127.0.0.1:5000/*/*/*/*" "*/*/*/busybox" \
+		"*/cloudrun/container/hello" "registry.hub.docker.com/library/busybox"
+
 export
 
 #
@@ -123,7 +126,7 @@ help:
 
 
 .PHONY: release
-release: clean rmi rmitest dregsy imgdregsy imgtests tests registrydown
+release: clean rmi dregsy imgdregsy imgtests tests registrydown
 #	clean, do an isolated build, create container images, and test
 #
 
@@ -186,8 +189,7 @@ rmi:
 rmitest:
 #	remove all test-related container images
 #
-	$(call utils, remove_test_images \
-		"127.0.0.1:5000/*/*/*/*" "*/*/*/busybox" "*/cloudrun/container/hello")
+	$(call utils, remove_test_images $(TEST_CLEANUP))
 
 
 .PHONY: tests
@@ -199,10 +201,12 @@ ifeq (,$(wildcard .makerc))
 	$(warning ***** Missing .makerc! Some tests may be skipped or fail!)
 endif
 ifeq ($(TEST_ALPINE),y)
+	$(call utils, remove_test_images $(TEST_CLEANUP))
 	$(call utils, registry_restart)
 	$(call utils, run_tests alpine)
 endif
 ifeq ($(TEST_UBUNTU),y)
+	$(call utils, remove_test_images $(TEST_CLEANUP))
 	$(call utils, registry_restart)
 	$(call utils, run_tests ubuntu)
 endif

README.md

@@ -70,7 +70,8 @@ tasks:
     # required, while 'to' can be dropped if the path should remain the same as
     # 'from'. Regular expressions are supported in both fields (read on below
     # for more details). Additionally, the tags being synced for a mapping can
-    # be limited by providing a 'tags' list. When omitted, all image tags are
+    # be limited by providing a 'tags' list. This list may contain semver and
+    # regular expressions filters (see below). When omitted, all image tags are
     # synced.
     mappings:
       - from: test/image
@@ -88,6 +89,26 @@ When syncing via a *Docker* relay, do not use the same *Docker* daemon for build
 
 The `mappings` section of a task can employ *Go* regular expressions for describing what images to sync, and how to change the destination path and name of an image. Details about how this works and examples can be found in this [design document](doc/design-image-matching.md). Note however that this is still an *alpha* feature, so things may not quite work as expected. Also keep in mind that regular expressions can be surprising at times, so it would be a good idea to try them out first in a *Go* playground. You may otherwise potentially sync large numbers of images, clogging your target registry, or running into rate limits. Feedback about this feature is encouraged! 
 
+
+### Tag Filtering
+
+The `tags` list of a task can use *semver* and regular expression filters, so you can do something like this:
+
+```yaml
+tags:
+  - 'semver: >=1.31.0 <1.31.9'
+  - 'regex: 1\.26\.[0-9]-(glibc|uclibc|musl)'
+  - '1.29.4'
+  - 'latest'
+```
+
+This would sync all tags describing versions equal to or larger than `1.31.0`, but lower than `1.31.9`, via the `semver:` filter. The `regex:` filter additionally syncs any `1.26.`x image with suffix `-glibc`, `-uclibc`, or `-musl`. Finally, the verbatim tags `1.29.4` and `latest` are also synced.
+
+Note that the tags of an image need to conform to the *semver* specification *2.0.0* in order to be considered during filtering. The implementation uses the [blang/semver](https://github.com/blang/semver) lib. Have a look at their page or [the GoDoc](https://pkg.go.dev/github.com/blang/semver/v4) for more info on how to write *semver* filter expressions. Semver filtering tolerates and handles tags starting with a `v` prefix. Semver filter expressions however must not use a `v` prefix. Regex filters use standard *Go* regular expressions. When the first non-whitespace character after `regex:` is `!`, the filter will use inverted match. Keep in mind that when a regex contains a backslash, you need to place it inside single quotes to keep the YAML valid.
+
+You can add multiple `semver:` and `regex:` filters under `tags`. Note however that the filters are simply ORed, i.e. a tag is synced if it satisfies at least one of the items under `tags`, be it semver, regex, or verbatim. So this is not a filter chain. Also, no sanity checks are done on the filters, so care must be taken to avoid competing or contradicting filters that select all or nothing at all.
+
+
 ### Repository Validation & Client Authentication with TLS
 
 When connecting to source and target repository servers, TLS validation is performed to verify the identity of a server. If you're using self-signed certificates for a repo server, or a server's certificate cannot be validated with the CA bundle available on your system, you need to provide the required CA certs. The *dregsy* *Docker* image includes the CA bundle that comes with the *Alpine* base image. Also, if a repo server requires client authentication, i.e. mutual TLS, you need to provide an appropriate client key & cert pair.

cmd/dregsy/main_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/xelalexv/dregsy/internal/pkg/sync"
 	"github.com/xelalexv/dregsy/internal/pkg/test"
 	"github.com/xelalexv/dregsy/internal/pkg/test/registries"
+	"github.com/xelalexv/dregsy/internal/pkg/util"
 )
 
 //
@@ -85,6 +86,70 @@ func TestE2EDockerGARNoAuth(t *testing.T) {
 		1, 0, true, nil, test.GetParams())
 }
 
+//
+func TestE2EDockerMappingDockerhub(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-dockerhub.yaml",
+		0, 0, true, map[string][]string{
+			"mapping-docker/dh/xelalex/dregsy-dummy-public":  {"latest"},
+			"mapping-docker/dh/xelalex/dregsy-dummy-private": {"latest"},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2EDockerMappingLocal(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-local.yaml",
+		0, 0, true, map[string][]string{
+			"mapping-docker/dh-copy/xelalex/dregsy-dummy-public":  {"latest"},
+			"mapping-docker/dh-copy/xelalex/dregsy-dummy-private": {"latest"},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2EDockerMappingDockerhubSearch(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-dh-search.yaml",
+		0, 0, true, map[string][]string{
+			"mapping-docker/dh/other-jenkins/jnlp-slave": {"latest"},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2EDockerMappingECR(t *testing.T) {
+	registries.SkipIfECRNotConfigured(t)
+	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-ecr.yaml",
+		0, 0, true, map[string][]string{
+			"mapping-docker/ecr/kubika/brucket":       {"v0.0.1"},
+			"mapping-docker/ecr/kubika/brucket-shell": {"v0.0.1"},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2EDockerTagSetsRange(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/tagsets/docker-range.yaml",
+		0, 0, true, map[string][]string{
+			"tagsets-docker/range/busybox": {
+				"latest", "1.31", "1.31.0", "1.31.1-musl", "1.31.1",
+				"1.31.1-uclibc", "1.31.1-glibc",
+			},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2EDockerTagSetsRegex(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/tagsets/docker-regex.yaml",
+		0, 0, true, map[string][]string{
+			"tagsets-docker/regex/busybox": {
+				"1.26.1-musl", "1.26.1-glibc", "1.26.1-uclibc",
+			},
+			"tagsets-docker/regexinv/busybox": {"1.26.1-uclibc"},
+		},
+		test.GetParams())
+}
+
 //
 func TestE2ESkopeo(t *testing.T) {
 	tryConfig(test.NewTestHelper(t), "e2e/base/skopeo.yaml",
@@ -133,46 +198,6 @@ func TestE2ESkopeoGARNoAuth(t *testing.T) {
 		1, 0, true, nil, test.GetParams())
 }
 
-//
-func TestE2EDockerMappingDockerhub(t *testing.T) {
-	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-dockerhub.yaml",
-		0, 0, true, map[string][]string{
-			"mapping-docker/dh/xelalex/dregsy-dummy-public":  {"latest"},
-			"mapping-docker/dh/xelalex/dregsy-dummy-private": {"latest"},
-		},
-		test.GetParams())
-}
-
-//
-func TestE2EDockerMappingLocal(t *testing.T) {
-	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-local.yaml",
-		0, 0, true, map[string][]string{
-			"mapping-docker/dh-copy/xelalex/dregsy-dummy-public":  {"latest"},
-			"mapping-docker/dh-copy/xelalex/dregsy-dummy-private": {"latest"},
-		},
-		test.GetParams())
-}
-
-//
-func TestE2EDockerMappingDockerhubSearch(t *testing.T) {
-	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-dh-search.yaml",
-		0, 0, true, map[string][]string{
-			"mapping-docker/dh/other-jenkins/jnlp-slave": {"latest"},
-		},
-		test.GetParams())
-}
-
-//
-func TestE2EDockerMappingECR(t *testing.T) {
-	registries.SkipIfECRNotConfigured(t)
-	tryConfig(test.NewTestHelper(t), "e2e/mapping/docker-ecr.yaml",
-		0, 0, true, map[string][]string{
-			"mapping-docker/ecr/kubika/brucket":       {"v0.0.1"},
-			"mapping-docker/ecr/kubika/brucket-shell": {"v0.0.1"},
-		},
-		test.GetParams())
-}
-
 //
 func TestE2ESkopeoMappingDockerhub(t *testing.T) {
 	tryConfig(test.NewTestHelper(t), "e2e/mapping/skopeo-dockerhub.yaml",
@@ -213,6 +238,30 @@ func TestE2ESkopeoMappingECR(t *testing.T) {
 		test.GetParams())
 }
 
+//
+func TestE2ESkopeoTagSetsRange(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/tagsets/skopeo-range.yaml",
+		0, 0, true, map[string][]string{
+			"tagsets-skopeo/range/busybox": {
+				"latest", "1.31", "1.31.0", "1.31.1-musl", "1.31.1",
+				"1.31.1-uclibc", "1.31.1-glibc",
+			},
+		},
+		test.GetParams())
+}
+
+//
+func TestE2ESkopeoTagSetsRegex(t *testing.T) {
+	tryConfig(test.NewTestHelper(t), "e2e/tagsets/skopeo-regex.yaml",
+		0, 0, true, map[string][]string{
+			"tagsets-skopeo/regex/busybox": {
+				"1.26.1-musl", "1.26.1-glibc", "1.26.1-uclibc",
+			},
+			"tagsets-skopeo/regexinv/busybox": {"1.26.1-uclibc"},
+		},
+		test.GetParams())
+}
+
 //
 func tryConfig(th *test.TestHelper, file string, ticks int, wait time.Duration,
 	verify bool, expectations map[string][]string, data interface{}) {
@@ -260,7 +309,7 @@ func validateAgainstExpectations(th *test.TestHelper, c *sync.SyncConfig,
 	for eRef, eTags := range expectations {
 		ref := fmt.Sprintf("%s/%s", t.Target.Registry, eRef)
 		tags, err := skopeo.ListAllTags(ref,
-			skopeo.DecodeJSONAuth(t.Target.GetAuth()),
+			util.DecodeJSONAuth(t.Target.GetAuth()),
 			"", t.Target.SkipTLSVerify)
 		th.AssertNoError(err)
 		th.AssertEquivalentSlices(eTags, tags)
@@ -275,7 +324,7 @@ func validateAgainstTaskMapping(th *test.TestHelper, c *sync.SyncConfig) {
 		for _, m := range t.Mappings {
 			ref := fmt.Sprintf("%s%s", t.Target.Registry, m.To)
 			tags, err := skopeo.ListAllTags(ref,
-				skopeo.DecodeJSONAuth(t.Target.GetAuth()),
+				util.DecodeJSONAuth(t.Target.GetAuth()),
 				"", t.Target.SkipTLSVerify)
 			th.AssertNoError(err)
 			th.AssertEquivalentSlices(m.Tags, tags)

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/aws/aws-sdk-go v1.38.13
+	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/containerd v1.3.3 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 // indirect

go.sum

@@ -86,7 +86,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

internal/pkg/relays/docker/docker.go

@@ -22,14 +22,15 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/jsonmessage"
 	"golang.org/x/crypto/ssh/terminal"
+
+	"github.com/xelalexv/dregsy/internal/pkg/util"
 )
 
 //
@@ -50,29 +51,6 @@ func (s *image) refWithTags() string {
 	return fmt.Sprintf("%s/%s:%v", s.Repo, s.Path, s.Tags)
 }
 
-//
-func SplitRef(ref string) (repo, path, tag string) {
-
-	ix := strings.Index(ref, "/")
-
-	if ix == -1 {
-		repo = ""
-		path = ref
-	} else {
-		repo = ref[:ix]
-		path = ref[ix+1:]
-	}
-
-	ix = strings.Index(path, ":")
-
-	if ix > -1 {
-		tag = path[ix+1:]
-		path = path[:ix]
-	}
-
-	return
-}
-
 //
 type dockerClient struct {
 	host    string
@@ -155,7 +133,7 @@ func (dc *dockerClient) listImages(ref string) ([]*image, error) {
 	ret := []*image{}
 
 	if err == nil {
-		fRepo, fPath, fTag := SplitRef(ref)
+		fRepo, fPath, fTag := util.SplitRef(ref)
 		for _, img := range imgs {
 			var i *image
 			for _, rt := range img.RepoTags {
@@ -164,7 +142,7 @@ func (dc *dockerClient) listImages(ref string) ([]*image, error) {
 					return ret, err
 				}
 				if matched {
-					repo, path, tag := SplitRef(rt)
+					repo, path, tag := util.SplitRef(rt)
 					if i == nil {
 						i = &image{
 							ID:   img.ID,
@@ -192,14 +170,14 @@ func match(filterRepo, filterPath, filterTag, ref string) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("malformed ref in filter '%s', %v", filter, err)
 	}
-	filterRepo, filterPath, _ = SplitRef(filterCanon.String())
+	filterRepo, filterPath, _ = util.SplitRef(filterCanon.String())
 
 	refCanon, err := reference.ParseAnyReference(ref)
 	if err != nil {
 		return false, fmt.Errorf("malformed image ref '%s': %v", ref, err)
 	}
 
-	repo, path, tag := SplitRef(refCanon.String())
+	repo, path, tag := util.SplitRef(refCanon.String())
 	return (filterRepo == "" || filterRepo == repo) &&
 		(filterPath == "" || filterPath == path) &&
 		(filterTag == "" || filterTag == tag), nil