From bc7f16e9b16c40ecba4103170863412e04df9c80 Mon Sep 17 00:00:00 2001 From: Aurel Canciu Date: Thu, 3 Dec 2020 00:40:08 +0200 Subject: [PATCH] Implement policy-level tag regex filtering Tag regex filtering allows the user to filter tags based on a regular expression pattern and enables tag version extraction through capture group replacement reference. Fixes #73 Signed-off-by: Aurel Canciu --- api/go.sum | 2 - api/v1alpha1/imagepolicy_types.go | 17 +++++ api/v1alpha1/zz_generated.deepcopy.go | 20 ++++++ ...image.toolkit.fluxcd.io_imagepolicies.yaml | 15 ++++ controllers/imagepolicy_controller.go | 10 ++- internal/policy/alphabetical.go | 15 +++- internal/policy/alphabetical_test.go | 51 +++++++++++--- internal/policy/filter.go | 70 +++++++++++++++++++ internal/policy/filter_test.go | 67 ++++++++++++++++++ internal/policy/policer.go | 2 +- internal/policy/semver.go | 18 +++-- internal/policy/semver_test.go | 40 +++++++++-- 12 files changed, 302 insertions(+), 25 deletions(-) create mode 100644 internal/policy/filter.go create mode 100644 internal/policy/filter_test.go diff --git a/api/go.sum b/api/go.sum index 8d7cbca7..2b078f66 100644 --- a/api/go.sum +++ b/api/go.sum @@ -61,8 +61,6 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fluxcd/pkg/apis/meta v0.4.0 h1:JChqB9GGgorW9HWKxirTVV0rzrcLyzBaVjinmqZ0iHA= -github.com/fluxcd/pkg/apis/meta v0.4.0/go.mod h1:wOzQQx8CdtUQCGaLzqGu4QgnNxYkI6/wvdvlovxWhF0= github.com/fluxcd/pkg/apis/meta v0.5.0 h1:FaU++mQY0g4sVVl+hG+vk0CXBLbb4EVfRuzs3IjLXvo= github.com/fluxcd/pkg/apis/meta v0.5.0/go.mod h1:aEUuZIawboAAFLlYz/juVJ7KNmlWbBtJFYkOWWmGUR4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= diff --git a/api/v1alpha1/imagepolicy_types.go b/api/v1alpha1/imagepolicy_types.go index 0d46a98c..3f646db2 100644 --- a/api/v1alpha1/imagepolicy_types.go +++ b/api/v1alpha1/imagepolicy_types.go @@ -36,6 +36,11 @@ type ImagePolicySpec struct { // selecting the most recent image // +required Policy ImagePolicyChoice `json:"policy"` + // FilterTags enables filtering for only a subset of tags based on a set of + // rules. If no rules are provided, all the tags from the repository will be + // ordered and compared. + // +optional + FilterTags *TagFilter `json:"filterTags,omitempty"` } // ImagePolicyChoice is a union of all the types of policy that can be @@ -69,6 +74,18 @@ type AlphabeticalPolicy struct { Order string `json:"order,omitempty"` } +// TagFilter enables filtering tags based on a set of defined rules +type TagFilter struct { + // Pattern specifies a regular expression pattern used to filter for image + // tags. + // +optional + Pattern string `json:"pattern"` + // Extract allows a capture group to be extracted from the specified regular + // expression pattern, useful before tag evaluation. + // +optional + Extract string `json:"extract"` +} + // ImagePolicyStatus defines the observed state of ImagePolicy type ImagePolicyStatus struct { // LatestImage gives the first in the list of images scanned by diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1fb469b4..e12f6e37 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -130,6 +130,11 @@ func (in *ImagePolicySpec) DeepCopyInto(out *ImagePolicySpec) { *out = *in out.ImageRepositoryRef = in.ImageRepositoryRef in.Policy.DeepCopyInto(&out.Policy) + if in.FilterTags != nil { + in, out := &in.FilterTags, &out.FilterTags + *out = new(TagFilter) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePolicySpec. @@ -307,3 +312,18 @@ func (in *SemVerPolicy) DeepCopy() *SemVerPolicy { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TagFilter) DeepCopyInto(out *TagFilter) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TagFilter. +func (in *TagFilter) DeepCopy() *TagFilter { + if in == nil { + return nil + } + out := new(TagFilter) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml b/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml index 89e30701..f06a473e 100644 --- a/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml +++ b/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml @@ -41,6 +41,21 @@ spec: description: ImagePolicySpec defines the parameters for calculating the ImagePolicy properties: + filterTags: + description: FilterTags enables filtering for only a subset of tags + based on a set of rules. If no rules are provided, all the tags + from the repository will be ordered and compared. + properties: + extract: + description: Extract allows a capture group to be extracted from + the specified regular expression pattern, useful before tag + evaluation. + type: string + pattern: + description: Pattern specifies a regular expression pattern used + to filter for image tags. + type: string + type: object imageRepositoryRef: description: ImageRepositoryRef points at the object specifying the image being scanned diff --git a/controllers/imagepolicy_controller.go b/controllers/imagepolicy_controller.go index 92789259..cc496184 100644 --- a/controllers/imagepolicy_controller.go +++ b/controllers/imagepolicy_controller.go @@ -127,7 +127,15 @@ func (r *ImagePolicyReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) if policer != nil { tags, err := r.Database.Tags(repo.Status.CanonicalImageName) if err == nil { - latest, err = policer.Latest(tags) + var filter *policy.RegexFilter + if pol.Spec.FilterTags != nil { + filter, err = policy.NewRegexFilter(pol.Spec.FilterTags.Pattern, pol.Spec.FilterTags.Extract) + if err != nil { + return ctrl.Result{}, err + } + filter.Apply(tags) + } + latest, err = policer.Latest(tags, filter) } } if err != nil { diff --git a/internal/policy/alphabetical.go b/internal/policy/alphabetical.go index c4805599..cb92714e 100644 --- a/internal/policy/alphabetical.go +++ b/internal/policy/alphabetical.go @@ -50,17 +50,26 @@ func NewAlphabetical(order string) (*Alphabetical, error) { } // Latest returns latest version from a provided list of strings -func (p *Alphabetical) Latest(versions []string) (string, error) { +func (p *Alphabetical) Latest(versions []string, filter *RegexFilter) (string, error) { if len(versions) == 0 { return "", fmt.Errorf("version list argument cannot be empty") } - sorted := sort.StringSlice(versions) + var sorted sort.StringSlice = versions + if filter != nil { + sorted = filter.Items() + } + if p.Order == AlphabeticalOrderDesc { sort.Sort(sorted) } else { sort.Sort(sort.Reverse(sorted)) } - return sorted[0], nil + latest := sorted[0] + if filter != nil { + return filter.GetOriginalTag(latest), nil + } + + return latest, nil } diff --git a/internal/policy/alphabetical_test.go b/internal/policy/alphabetical_test.go index 97a5de36..3dba41f7 100644 --- a/internal/policy/alphabetical_test.go +++ b/internal/policy/alphabetical_test.go @@ -63,48 +63,77 @@ func TestAlphabetical_Latest(t *testing.T) { label string order string versions []string + filter *RegexFilter expectedVersion string expectErr bool }{ { - label: "Ubuntu CalVer", + label: "With Ubuntu CalVer", versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"}, expectedVersion: "20.10", }, - { - label: "Ubuntu CalVer descending", + label: "With Ubuntu CalVer prefix include", + versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"}, + filter: newRegexFilter("16", ""), + expectedVersion: "16.10", + }, + { + label: "With Ubuntu CalVer descending", versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"}, order: AlphabeticalOrderDesc, expectedVersion: "16.04", }, { - label: "Ubuntu code names", + label: "With Ubuntu code names", versions: []string{"xenial", "yakkety", "zesty", "artful", "bionic"}, expectedVersion: "zesty", }, { - label: "Ubuntu code names descending", + label: "With Ubuntu code names descending", versions: []string{"xenial", "yakkety", "zesty", "artful", "bionic"}, order: AlphabeticalOrderDesc, expectedVersion: "artful", }, { - label: "Timestamps", + label: "With Timestamps", versions: []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"}, expectedVersion: "1606364286", }, { - label: "Timestamps desc", + label: "With Unix Timestamps desc", versions: []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"}, order: AlphabeticalOrderDesc, expectedVersion: "1606234201", }, { - label: "Timestamps with prefix", + label: "With Unix Timestamps prefix", versions: []string{"rel-1606234201", "rel-1606364286", "rel-1606334092", "rel-1606334284", "rel-1606334201"}, expectedVersion: "rel-1606364286", }, + { + label: "With RFC3339", + versions: []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"}, + expectedVersion: "2023-05-08T00-20-00Z", + }, + { + label: "With RFC3339 desc", + versions: []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"}, + order: AlphabeticalOrderDesc, + expectedVersion: "1990-01-08T00-20-00Z", + }, + { + label: "With prefix filter", + versions: []string{"rel-1", "rel-2", "rel-3", "rel-4", "dev-5", "dev-6"}, + filter: newRegexFilter("rel-", ""), + expectedVersion: "rel-4", + }, + { + label: "With prefix include replace capture group", + versions: []string{"rel-11", "rel-12", "rel-13", "rel-15", "dev-5", "dev-6", "ver-12", "ver-10", "gen-50"}, + filter: newRegexFilter("(rel|ver)-(?P.*)", "$tag"), + expectedVersion: "rel-15", + }, { label: "Empty version list", versions: []string{}, @@ -118,8 +147,10 @@ func TestAlphabetical_Latest(t *testing.T) { if err != nil { t.Fatalf("returned unexpected error: %s", err) } - - latest, err := policy.Latest(tt.versions) + if tt.filter != nil { + tt.filter.Apply(tt.versions) + } + latest, err := policy.Latest(tt.versions, tt.filter) if tt.expectErr && err == nil { t.Fatalf("expecting error, got nil") } diff --git a/internal/policy/filter.go b/internal/policy/filter.go new file mode 100644 index 00000000..f3670a7e --- /dev/null +++ b/internal/policy/filter.go @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "fmt" + "regexp" +) + +// RegexFilter represents a regular expression filter +type RegexFilter struct { + filtered map[string]string + + Regexp *regexp.Regexp + Replace string +} + +// NewRegexFilter constructs new RegexFilter object +func NewRegexFilter(pattern string, replace string) (*RegexFilter, error) { + m, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid regular expression pattern '%s': %s", pattern, err.Error()) + } + return &RegexFilter{ + Regexp: m, + Replace: replace, + }, nil +} + +// Apply will construct the filtered list of tags based on the provided list of tags +func (f *RegexFilter) Apply(list []string) { + f.filtered = map[string]string{} + for _, item := range list { + if f.Regexp.MatchString(item) { + tag := item + if f.Replace != "" { + tag = f.Regexp.ReplaceAllString(item, f.Replace) + } + f.filtered[tag] = item + } + } +} + +// Items returns the list of filtered tags +func (f *RegexFilter) Items() []string { + var filtered []string + for k := range f.filtered { + filtered = append(filtered, k) + } + return filtered +} + +// GetOriginalTag returns the original tag before replace extraction +func (f *RegexFilter) GetOriginalTag(tag string) string { + return f.filtered[tag] +} diff --git a/internal/policy/filter_test.go b/internal/policy/filter_test.go new file mode 100644 index 00000000..5b4db232 --- /dev/null +++ b/internal/policy/filter_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "reflect" + "sort" + "testing" +) + +func TestRegexFilter(t *testing.T) { + cases := []struct { + label string + tags []string + pattern string + extract string + expected []string + }{ + { + label: "none", + tags: []string{"a"}, + expected: []string{"a"}, + }, + { + label: "valid pattern", + tags: []string{"ver1", "ver2", "ver3", "rel1"}, + pattern: "^ver", + expected: []string{"ver1", "ver2", "ver3"}, + }, + { + label: "valid pattern with capture group", + tags: []string{"ver1", "ver2", "ver3", "rel1"}, + pattern: `ver(\d+)`, + extract: `$1`, + expected: []string{"1", "2", "3"}, + }, + } + for _, tt := range cases { + t.Run(tt.label, func(t *testing.T) { + filter := newRegexFilter(tt.pattern, tt.extract) + filter.Apply(tt.tags) + r := sort.StringSlice(filter.Items()) + if reflect.DeepEqual(r, tt.expected) { + t.Errorf("incorrect value returned, got '%s', expected '%s'", r, tt.expected) + } + }) + } +} + +func newRegexFilter(pattern string, extract string) *RegexFilter { + f, _ := NewRegexFilter(pattern, extract) + return f +} diff --git a/internal/policy/policer.go b/internal/policy/policer.go index 2530d7f9..5ce2d37c 100644 --- a/internal/policy/policer.go +++ b/internal/policy/policer.go @@ -18,5 +18,5 @@ package policy // Policer is an interface representing a policy implementation type type Policer interface { - Latest([]string) (string, error) + Latest([]string, *RegexFilter) (string, error) } diff --git a/internal/policy/semver.go b/internal/policy/semver.go index 0a2e12c9..6040ff5a 100644 --- a/internal/policy/semver.go +++ b/internal/policy/semver.go @@ -44,21 +44,31 @@ func NewSemVer(r string) (*SemVer, error) { } // Latest returns latest version from a provided list of strings -func (p *SemVer) Latest(versions []string) (string, error) { +func (p *SemVer) Latest(versions []string, filter *RegexFilter) (string, error) { if len(versions) == 0 { return "", fmt.Errorf("version list argument cannot be empty") } + if filter != nil { + versions = filter.Items() + } + var latestVersion *semver.Version - for _, ver := range versions { - if v, err := version.ParseVersion(ver); err == nil { + for _, tag := range versions { + if v, err := version.ParseVersion(tag); err == nil { if p.constraint.Check(v) && (latestVersion == nil || v.GreaterThan(latestVersion)) { latestVersion = v } } } + if latestVersion != nil { - return latestVersion.Original(), nil + tag := latestVersion.Original() + if filter != nil { + return filter.GetOriginalTag(tag), nil + } + return tag, nil } + return "", fmt.Errorf("unable to determine latest version from provided list") } diff --git a/internal/policy/semver_test.go b/internal/policy/semver_test.go index 2cdcd157..924b4ae6 100644 --- a/internal/policy/semver_test.go +++ b/internal/policy/semver_test.go @@ -57,27 +57,56 @@ func TestSemVer_Latest(t *testing.T) { label string semverRange string versions []string + filter *RegexFilter expectedVersion string expectErr bool }{ { - label: "Regular", + label: "With valid format", versions: []string{"1.0.0", "1.0.0.1", "1.0.0p", "1.0.1", "1.2.0", "0.1.0"}, semverRange: "1.0.x", expectedVersion: "1.0.1", }, { - label: "Regular with prefix", + label: "With valid format prefix", versions: []string{"v1.2.3", "v1.0.0", "v0.1.0"}, semverRange: "1.0.x", expectedVersion: "v1.0.0", }, { - label: "With invalid prefix", + label: "With valid format prefix strip", + versions: []string{"v1.2.3", "v1.0.0", "v0.1.0"}, + filter: newRegexFilter("v", ""), + semverRange: "1.0.x", + expectedVersion: "v1.0.0", + }, + { + label: "With invalid format prefix", versions: []string{"b1.2.3", "b1.0.0", "b0.1.0"}, semverRange: "1.0.x", expectErr: true, }, + { + label: "With invalid format prefix extract", + versions: []string{"b1.2.3", "b1.0.0", "b0.1.0"}, + filter: newRegexFilter("b(.*)", "$1"), + semverRange: "1.0.x", + expectedVersion: "b1.0.0", + }, + { + label: "With invalid format prefix include", + versions: []string{"ver1.0.3", "ver1.0.0", "ver0.1.0", "dev-v1.0.2", "dev-v1.0.0"}, + filter: newRegexFilter("dev-", ""), + semverRange: "1.0.x", + expectErr: true, + }, + { + label: "With invalid format prefix extract", + versions: []string{"ver1.0.3", "ver1.0.0", "ver0.1.0", "dev-v1.0.2", "dev-v1.0.0"}, + filter: newRegexFilter("dev-(.*)", "$1"), + semverRange: "1.0.x", + expectedVersion: "dev-v1.0.2", + }, { label: "With empty list", versions: []string{}, @@ -99,7 +128,10 @@ func TestSemVer_Latest(t *testing.T) { t.Fatalf("returned unexpected error: %s", err) } - latest, err := policy.Latest(tt.versions) + if tt.filter != nil { + tt.filter.Apply(tt.versions) + } + latest, err := policy.Latest(tt.versions, tt.filter) if tt.expectErr && err == nil { t.Fatalf("expecting error, got nil") }