From 10434e7e1fbba19196d01431fad8b6a0bf82e371 Mon Sep 17 00:00:00 2001 From: Soule BA Date: Mon, 18 Mar 2024 10:57:16 +0100 Subject: [PATCH 1/2] Introduce a semver filter in OCIRepository If implemented a semver filter regex can be declared in conjuction with a semver range in the OCIRepository `spec.Reference` Signed-off-by: Soule BA --- api/v1beta2/ocirepository_types.go | 4 ++ ...rce.toolkit.fluxcd.io_ocirepositories.yaml | 4 ++ docs/api/v1beta2/source.md | 12 ++++++ .../controller/ocirepository_controller.go | 35 ++++++++++++++++-- .../ocirepository_controller_test.go | 27 +++++++++++++- .../testdata/podinfo/podinfo-6.1.5-beta.1.tar | Bin 0 -> 14848 bytes .../testdata/podinfo/podinfo-6.1.5-rc.1.tar | Bin 0 -> 14848 bytes .../testdata/podinfo/podinfo-6.1.6-rc.1.tar | Bin 0 -> 14848 bytes 8 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 internal/controller/testdata/podinfo/podinfo-6.1.5-beta.1.tar create mode 100644 internal/controller/testdata/podinfo/podinfo-6.1.5-rc.1.tar create mode 100644 internal/controller/testdata/podinfo/podinfo-6.1.6-rc.1.tar diff --git a/api/v1beta2/ocirepository_types.go b/api/v1beta2/ocirepository_types.go index 540a18ac2..5c4df35ce 100644 --- a/api/v1beta2/ocirepository_types.go +++ b/api/v1beta2/ocirepository_types.go @@ -157,6 +157,10 @@ type OCIRepositoryRef struct { // +optional SemVer string `json:"semver,omitempty"` + // SemverFilter is a regex pattern to filter the tags within the SemVer range. + // +optional + SemverFilter string `json:"semverFilter,omitempty"` + // Tag is the image tag to pull, defaults to latest. // +optional Tag string `json:"tag,omitempty"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml index f083276ba..25c33512e 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml @@ -146,6 +146,10 @@ spec: SemVer is the range of tags to pull selecting the latest within the range, takes precedence over Tag. type: string + semverFilter: + description: SemverFilter is a regex pattern to filter the tags + within the SemVer range. + type: string tag: description: Tag is the image tag to pull, defaults to latest. type: string diff --git a/docs/api/v1beta2/source.md b/docs/api/v1beta2/source.md index 04c3e328f..b5d50e9fe 100644 --- a/docs/api/v1beta2/source.md +++ b/docs/api/v1beta2/source.md @@ -2938,6 +2938,18 @@ the range, takes precedence over Tag.

+semverFilter
+ +string + + + +(Optional) +

SemverFilter is a regex pattern to filter the tags within the SemVer range.

+ + + + tag
string diff --git a/internal/controller/ocirepository_controller.go b/internal/controller/ocirepository_controller.go index ff44b414c..3de4faaa7 100644 --- a/internal/controller/ocirepository_controller.go +++ b/internal/controller/ocirepository_controller.go @@ -26,6 +26,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -116,6 +117,8 @@ var ociRepositoryFailConditions = []string{ sourcev1.StorageOperationFailedCondition, } +type filterFunc func(tags []string) ([]string, error) + type invalidOCIURLError struct { err error } @@ -821,7 +824,7 @@ func (r *OCIRepositoryReconciler) getArtifactRef(obj *ociv1.OCIRepository, optio } if obj.Spec.Reference.SemVer != "" { - return r.getTagBySemver(repo, obj.Spec.Reference.SemVer, options) + return r.getTagBySemver(repo, obj.Spec.Reference.SemVer, filterTags(obj.Spec.Reference.SemverFilter), options) } if obj.Spec.Reference.Tag != "" { @@ -834,19 +837,24 @@ func (r *OCIRepositoryReconciler) getArtifactRef(obj *ociv1.OCIRepository, optio // getTagBySemver call the remote container registry, fetches all the tags from the repository, // and returns the latest tag according to the semver expression. -func (r *OCIRepositoryReconciler) getTagBySemver(repo name.Repository, exp string, options []remote.Option) (name.Reference, error) { +func (r *OCIRepositoryReconciler) getTagBySemver(repo name.Repository, exp string, filter filterFunc, options []remote.Option) (name.Reference, error) { tags, err := remote.List(repo, options...) if err != nil { return nil, err } + validTags, err := filter(tags) + if err != nil { + return nil, err + } + constraint, err := semver.NewConstraint(exp) if err != nil { return nil, fmt.Errorf("semver '%s' parse error: %w", exp, err) } var matchingVersions []*semver.Version - for _, t := range tags { + for _, t := range validTags { v, err := version.ParseVersion(t) if err != nil { continue @@ -1298,3 +1306,24 @@ func layerSelectorEqual(a, b *ociv1.OCILayerSelector) bool { } return *a == *b } + +func filterTags(filter string) filterFunc { + return func(tags []string) ([]string, error) { + if filter == "" { + return tags, nil + } + + match, err := regexp.Compile(filter) + if err != nil { + return nil, err + } + + validTags := []string{} + for _, tag := range tags { + if match.MatchString(tag) { + validTags = append(validTags, tag) + } + } + return validTags, nil + } +} diff --git a/internal/controller/ocirepository_controller_test.go b/internal/controller/ocirepository_controller_test.go index faf31fd76..4d0b51c16 100644 --- a/internal/controller/ocirepository_controller_test.go +++ b/internal/controller/ocirepository_controller_test.go @@ -2757,7 +2757,14 @@ func TestOCIRepository_getArtifactRef(t *testing.T) { server.Close() }) - imgs, err := pushMultiplePodinfoImages(server.registryHost, true, "6.1.4", "6.1.5", "6.1.6") + imgs, err := pushMultiplePodinfoImages(server.registryHost, true, + "6.1.4", + "6.1.5-beta.1", + "6.1.5-rc.1", + "6.1.5", + "6.1.6-rc.1", + "6.1.6", + ) g.Expect(err).ToNot(HaveOccurred()) tests := []struct { @@ -2801,6 +2808,24 @@ func TestOCIRepository_getArtifactRef(t *testing.T) { url: "ghcr.io/stefanprodan/charts", wantErr: true, }, + { + name: "valid url with semver filter", + url: fmt.Sprintf("oci://%s/podinfo", server.registryHost), + reference: &ociv1.OCIRepositoryRef{ + SemVer: ">= 6.1.x-0", + SemverFilter: ".*-rc.*", + }, + want: server.registryHost + "/podinfo:6.1.6-rc.1", + }, + { + name: "valid url with semver filter and unexisting version", + url: fmt.Sprintf("oci://%s/podinfo", server.registryHost), + reference: &ociv1.OCIRepositoryRef{ + SemVer: ">= 6.1.x-0", + SemverFilter: ".*-alpha.*", + }, + wantErr: true, + }, } clientBuilder := fakeclient.NewClientBuilder(). diff --git a/internal/controller/testdata/podinfo/podinfo-6.1.5-beta.1.tar b/internal/controller/testdata/podinfo/podinfo-6.1.5-beta.1.tar new file mode 100644 index 0000000000000000000000000000000000000000..335d6a5ad4fb20c1a81bf57d507be7c60dbeeae4 GIT binary patch literal 14848 zcmeHNZExE+63*xR3POtmZh_h+^=b*ohrOhS2G?tXH0c6c6a__5Hm{PXrDS8>Apd=b zdW-BjaZaleL*;n27l6Urv|GSr0Z_h8@JZ3K@7`wLJ#(47=JZH7ECjJC__7U1UCXX4_w*BPs zC*$wp<<{H7?+@mIz4QNnh0J!0X-EDO)9h+PTmL;1>i>6{nLl~{-x?w^0%PB4V7xGx z;X3S(F(~IYL9iVWC%h(v-W?4Si~9je``+sMt{2KPa>Wta+UI|9It4a9I!tf|3~(Jmg$o9{3o`B zA^rpYJ75qVP(^d`j;lh*EI>5Ri^-B41D1&bbk@y>)q9qNE5z^=^K)D1Y7R%!z z=S)*mkzH{bt=@9Lc2N-^3y6v@h1UE^K!s7O*CG{VfE;Mam7FPF6lWll03__hTnw#6 zsQ^jNR$4J=kqNNOZgQ}I*CgnV{&kblB^3z`6R<~%Aylq%c)scjXmvc{3=F7cQ(Cfl zjlWuDFz)*iffk z_n302z!`?``CiKcV*ylnuXV#N7)j%8t*t}Toxo^4Hc{8H(7-i7vpG|hc_oi&2G&G0 zn>2w8+%XyJ$kd-o;t~MxrA!1{1?W6}Bg;z&8I*^POAn#z9ig`Wd0FNiPtVZHYQ+B3 zpY~e8j8Zb01nBzovZMD^jlgM2u%c`HQYHAX*ZR21naWv@xpQM-Er+B9tArnwTn zdA95iwJrdc7iZ^h{(bTB=KSO7#p{c! zP8mW=nk@9}|HZN$&tvWEMkoH#6b6xU*lTWSgRL*r z!yjd6+M3nDMpnp$V!W551itEE_kt3 z37i&h*Wucr;(>K+Ab*_)?(yF|r-uf9jY&K3A7I0c^M7Kx-f;f+bpd^*)EfRDAR;m15oe^plVI9HKE1}y01|S33*+`s?r1~rYdQ*F>Mb}zp7R!^Od!2 zYl|vxN|A`q74U2L5c&mG!$XT4i5M*kfLTZsh=y{>drZFn4V z?&o4V1NgW;pdWvlR{NPH(Y2u67yhR?FRD|aO{NMBj8MCXz?>eTjT426*1*?lejR_? zsr_{h`N^cZm{0nVIjj?OUO#gFgKgq%F1=Ovu;@uw*Y}Xn zl{Lhum>%%bKps3Wcwq3r|DOk#J+-)R5{F`JhB$_MpfT}dY)6cS6f^i@?pdbgnV#j+ zID$8GoME^nYVs+Mr{n$a|6t46c>iM(><;+)S9b(p+1@uZjk~)c9iI@Jfo1T(;DNyd-_iqr1CU#khyVZp literal 0 HcmV?d00001 diff --git a/internal/controller/testdata/podinfo/podinfo-6.1.5-rc.1.tar b/internal/controller/testdata/podinfo/podinfo-6.1.5-rc.1.tar new file mode 100644 index 0000000000000000000000000000000000000000..335d6a5ad4fb20c1a81bf57d507be7c60dbeeae4 GIT binary patch literal 14848 zcmeHNZExE+63*xR3POtmZh_h+^=b*ohrOhS2G?tXH0c6c6a__5Hm{PXrDS8>Apd=b zdW-BjaZaleL*;n27l6Urv|GSr0Z_h8@JZ3K@7`wLJ#(47=JZH7ECjJC__7U1UCXX4_w*BPs zC*$wp<<{H7?+@mIz4QNnh0J!0X-EDO)9h+PTmL;1>i>6{nLl~{-x?w^0%PB4V7xGx z;X3S(F(~IYL9iVWC%h(v-W?4Si~9je``+sMt{2KPa>Wta+UI|9It4a9I!tf|3~(Jmg$o9{3o`B zA^rpYJ75qVP(^d`j;lh*EI>5Ri^-B41D1&bbk@y>)q9qNE5z^=^K)D1Y7R%!z z=S)*mkzH{bt=@9Lc2N-^3y6v@h1UE^K!s7O*CG{VfE;Mam7FPF6lWll03__hTnw#6 zsQ^jNR$4J=kqNNOZgQ}I*CgnV{&kblB^3z`6R<~%Aylq%c)scjXmvc{3=F7cQ(Cfl zjlWuDFz)*iffk z_n302z!`?``CiKcV*ylnuXV#N7)j%8t*t}Toxo^4Hc{8H(7-i7vpG|hc_oi&2G&G0 zn>2w8+%XyJ$kd-o;t~MxrA!1{1?W6}Bg;z&8I*^POAn#z9ig`Wd0FNiPtVZHYQ+B3 zpY~e8j8Zb01nBzovZMD^jlgM2u%c`HQYHAX*ZR21naWv@xpQM-Er+B9tArnwTn zdA95iwJrdc7iZ^h{(bTB=KSO7#p{c! zP8mW=nk@9}|HZN$&tvWEMkoH#6b6xU*lTWSgRL*r z!yjd6+M3nDMpnp$V!W551itEE_kt3 z37i&h*Wucr;(>K+Ab*_)?(yF|r-uf9jY&K3A7I0c^M7Kx-f;f+bpd^*)EfRDAR;m15oe^plVI9HKE1}y01|S33*+`s?r1~rYdQ*F>Mb}zp7R!^Od!2 zYl|vxN|A`q74U2L5c&mG!$XT4i5M*kfLTZsh=y{>drZFn4V z?&o4V1NgW;pdWvlR{NPH(Y2u67yhR?FRD|aO{NMBj8MCXz?>eTjT426*1*?lejR_? zsr_{h`N^cZm{0nVIjj?OUO#gFgKgq%F1=Ovu;@uw*Y}Xn zl{Lhum>%%bKps3Wcwq3r|DOk#J+-)R5{F`JhB$_MpfT}dY)6cS6f^i@?pdbgnV#j+ zID$8GoME^nYVs+Mr{n$a|6t46c>iM(><;+)S9b(p+1@uZjk~)c9iI@Jfo1T(;DNyd-_iqr1CU#khyVZp literal 0 HcmV?d00001 diff --git a/internal/controller/testdata/podinfo/podinfo-6.1.6-rc.1.tar b/internal/controller/testdata/podinfo/podinfo-6.1.6-rc.1.tar new file mode 100644 index 0000000000000000000000000000000000000000..09616c2dfabc3ca74b251763d2de655d892e2629 GIT binary patch literal 14848 zcmeHNZExE)5YFfP3PQzz4XADE{RPN}E$Pr;SskQF2dpRpTB2;OlBiL%Q8&nc-%)Ro zT_;XzHz-yK0U{soj>kLR@gYSo@;vIhB-F9%IuUkE98Mw&`_#r5$2z7_>>8o1o4V&< z2UFK`V$U>VOfBM?=ICJigRzBO+dcq=>RJEarB&~`YwN}VvbU?=onu}|BEY0wH7U#C zX$QBfzGss6c8&kLR~K(j$8R1o7wZ^1mesnZ@>J_8_-Qt6l?x4Q+xP(O9FvEPYTADA z`0K-G@p9ws{{MI8ft~aJpMqyw#b;D?DLzDj&hWh_3GxG<}{~L`(TA=MZ4YZe9 zC@7IsDT5&DgYxeZQU-2J{PsHj?g zm6C+Bw=!ax7Q`ee=u54o@wxnxNCk-2oR@OhzK?U7t?DU%jk6GpsQF)I3oXdzN%B%V zPGLmV>uE}+wW05LGEZsIwEV@zN0v=bSZ(dAOp0aBSq64hvg7QxIZIkqXJkT?TKu;3 zOz)4@AGce|N#DFj5A2%wql7G}7%B{X@AGi|*Bz)AYxqw$%^v^V>(6~Q3`ZYf1>`CF zO!ujFMVDICpSJ&l-KO2||5lv%jefMzA4N1z_%fxL94<+k?6E%;{|EMers3$T`LA0h zhWPjRZ;wH^M-|E0J1PpsvjCAiFGdUf5U@-Xpp$k!9Hvx~h)5C~BBan+fN~BY191N|6 z6o4eBOQjgJ@B~<cWN*zsT2nJNLDUso< z!e5Ot828CTcro3b+PnItxWC?};B%g^a2cS}_zjom5HctaEteWX+dD!{|1&A`mZxLrc{yU=^Sw?h zm?3${lK@>FpSSd`ssT7nu&`(wzf=i6?6f|taw_t$!`!;Dw$AtcPU~h>Ax~2RDbg|y z(6Di2hc*}}2OX^pcz`NuGpHjK{}mZ!UoF_RMG2Nouq9fskPcKH9yMvGCWLwvS3(yw zIV!WWB35Jp7V~rvQ2=a5 z(eUFMZEX%JwuFZynZbTDP!So;C`sh(b4QM4Ofr(3C=ay-NmZ-nku0UK*WAzsTVJS$ z-^C=wKpit8kN7$oKOsTw}akY-D^jTw0Rb#)vor<`M zH*2Js11ceaG7YFT00l1qs#XL`5h}c(`WiHlkXBW!EKPu7%92(Y)ARuKt84}QS6bV& zHmLGSvV?sufnUvs&jjn951jvCOLy1u-?W|n{Aa&GxJO$z|J$RVx78l=5zBjZ;&Plo zO`zgbZQxf=Th@*+IwuC*Eq=WMs#aL>?hXhxZNID4x!SF_yF9fsEM@Pu%w4|Do_~|d z3SyK@_jsu%_a5jy(0kzj=K<=5wxMG`_NWQBE?tLso)<^9t4ATGcw)f?P>*;x@=dH8 z*t1-V7{tfU1QR0)!{P4tf3RtEF&xA`CXY93*iL=;{s-LtUC)1AcX5CIx81RO;D8K1u5aUMdsz!0o8dT)Uy^B= z%^N>DHj}PszM55^A2p~IXX-jpDY?I!Nl*$t^hXVwCyRhs6%4+PGg@rs*qyp9{~_Dg y_W!zL_xZoO;~$~nMtfX_s#UwZBLK_xu9>Ob-uCJEh}iTjy$5;^^d9(@9{3LiHH=>X literal 0 HcmV?d00001 From 6d7189dbd1deca1482e048a2c5723f432a9ad17b Mon Sep 17 00:00:00 2001 From: Soule BA Date: Mon, 25 Mar 2024 22:53:45 +0100 Subject: [PATCH 2/2] adding an example for semverFilter in the the documentation Signed-off-by: Soule BA --- docs/spec/v1beta2/ocirepositories.md | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/spec/v1beta2/ocirepositories.md b/docs/spec/v1beta2/ocirepositories.md index 39a34e217..f40dab066 100644 --- a/docs/spec/v1beta2/ocirepositories.md +++ b/docs/spec/v1beta2/ocirepositories.md @@ -441,6 +441,37 @@ spec: This field takes precedence over [`.tag`](#tag-example). +#### SemverFilter example + +`.spec.ref.semverFilter` is an optional field to specify a SemVer filter to apply +when fetching tags from the OCI repository. The filter is a regular expression +that is applied to the tags fetched from the repository. Only tags that match +the filter are considered for the semver range resolution. + +**Note:** The filter is only taken into account when the `.spec.ref.semver` field +is set. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 5m0s + url: oci://ghcr.io/stefanprodan/manifests/podinfo + ref: + # SemVer comparisons using constraints without a prerelease comparator will skip prerelease versions. + # Adding a `-0` suffix to the semver range will include prerelease versions. + semver: ">= 6.1.x-0" + semverFilter: ".*-rc.*" +``` + +In the above example, the controller fetches tags from the `ghcr.io/stefanprodan/manifests/podinfo` +repository and filters them using the regular expression `.*-rc.*`. Only tags that +contain the `-rc` suffix are considered for the semver range resolution. + #### Digest example To pull a specific digest, use `.spec.ref.digest`: