diff --git a/apis/artifacts/v1alpha1/uri.go b/apis/artifacts/v1alpha1/uri.go index 1a90c6ce..f6c69905 100644 --- a/apis/artifacts/v1alpha1/uri.go +++ b/apis/artifacts/v1alpha1/uri.go @@ -136,6 +136,15 @@ func (u URI) WithDigestString() string { return ref } +func (u URI) Repository() string { + ref := u.Host + if u.Path != "" { + ref = fmt.Sprintf("%s/%s", strings.Trim(ref, "/"), strings.Trim(u.Path, "/")) + } + + return ref +} + // ParseURI parse uri to URI struct func ParseURI(uri string, t ArtifactType) (URI, error) { var u = URI{Raw: uri} diff --git a/go.mod b/go.mod index 1dc0947d..18f596d6 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/opencontainers/go-digest v1.0.0 github.com/pkg/errors v0.9.1 + github.com/regclient/regclient v0.0.0-20230312202530-e07b6ce43595 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 @@ -74,6 +75,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -139,7 +141,7 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.1.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 841b53f6..a37ff134 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -432,6 +434,8 @@ github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ= github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/regclient/regclient v0.0.0-20230312202530-e07b6ce43595 h1:U6lIewvPcuRWbvqzyilWH3jEwc/0juMXip6ejiTe09s= +github.com/regclient/regclient v0.0.0-20230312202530-e07b6ce43595/go.mod h1:Q8W+IYHycJVt73Bw00HqRSNDesxYqnKJBfQx8JYzXCg= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -697,8 +701,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= diff --git a/hash/hash.go b/hash/hash.go index 1331f2d0..4abde819 100644 --- a/hash/hash.go +++ b/hash/hash.go @@ -77,13 +77,19 @@ func hashString(hashFunc func() hash.Hash, secretKey string, value []byte) (stri return hashValue, nil } +type HashFolderFilter func(path string, d fs.DirEntry) bool + // HashFolder generates a hash for the folder -func HashFolder(ctx context.Context, folder string) (hash string, err error) { +func HashFolder(ctx context.Context, folder string, filters ...HashFolderFilter) (hash string, err error) { digests := make([]digest.Digest, 0, 100) log := logging.FromContext(ctx) err = filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) (walkErr error) { + for _, filter := range filters { + if filter != nil && !filter(path, d) { + return + } + } if !d.IsDir() { - if data, readErr := ioutil.ReadFile(path); readErr == nil { digestHash := digest.FromBytes(data) log.Debugf("hash for path: %s hash: %q", path, digestHash) diff --git a/hash/hash_test.go b/hash/hash_test.go index 75ccdb51..c3d2deea 100644 --- a/hash/hash_test.go +++ b/hash/hash_test.go @@ -20,8 +20,11 @@ import ( "context" "fmt" "hash/adler32" + "io/fs" + "k8s.io/apimachinery/pkg/util/rand" "os" "os/exec" + "strings" "testing" "github.com/katanomi/pkg/command/io" @@ -260,6 +263,7 @@ func TestHashFolder(t *testing.T) { Action func(folder string, t *testing.T) error Expected string Error error + filter HashFolderFilter AfterAction func() }{ "chart folder": { @@ -317,6 +321,31 @@ func TestHashFolder(t *testing.T) { os.RemoveAll("testdata/copy") }, }, + "filter rand files": { + Folder: "testdata/copy", + Action: func(folder string, t *testing.T) error { + os.RemoveAll("testdata/copy") + if err := io.Copy("testdata/chart", "testdata/copy"); err != nil { + return err + } + + for i := 0; i < 10; i++ { + randFile := fmt.Sprintf("testdata/copy/rand.%s.yaml", rand.String(10)) + io.Copy("testdata/copy/values.yaml", randFile) + } + + return nil + }, + // new hash + Expected: "sha256:75c80677202215d8788b7a271e3eab03c143ecfe17a782bdd3c82f4720fc25da", + Error: nil, + AfterAction: func() { + os.RemoveAll("testdata/copy") + }, + filter: func(path string, d fs.DirEntry) bool { + return !strings.HasPrefix(path, "testdata/copy/rand") + }, + }, } for k, test := range table { @@ -328,7 +357,7 @@ func TestHashFolder(t *testing.T) { g.Expect(test.Action(test.Folder, t)).To(gomega.Succeed()) } - hashResult, err := HashFolder(context.TODO(), test.Folder) + hashResult, err := HashFolder(context.TODO(), test.Folder, test.filter) if test.AfterAction != nil { test.AfterAction() } diff --git a/registry/manifests.go b/registry/manifests.go new file mode 100644 index 00000000..42d3ef10 --- /dev/null +++ b/registry/manifests.go @@ -0,0 +1,119 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "context" + "fmt" + + "github.com/regclient/regclient" + "github.com/regclient/regclient/mod" + "github.com/regclient/regclient/types/docker/schema2" + "github.com/regclient/regclient/types/manifest" + "github.com/regclient/regclient/types/ref" +) + +type ManifestClient struct { + *regclient.RegClient +} + +func NewManifestClient(options ...regclient.Opt) *ManifestClient { + regClient := newRegClient(options) + + return &ManifestClient{RegClient: regClient} +} + +func (c *ManifestClient) Close(ctx context.Context, ref ref.Ref) error { + return c.RegClient.Close(ctx, ref) +} + +// GetAnnotations get annotations from a reference image +func (c *ManifestClient) GetAnnotations(ctx context.Context, reference string) (map[string]string, error) { + r, err := ref.New(reference) + if err != nil { + return nil, err + } + + m, err := c.ManifestGet(ctx, r) + if err != nil { + return nil, fmt.Errorf("get manifest error: %s", err.Error()) + } + + if anno, ok := m.(manifest.Annotator); ok { + return anno.GetAnnotations() + } + + return nil, fmt.Errorf("manifest can not access annotation") +} + +// PutEmptyIndex create an empty manifest list with annotations +func (c *ManifestClient) PutEmptyIndex(ctx context.Context, reference string, annotations map[string]string) error { + r, err := ref.New(reference) + if err != nil { + return err + } + + options := []manifest.Opts{ + manifest.WithOrig(schema2.ManifestList{ + Versioned: schema2.ManifestListSchemaVersion, + Annotations: annotations, + }), + } + + m, err := manifest.New(options...) + if err != nil { + return err + } + + return c.ManifestPut(ctx, r, m) +} + +// SetAnnotation append annotation to a reference image +// annotation key will be deleted with empty value +func (c *ManifestClient) SetAnnotation(ctx context.Context, reference string, annotations map[string]string) error { + r, err := ref.New(reference) + if err != nil { + return err + } + + if r.Tag == "" { + return fmt.Errorf("cannot replace an image digest, must include a tag") + } + + modOptions := make([]mod.Opts, 0) + for key, value := range annotations { + modOptions = append(modOptions, mod.WithAnnotation(key, value)) + } + + output, err := mod.Apply(ctx, c.RegClient, r, modOptions...) + if err != nil { + return fmt.Errorf("apply annotation error: %s", err.Error()) + } + + err = c.RegClient.ImageCopy(ctx, output, r) + if err != nil { + return fmt.Errorf("failed copying image to new name: %w", err) + } + + return nil +} + +func newRegClient(options []regclient.Opt) *regclient.RegClient { + options = append(options, regclient.WithDockerCreds()) + + return regclient.New(options...) +} diff --git a/registry/manifests_test.go b/registry/manifests_test.go new file mode 100644 index 00000000..428d856f --- /dev/null +++ b/registry/manifests_test.go @@ -0,0 +1,169 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "context" + "encoding/json" + "fmt" + "github.com/google/go-cmp/cmp" + "github.com/regclient/regclient" + "github.com/regclient/regclient/config" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/regclient/regclient/types/docker/schema2" +) + +func TestManifestClient_GetAnnotations(t *testing.T) { + tests := []struct { + repo string + tag string + annotations map[string]string + }{ + { + repo: "abc", + tag: "def", + annotations: nil, + }, + + { + repo: "abc", + tag: "def", + annotations: map[string]string{ + "abc": "def", + }, + }, + { + repo: "abc/def", + tag: "xyz", + annotations: map[string]string{ + "abc/def": "xyz", + }, + }, + } + + for i, item := range tests { + t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) { + m := schema2.ManifestList{ + Versioned: schema2.ManifestListSchemaVersion, + Annotations: item.annotations, + } + handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + path := fmt.Sprintf("v2/%s/manifests/%s", item.repo, item.tag) + if strings.Contains(request.URL.String(), path) { + resp, _ := json.Marshal(m) + writer.Header().Set("Content-Type", m.MediaType) + writer.Write(resp) + } + }) + + server := httptest.NewUnstartedServer(handler) + server.Start() + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + + client := NewManifestClient(regclient.WithConfigHost(config.Host{ + Name: host, + Hostname: host, + TLS: config.TLSDisabled, + })) + + image := fmt.Sprintf("%s/%s:%s", host, item.repo, item.tag) + annotations, err := client.GetAnnotations(context.TODO(), image) + + if err != nil { + t.Errorf("get annotations error: %s", err.Error()) + } + + if diff := cmp.Diff(item.annotations, annotations); diff != "" { + t.Errorf("diff: %s", diff) + } + }) + } +} + +func TestManifestClient_PutEmptyIndex(t *testing.T) { + tests := []struct { + repo string + tag string + annotations map[string]string + }{ + { + repo: "abc", + tag: "def", + annotations: nil, + }, + + { + repo: "abc", + tag: "def", + annotations: map[string]string{ + "abc": "def", + }, + }, + { + repo: "abc/def", + tag: "xyz", + annotations: map[string]string{ + "abc/def": "xyz", + }, + }, + } + + for i, item := range tests { + t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) { + var body map[string]map[string]string + handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + path := fmt.Sprintf("v2/%s/manifests/%s", item.repo, item.tag) + if strings.Contains(request.URL.String(), path) { + resp, _ := io.ReadAll(request.Body) + _ = json.Unmarshal(resp, &body) + + writer.WriteHeader(201) + } + }) + + server := httptest.NewUnstartedServer(handler) + server.Start() + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + + client := NewManifestClient(regclient.WithConfigHost(config.Host{ + Name: host, + Hostname: host, + TLS: config.TLSDisabled, + })) + + image := fmt.Sprintf("%s/%s:%s", host, item.repo, item.tag) + err := client.PutEmptyIndex(context.TODO(), image, item.annotations) + + if err != nil { + t.Errorf("put annotations error: %s", err.Error()) + } + + if diff := cmp.Diff(item.annotations, body["annotations"]); diff != "" { + t.Errorf("diff: %s", diff) + } + }) + } +}