diff --git a/Gopkg.lock b/Gopkg.lock index 6cbca7c1fb..3d7d8dd0b2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -142,7 +142,7 @@ [[projects]] branch = "master" - digest = "1:123ca2e74131111f6302f6e0eb27bbae6d8989b7dae00ca7a624793b4549353b" + digest = "1:2241fb4f2d265698118f5397b427f3c82c558494c38dcc90376bc1778fc00909" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -161,7 +161,7 @@ "pkg/v1/v1util", ] pruneopts = "UT" - revision = "6225ca1a4de721ff14f6c4cbbfd141ab462bdb22" + revision = "8d4083db9aa0d2fae6588c1acdbe6a1f5db461e3" [[projects]] branch = "master" diff --git a/cmd/ko/commands.go b/cmd/ko/commands.go index 9a20e4ea0c..c04a1dc899 100644 --- a/cmd/ko/commands.go +++ b/cmd/ko/commands.go @@ -131,7 +131,21 @@ func addKubeCommands(topLevel *cobra.Command) { if err != nil { log.Fatalf("error piping to 'kubectl apply': %v", err) } - go resolveFilesToWriter(fo, no, lo, ta, stdin) + + go func() { + // kubectl buffers data before starting to apply it, which + // can lead to resources being created more slowly than desired. + // In the case of --watch, it can lead to resources not being + // applied at all until enough iteration has occurred. To work + // around this, we prime the stream with a bunch of empty objects + // which kubectl will discard. + // See https://github.com/google/go-containerregistry/pull/348 + for i := 0; i < 1000; i++ { + stdin.Write([]byte("---\n")) + } + // Once primed kick things off. + resolveFilesToWriter(fo, no, lo, ta, stdin) + }() // Run it. if err := kubectlCmd.Run(); err != nil { diff --git a/cmd/ko/flatname.go b/cmd/ko/flatname.go index a4521526ae..1fed878fb2 100644 --- a/cmd/ko/flatname.go +++ b/cmd/ko/flatname.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "path/filepath" + "github.com/google/ko/pkg/publish" "github.com/spf13/cobra" ) @@ -50,3 +51,12 @@ func preserveImportPath(importpath string) string { func baseImportPaths(importpath string) string { return filepath.Base(importpath) } + +func makeNamer(no *NameOptions) publish.Namer { + if no.PreserveImportPaths { + return preserveImportPath + } else if no.BaseImportPaths { + return baseImportPaths + } + return packageWithMD5 +} diff --git a/cmd/ko/publish.go b/cmd/ko/publish.go index 950aa4d99e..15158922d5 100644 --- a/cmd/ko/publish.go +++ b/cmd/ko/publish.go @@ -24,7 +24,6 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/publish" ) @@ -75,14 +74,7 @@ func publishImages(importpaths []string, no *NameOptions, lo *LocalOptions, ta * var pub publish.Interface repoName := os.Getenv("KO_DOCKER_REPO") - var namer publish.Namer - if no.PreserveImportPaths { - namer = preserveImportPath - } else if no.BaseImportPaths { - namer = baseImportPaths - } else { - namer = packageWithMD5 - } + namer := makeNamer(no) if lo.Local || repoName == publish.LocalDomain { pub = publish.NewDaemon(namer, ta.Tags) diff --git a/cmd/ko/resolve.go b/cmd/ko/resolve.go index 2c6c77b02b..702b7060b1 100644 --- a/cmd/ko/resolve.go +++ b/cmd/ko/resolve.go @@ -24,11 +24,10 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" - "github.com/mattmoor/dep-notify/pkg/graph" - "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/publish" "github.com/google/ko/pkg/resolve" + "github.com/mattmoor/dep-notify/pkg/graph" ) func gobuildOptions() ([]build.Option, error) { @@ -78,12 +77,7 @@ func makePublisher(no *NameOptions, lo *LocalOptions, ta *TagsOptions) (publish. // Create the publish.Interface that we will use to publish image references // to either a docker daemon or a container image registry. innerPublisher, err := func() (publish.Interface, error) { - namer := func() publish.Namer { - if no.PreserveImportPaths { - return preserveImportPath - } - return packageWithMD5 - }() + namer := makeNamer(no) repoName := os.Getenv("KO_DOCKER_REPO") if lo.Local || repoName == publish.LocalDomain { diff --git a/cmd/ko/test/test.yaml b/cmd/ko/test/test.yaml index 710820c44e..8a0b0cfe42 100644 --- a/cmd/ko/test/test.yaml +++ b/cmd/ko/test/test.yaml @@ -20,5 +20,5 @@ metadata: spec: containers: - name: obiwan - image: github.com/google/go-containerregistry/cmd/ko/test + image: github.com/google/ko/cmd/test restartPolicy: Never diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 6a378cebff..7300f5e22c 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -31,7 +31,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" ) -const appPath = "/ko-app" +const ( + appDir = "/ko-app" + defaultAppFilename = "ko-app" +) // GetBase takes an importpath and returns a base v1.Image. type GetBase func(string) (v1.Image, error) @@ -117,11 +120,53 @@ func build(ip string) (string, error) { return file, nil } -func tarBinary(binary string) (*bytes.Buffer, error) { +func appFilename(importpath string) string { + base := filepath.Base(importpath) + + // If we fail to determine a good name from the importpath then use a + // safe default. + if base == "." || base == string(filepath.Separator) { + return defaultAppFilename + } + + return base +} + +func tarAddDirectories(tw *tar.Writer, dir string) error { + if dir == "." || dir == string(filepath.Separator) { + return nil + } + + // Write parent directories first + if err := tarAddDirectories(tw, filepath.Dir(dir)); err != nil { + return err + } + + // write the directory header to the tarball archive + if err := tw.WriteHeader(&tar.Header{ + Name: dir, + Typeflag: tar.TypeDir, + // Use a fixed Mode, so that this isn't sensitive to the directory and umask + // under which it was created. Additionally, windows can only set 0222, + // 0444, or 0666, none of which are executable. + Mode: 0555, + }); err != nil { + return err + } + + return nil +} + +func tarBinary(name, binary string) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) tw := tar.NewWriter(buf) defer tw.Close() + // write the parent directories to the tarball archive + if err := tarAddDirectories(tw, filepath.Dir(name)); err != nil { + return nil, err + } + file, err := os.Open(binary) if err != nil { return nil, err @@ -132,7 +177,7 @@ func tarBinary(binary string) (*bytes.Buffer, error) { return nil, err } header := &tar.Header{ - Name: appPath, + Name: name, Size: stat.Size(), Typeflag: tar.TypeReg, // Use a fixed Mode, so that this isn't sensitive to the directory and umask @@ -249,8 +294,10 @@ func (gb *gobuild) Build(s string) (v1.Image, error) { } layers = append(layers, dataLayer) + appPath := filepath.Join(appDir, appFilename(s)) + // Construct a tarball with the binary and produce a layer. - binaryLayerBuf, err := tarBinary(file) + binaryLayerBuf, err := tarBinary(appPath, file) if err != nil { return nil, err } diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index dc5a12ddbd..9b234c9044 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -19,9 +19,8 @@ import ( "io" "io/ioutil" "path/filepath" - "time" - "testing" + "time" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" @@ -117,7 +116,7 @@ func TestGoBuildNoKoData(t *testing.T) { t.Run("check determinism", func(t *testing.T) { expectedHash := v1.Hash{ Algorithm: "sha256", - Hex: "1d4fb5a6e81840aa5996d6efad00cca54b14412917ed42acf51d88d3f9482fd0", + Hex: "fb82c95fc73eaf26d0b18b1bc2d23ee32059e46806a83a313e738aac4d039492", } appLayer := ls[baseLayers+1] @@ -139,7 +138,7 @@ func TestGoBuildNoKoData(t *testing.T) { t.Errorf("len(entrypoint) = %v, want %v", got, want) } - if got, want := entrypoint[0], appPath; got != want { + if got, want := entrypoint[0], "/ko-app/ko"; got != want { t.Errorf("entrypoint = %v, want %v", got, want) } }) @@ -194,7 +193,7 @@ func TestGoBuild(t *testing.T) { t.Run("check determinism", func(t *testing.T) { expectedHash := v1.Hash{ Algorithm: "sha256", - Hex: "481f1025f9a594d8742cadb1928d1d601115a14a77001958dc539cee04fddfcf", + Hex: "4c7f97dda30576670c3a8967424f7dea023030bb3df74fc4bd10329bcb266fc2", } appLayer := ls[baseLayers+1] @@ -275,7 +274,7 @@ func TestGoBuild(t *testing.T) { t.Errorf("len(entrypoint) = %v, want %v", got, want) } - if got, want := entrypoint[0], appPath; got != want { + if got, want := entrypoint[0], "/ko-app/test"; got != want { t.Errorf("entrypoint = %v, want %v", got, want) } }) diff --git a/pkg/publish/daemon_test.go b/pkg/publish/daemon_test.go index 1641861bd1..70efdec734 100644 --- a/pkg/publish/daemon_test.go +++ b/pkg/publish/daemon_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/random" ) @@ -49,7 +48,7 @@ func init() { } func TestDaemon(t *testing.T) { - importpath := "github.com/google/go-containerregistry/cmd/ko" + importpath := "github.com/google/ko/cmd/ko" img, err := random.Image(1024, 1) if err != nil { t.Fatalf("random.Image() = %v", err) @@ -66,7 +65,7 @@ func TestDaemon(t *testing.T) { func TestDaemonTags(t *testing.T) { Tags = nil - importpath := "github.com/google/go-containerregistry/cmd/ko" + importpath := "github.com/google/ko/cmd/ko" img, err := random.Image(1024, 1) if err != nil { t.Fatalf("random.Image() = %v", err) @@ -79,7 +78,7 @@ func TestDaemonTags(t *testing.T) { t.Errorf("Publish() = %v, wanted prefix %v", got, want) } - expected := []string{"ko.local/d502d3a1d9858acbab6106d78a0e05f0:v2.0.0", "ko.local/d502d3a1d9858acbab6106d78a0e05f0:v1.2.3", "ko.local/d502d3a1d9858acbab6106d78a0e05f0:production"} + expected := []string{"ko.local/099ba5bcefdead87f92606265fb99ac0:v2.0.0", "ko.local/099ba5bcefdead87f92606265fb99ac0:v1.2.3", "ko.local/099ba5bcefdead87f92606265fb99ac0:production"} for i, v := range expected { if Tags[i] != v { diff --git a/pkg/resolve/fixed_test.go b/pkg/resolve/fixed_test.go index d766906dae..62e372ef74 100644 --- a/pkg/resolve/fixed_test.go +++ b/pkg/resolve/fixed_test.go @@ -21,7 +21,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/publish" ) diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index bee4538c44..cdc213b692 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -20,11 +20,10 @@ import ( "io" "sync" - "golang.org/x/sync/errgroup" - yaml "gopkg.in/yaml.v2" - "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/publish" + "golang.org/x/sync/errgroup" + yaml "gopkg.in/yaml.v2" ) // ImageReferences resolves supported references to images within the input yaml diff --git a/pkg/resolve/resolve_test.go b/pkg/resolve/resolve_test.go index c4b1942cbc..e8a957387f 100644 --- a/pkg/resolve/resolve_test.go +++ b/pkg/resolve/resolve_test.go @@ -19,13 +19,12 @@ import ( "io" "testing" - yaml "gopkg.in/yaml.v2" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/random" + yaml "gopkg.in/yaml.v2" ) var ( @@ -309,7 +308,7 @@ func TestMultiDocumentYAMLs(t *testing.T) { } func mustRandom() v1.Image { - img, err := random.Image(5, 1024) + img, err := random.Image(1024, 5) if err != nil { panic(err) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go new file mode 100644 index 0000000000..aa574eb8b8 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -0,0 +1,56 @@ +package remote + +import ( + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// CheckPushPermission returns an error if the given keychain cannot authorize +// a push operation to the given ref. +// +// This can be useful to check whether the caller has permission to push an +// image before doing work to construct the image. +// +// TODO(#412): Remove the need for this method. +func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { + auth, err := kc.Resolve(ref.Context().Registry) + if err != nil { + return err + } + + scopes := []string{ref.Scope(transport.PushScope)} + tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + if err != nil { + return err + } + // TODO(jasonhall): Against GCR, just doing the token handshake is + // enough, but this doesn't extend to Dockerhub + // (https://github.com/docker/hub-feedback/issues/1771), so we actually + // need to initiate an upload to tell whether the credentials can + // authorize a push. Figure out how to return early here when we can, + // to avoid a roundtrip for spec-compliant registries. + w := writer{ + ref: ref, + client: &http.Client{Transport: tr}, + } + loc, _, err := w.initiateUpload("", "") + if loc != "" { + // Since we're only initiating the upload to check whether we + // can, we should attempt to cancel it, in case initiating + // reserves some resources on the server. We shouldn't wait for + // cancelling to complete, and we don't care if it fails. + go w.cancelUpload(loc) + } + return err +} + +func (w *writer) cancelUpload(loc string) { + req, err := http.NewRequest(http.MethodDelete, loc, nil) + if err != nil { + return + } + _, _ = w.client.Do(req) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go new file mode 100644 index 0000000000..9c570b7f8a --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -0,0 +1,271 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image.. + platform v1.Platform +} + +type imageOpener struct { + auth authn.Authenticator + transport http.RoundTripper + ref name.Reference + client *http.Client + platform v1.Platform +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + types.DockerManifestList, + types.OCIImageIndex, + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + return get(ref, acceptable, options...) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOption) (*Descriptor, error) { + i := &imageOpener{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + ref: ref, + platform: defaultPlatform, + } + + for _, option := range options { + if err := option(i); err != nil { + return nil, err + } + } + tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + + f := fetcher{ + Ref: i.ref, + Client: &http.Client{Transport: tr}, + } + + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + + return &Descriptor{ + fetcher: f, + Manifest: b, + Descriptor: *desc, + platform: i.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if mediaType == types.DockerManifestSchema1Signed { + // Digests for this are stupid to calculate, ignore it. + } else if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } else { + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/docker/distribution/issues/2395 + // https://github.com/GoogleContainerTools/kaniko/issues/298 + } + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + } + + return manifest, &desc, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 1be0ad2ea4..9f2d51e81b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,16 +15,11 @@ package remote import ( - "bytes" - "fmt" "io" "io/ioutil" "net/http" - "net/url" - "strings" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -33,11 +28,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/v1util" ) -var defaultPlatform = v1.Platform{ - Architecture: "amd64", - OS: "linux", -} - // remoteImage accesses an image from a remote registry type remoteImage struct { fetcher @@ -46,135 +36,27 @@ type remoteImage struct { configLock sync.Mutex // Protects config config []byte mediaType types.MediaType - platform v1.Platform } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - var _ partial.CompressedImageCore = (*remoteImage)(nil) -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} - -func (i *imageOpener) Open() (v1.Image, error) { - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) - if err != nil { - return nil, err - } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - platform: i.platform, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: i.ref, - }, nil -} - // Image provides access to a remote image reference, applying functional options // to the underlying imageOpener before resolving the reference into a v1.Image. func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { - img := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, - } - - for _, option := range options { - if err := option(img); err != nil { - return nil, err - } - } - return img.Open() -} - -// fetcher implements methods for reading from a remote image. -type fetcher struct { - Ref name.Reference - Client *http.Client -} - -// url returns a url.Url for the specified path in the context of this remote image reference. -func (f *fetcher) url(resource, identifier string) url.URL { - return url.URL{ - Scheme: f.Ref.Context().Registry.Scheme(), - Host: f.Ref.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), - } -} - -func (f *fetcher) fetchManifest(acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { - u := f.url("manifests", f.Ref.Identifier()) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, nil, err - } - accept := []string{} - for _, mt := range acceptable { - accept = append(accept, string(mt)) - } - req.Header.Set("Accept", strings.Join(accept, ",")) - - resp, err := f.Client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK); err != nil { - return nil, nil, err - } - - manifest, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + // We resolve these to images later. + types.DockerManifestList, + types.OCIImageIndex, } - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + desc, err := get(ref, acceptable, options...) if err != nil { - return nil, nil, err - } - - // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := f.Ref.(name.Digest); ok { - if digest.String() != dgst.DigestStr() { - return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) - } - } else { - // Do nothing for tags; I give up. - // - // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, - // but so many registries implement this incorrectly that it's not worth checking. - // - // For reference: - // https://github.com/docker/distribution/issues/2395 - // https://github.com/GoogleContainerTools/kaniko/issues/298 - } - - // Return all this info since we have to calculate it anyway. - desc := v1.Descriptor{ - Digest: digest, - Size: size, - MediaType: types.MediaType(resp.Header.Get("Content-Type")), + return nil, err } - return manifest, &desc, nil + return desc.Image() } func (r *remoteImage) MediaType() (types.MediaType, error) { @@ -184,7 +66,6 @@ func (r *remoteImage) MediaType() (types.MediaType, error) { return types.DockerManifestSchema2, nil } -// TODO(jonjohnsonjr): Handle manifest lists. func (r *remoteImage) RawManifest() ([]byte, error) { r.manifestLock.Lock() defer r.manifestLock.Unlock() @@ -192,26 +73,18 @@ func (r *remoteImage) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, - // We'll resolve these to an image based on the platform. - types.DockerManifestList, - types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } - // We want an image but the registry has an index, resolve it to an image. - for desc.MediaType == types.DockerManifestList || desc.MediaType == types.OCIImageIndex { - manifest, desc, err = r.matchImage(manifest) - if err != nil { - return nil, err - } - } - r.mediaType = desc.MediaType r.manifest = manifest return r.manifest, nil @@ -302,36 +175,3 @@ func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) digest: h, }, nil } - -// This naively matches the first manifest with matching Architecture and OS. -// -// We should probably use this instead: -// github.com/containerd/containerd/platforms -// -// But first we'd need to migrate to: -// github.com/opencontainers/image-spec/specs-go/v1 -func (r *remoteImage) matchImage(rawIndex []byte) ([]byte, *v1.Descriptor, error) { - index, err := v1.ParseIndexManifest(bytes.NewReader(rawIndex)) - if err != nil { - return nil, nil, err - } - for _, childDesc := range index.Manifests { - // If platform is missing from child descriptor, assume it's amd64/linux. - p := defaultPlatform - if childDesc.Platform != nil { - p = *childDesc.Platform - } - if r.platform.Architecture == p.Architecture && r.platform.OS == p.OS { - childRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), childDesc.Digest), name.StrictValidation) - if err != nil { - return nil, nil, err - } - r.fetcher = fetcher{ - Client: r.Client, - Ref: childRef, - } - return r.fetchManifest([]types.MediaType{childDesc.MediaType}) - } - } - return nil, nil, fmt.Errorf("no matching image for %s/%s, index: %s", r.platform.Architecture, r.platform.OS, string(rawIndex)) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 03afc481ad..2cf9922cbb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -17,14 +17,11 @@ package remote import ( "bytes" "fmt" - "net/http" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -39,27 +36,17 @@ type remoteIndex struct { // Index provides access to a remote index reference, applying functional options // to the underlying imageOpener before resolving the reference into a v1.ImageIndex. func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, + acceptable := []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - return &remoteIndex{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - }, nil + + return desc.ImageIndex() } func (r *remoteIndex) MediaType() (types.MediaType, error) { @@ -80,11 +67,14 @@ func (r *remoteIndex) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } @@ -103,37 +93,93 @@ func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { } func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { - imgRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: imgRef, - Client: r.Client, - }, + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + desc, err := r.childByHash(h) + if err != nil { + return nil, err } - imgCore, err := partial.CompressedToImage(ri) + return desc.ImageIndex() +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) if err != nil { - return imgCore, err + return nil, err } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: r.Ref, - }, nil + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() } -func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { - idxRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +// This naively matches the first manifest with matching Architecture and OS. +// +// We should probably use this instead: +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if platform.Architecture == p.Architecture && platform.OS == p.OS { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %s/%s in index %s", platform.Architecture, platform.OS, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +func (r *remoteIndex) childRef(h v1.Hash) (name.Reference, error) { + return name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref, err := r.childRef(child.Digest) + if err != nil { + return nil, err + } + manifest, desc, err := r.fetchManifest(ref, []types.MediaType{child.MediaType}) if err != nil { return nil, err } - return &remoteIndex{ + return &Descriptor{ fetcher: fetcher{ - Ref: idxRef, + Ref: ref, Client: r.Client, }, + Manifest: manifest, + Descriptor: *desc, + platform: platform, }, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 335e3fe5be..1af7606191 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -22,6 +22,9 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" ) +// ImageOption is a functional option for Image, index, and Get. +type ImageOption func(*imageOpener) error + // WithTransport is a functional option for overriding the default transport // on a remote image func WithTransport(t http.RoundTripper) ImageOption { @@ -56,6 +59,9 @@ func WithAuthFromKeychain(keys authn.Keychain) ImageOption { } } +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// The default platform is amd64/linux. func WithPlatform(p v1.Platform) ImageOption { return func(i *imageOpener) error { i.platform = p diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 00256e8f2e..d1a7e2c8cb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -15,6 +15,7 @@ package tarball import ( + "bytes" "compress/gzip" "io" "io/ioutil" @@ -103,6 +104,18 @@ func LayerFromOpener(opener Opener) (v1.Layer, error) { }, nil } +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { rc, err := opener() if err != nil { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go index 44dbe15aae..2ee81f0b80 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go @@ -28,31 +28,41 @@ import ( // WriteToFile writes in the compressed format to a tarball, on disk. // This is just syntactic sugar wrapping tarball.Write with a new file. -func WriteToFile(p string, tag name.Tag, img v1.Image) error { +func WriteToFile(p string, ref name.Reference, img v1.Image) error { w, err := os.Create(p) if err != nil { return err } defer w.Close() - return Write(tag, img, w) + return Write(ref, img, w) } // MultiWriteToFile writes in the compressed format to a tarball, on disk. // This is just syntactic sugar wrapping tarball.MultiWrite with a new file. func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWriteToFile(p, refToImage) +} + +// MultiRefWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file. +func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image) error { w, err := os.Create(p) if err != nil { return err } defer w.Close() - return MultiWrite(tagToImage, w) + return MultiRefWrite(refToImage, w) } // Write is a wrapper to write a single image and tag to a tarball. -func Write(tag name.Tag, img v1.Image, w io.Writer) error { - return MultiWrite(map[name.Tag]v1.Image{tag: img}, w) +func Write(ref name.Reference, img v1.Image, w io.Writer) error { + return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w) } // MultiWrite writes the contents of each image to the provided reader, in the compressed format. @@ -61,10 +71,23 @@ func Write(tag name.Tag, img v1.Image, w io.Writer) error { // One file for each layer, named after the layer's SHA. // One file for the config blob, named after its SHA. func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWrite(refToImage, w) +} + +// MultiRefWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error { tf := tar.NewWriter(w) defer tf.Close() - imageToTags := dedupTagToImage(tagToImage) + imageToTags := dedupRefToImage(refToImage) var td tarDescriptor for img, tags := range imageToTags { @@ -135,14 +158,20 @@ func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { return writeTarEntry(tf, "manifest.json", bytes.NewReader(tdBytes), int64(len(tdBytes))) } -func dedupTagToImage(tagToImage map[name.Tag]v1.Image) map[v1.Image][]string { +func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string { imageToTags := make(map[v1.Image][]string) - for tag, img := range tagToImage { - if tags, ok := imageToTags[img]; ok { - imageToTags[img] = append(tags, tag.String()) + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; ok && tags != nil { + imageToTags[img] = append(tags, tag.String()) + } else { + imageToTags[img] = []string{tag.String()} + } } else { - imageToTags[img] = []string{tag.String()} + if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } } }