diff --git a/README.md b/README.md index cb7459ad620a0..799cfe2880148 100644 --- a/README.md +++ b/README.md @@ -230,8 +230,8 @@ Keys supported by image output: * `unpack=true`: unpack image after creation (for use with containerd) * `dangling-name-prefix=[value]`: name image with `prefix@` , used for anonymous images * `name-canonical=true`: add additional canonical name `name@` -* `compression=[uncompressed,gzip]`: choose compression type for layer, gzip is default value - +* `compression=[uncompressed,gzip]`: choose compression type for layers newly created and cached, gzip is default value +* `compression-all=[uncompressed,gzip]`: choose compression type for all layers (including already existing layers) for exporting. compression type specified by `compression` flag is respected by default. If credentials are required, `buildctl` will attempt to read Docker configuration file `$DOCKER_CONFIG/config.json`. `$DOCKER_CONFIG` defaults to `~/.docker`. diff --git a/client/client_test.go b/client/client_test.go index f4a76fea452b8..0ed55e20d970b 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1857,6 +1857,21 @@ func testBuildExportWithUncompressed(t *testing.T, sb integration.Sandbox) { }, nil) require.NoError(t, err) + allCompressedTarget := registry + "/buildkit/build/exporter:withallcompressed" + _, err = c.Solve(context.TODO(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": allCompressedTarget, + "push": "true", + "compression-all": "gzip", + }, + }, + }, + }, nil) + require.NoError(t, err) + if cdAddress == "" { t.Skip("rest of test requires containerd worker") } @@ -1865,9 +1880,12 @@ func testBuildExportWithUncompressed(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) err = client.ImageService().Delete(ctx, compressedTarget, images.SynchronousDelete()) require.NoError(t, err) + err = client.ImageService().Delete(ctx, allCompressedTarget, images.SynchronousDelete()) + require.NoError(t, err) checkAllReleasable(t, c, sb, true) + // check if the new layer is compressed with compression option img, err := client.Pull(ctx, compressedTarget) require.NoError(t, err) @@ -1906,6 +1924,51 @@ func testBuildExportWithUncompressed(t *testing.T, sb integration.Sandbox) { require.True(t, ok) require.Equal(t, int32(item.Header.Typeflag), tar.TypeReg) require.Equal(t, []byte("gzip"), item.Data) + + err = client.ImageService().Delete(ctx, compressedTarget, images.SynchronousDelete()) + require.NoError(t, err) + + checkAllReleasable(t, c, sb, true) + + // check if all layers are compressed with compression-all option + img, err = client.Pull(ctx, allCompressedTarget) + require.NoError(t, err) + + dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target()) + require.NoError(t, err) + + mfst = struct { + MediaType string `json:"mediaType,omitempty"` + ocispec.Manifest + }{} + + err = json.Unmarshal(dt, &mfst) + require.NoError(t, err) + require.Equal(t, 2, len(mfst.Layers)) + require.Equal(t, images.MediaTypeDockerSchema2LayerGzip, mfst.Layers[0].MediaType) + require.Equal(t, images.MediaTypeDockerSchema2LayerGzip, mfst.Layers[1].MediaType) + + dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest}) + require.NoError(t, err) + + m, err = testutil.ReadTarToMap(dt, true) + require.NoError(t, err) + + item, ok = m["data"] + require.True(t, ok) + require.Equal(t, int32(item.Header.Typeflag), tar.TypeReg) + require.Equal(t, []byte("uncompressed"), item.Data) + + dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[1].Digest}) + require.NoError(t, err) + + m, err = testutil.ReadTarToMap(dt, true) + require.NoError(t, err) + + item, ok = m["data"] + require.True(t, ok) + require.Equal(t, int32(item.Header.Typeflag), tar.TypeReg) + require.Equal(t, []byte("gzip"), item.Data) } func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) { diff --git a/exporter/containerimage/converter.go b/exporter/containerimage/converter.go new file mode 100644 index 0000000000000..1b58b86cfe5f2 --- /dev/null +++ b/exporter/containerimage/converter.go @@ -0,0 +1,94 @@ +package containerimage + +import ( + "compress/gzip" + "context" + "fmt" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter/uncompress" + "github.com/containerd/containerd/labels" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func gzipLayerConvertFunc(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if !images.IsLayerType(desc.MediaType) || isGzipCompressedType(desc.MediaType) { + // No conversion. No need to return an error here. + return nil, nil + } + + // prepare the source and destination + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labelz := info.Labels + if labelz == nil { + labelz = make(map[string]string) + } + ra, err := cs.ReaderAt(ctx, desc) + if err != nil { + return nil, err + } + defer ra.Close() + ref := fmt.Sprintf("convert-gzip-from-%s", desc.Digest) + w, err := cs.Writer(ctx, content.WithRef(ref)) + if err != nil { + return nil, err + } + defer w.Close() + if err := w.Truncate(0); err != nil { // Old written data possibly remains + return nil, err + } + zw := gzip.NewWriter(w) + defer zw.Close() + + // convert this layer + diffID := digest.Canonical.Digester() + if _, err := io.Copy(zw, io.TeeReader(io.NewSectionReader(ra, 0, ra.Size()), diffID.Hash())); err != nil { + return nil, err + } + if err := zw.Close(); err != nil { // Flush the writer + return nil, err + } + labelz[labels.LabelUncompressed] = diffID.Digest().String() // update diffID label + if err = w.Commit(ctx, 0, "", content.WithLabels(labelz)); err != nil && !errdefs.IsAlreadyExists(err) { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + info, err = cs.Info(ctx, w.Digest()) + if err != nil { + return nil, err + } + + newDesc := desc + if uncompress.IsUncompressedType(newDesc.MediaType) { + if images.IsDockerType(newDesc.MediaType) { + newDesc.MediaType += ".gzip" + } else { + newDesc.MediaType += "+gzip" + } + } + newDesc.Digest = info.Digest + newDesc.Size = info.Size + return &newDesc, nil +} + +func isGzipCompressedType(mt string) bool { + switch mt { + case + images.MediaTypeDockerSchema2LayerGzip, + images.MediaTypeDockerSchema2LayerForeignGzip, + ocispec.MediaTypeImageLayerGzip, + ocispec.MediaTypeImageLayerNonDistributableGzip: + return true + default: + return false + } +} diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 0031facd92d41..a25461d92b98d 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -29,15 +29,16 @@ import ( ) const ( - keyImageName = "name" - keyPush = "push" - keyPushByDigest = "push-by-digest" - keyInsecure = "registry.insecure" - keyUnpack = "unpack" - keyDanglingPrefix = "dangling-name-prefix" - keyNameCanonical = "name-canonical" - keyLayerCompression = "compression" - ociTypes = "oci-mediatypes" + keyImageName = "name" + keyPush = "push" + keyPushByDigest = "push-by-digest" + keyInsecure = "registry.insecure" + keyUnpack = "unpack" + keyDanglingPrefix = "dangling-name-prefix" + keyNameCanonical = "name-canonical" + keyLayerCompression = "compression" + keyLayerCompressionAll = "compression-all" + ociTypes = "oci-mediatypes" ) type Opt struct { @@ -63,8 +64,9 @@ func New(opt Opt) (exporter.Exporter, error) { func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) { i := &imageExporterInstance{ - imageExporter: e, - layerCompression: compression.Default, + imageExporter: e, + layerCompression: compression.Default, + layerCompressionAll: compression.Any, } for k, v := range opt { @@ -142,6 +144,15 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp default: return nil, errors.Errorf("unsupported layer compression type: %v", v) } + case keyLayerCompressionAll: + switch v { + case "gzip": + i.layerCompressionAll = compression.Gzip + case "uncompressed": + i.layerCompressionAll = compression.Uncompressed + default: + return nil, errors.Errorf("unsupported layer compression type: %v", v) + } default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -154,16 +165,17 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp type imageExporterInstance struct { *imageExporter - targetName string - push bool - pushByDigest bool - unpack bool - insecure bool - ociTypes bool - nameCanonical bool - danglingPrefix string - layerCompression compression.Type - meta map[string][]byte + targetName string + push bool + pushByDigest bool + unpack bool + insecure bool + ociTypes bool + nameCanonical bool + danglingPrefix string + layerCompression compression.Type + layerCompressionAll compression.Type + meta map[string][]byte } func (e *imageExporterInstance) Name() string { @@ -184,7 +196,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, } defer done(context.TODO()) - desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, sessionID) + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.layerCompressionAll, sessionID) if err != nil { return nil, err } diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index ff88f68ed5adf..fc6562b269c4d 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -11,6 +11,8 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter" + "github.com/containerd/containerd/images/converter/uncompress" "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/exporter" @@ -44,7 +46,7 @@ type ImageWriter struct { opt WriterOpt } -func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, sessionID string) (*ocispec.Descriptor, error) { +func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, forceCompressionType compression.Type, sessionID string) (*ocispec.Descriptor, error) { platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey] if len(inp.Refs) > 0 && !ok { @@ -64,6 +66,16 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool mfstDesc.Annotations = make(map[string]string) } mfstDesc.Annotations["config.digest"] = configDesc.Digest.String() + + if forceCompressionType != compression.Any { + cvtDone := oneOffProgress(ctx, "converting manifest "+(*mfstDesc).Digest.String()) + mfstDesc, err = ic.ensureCompressionType(ctx, *mfstDesc, remotes, forceCompressionType) + if err != nil { + return nil, cvtDone(errors.Wrapf(err, "error converting manifest compression %s", (*mfstDesc).Digest)) + } + cvtDone(nil) + } + return mfstDesc, nil } @@ -145,6 +157,16 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool } idxDone(nil) + if forceCompressionType != compression.Any { + cvtDone := oneOffProgress(ctx, "converting manifest list "+idxDigest.String()) + newDesc, err := ic.ensureCompressionType(ctx, idxDesc, remotes, forceCompressionType) + if err != nil { + return nil, cvtDone(errors.Wrapf(err, "error converting manifest list compression %s", idxDigest)) + } + idxDesc = *newDesc + cvtDone(nil) + } + return &idxDesc, nil } @@ -177,6 +199,37 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, compressionType compres return out, nil } +func (ic *ImageWriter) ensureCompressionType(ctx context.Context, idxDesc ocispec.Descriptor, remotes []solver.Remote, compressionType compression.Type) (*ocispec.Descriptor, error) { + var layerConvertFunc converter.ConvertFunc + switch compressionType { + case compression.Any: + // If no compression type is specified, we don't need to convert it. + // lazy layers remain lazy. + return &idxDesc, nil + case compression.Uncompressed: + layerConvertFunc = uncompress.LayerConvertFunc + case compression.Gzip: + layerConvertFunc = gzipLayerConvertFunc + default: + return nil, fmt.Errorf("unknown compression type during conversion: %q", compressionType) + } + + // unlazy layers as converter uses layer contents in the content store + // TODO(ktock): un-lazy only layers whose type is different from the target, selectively. + // this will requires to patch containerd converter API. + for _, r := range remotes { + if unlazier, ok := r.Provider.(cache.Unlazier); ok { + if err := unlazier.Unlazy(ctx); err != nil { + return nil, err + } + } + } + + // convert the index. respect the platform and spec(OCI or Docker) of the original index. + return converter.DefaultIndexConvertFunc(layerConvertFunc, false, platforms.All)( + ctx, ic.opt.ContentStore, idxDesc) +} + func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, remote *solver.Remote, oci bool, inlineCache []byte) (*ocispec.Descriptor, *ocispec.Descriptor, error) { if len(config) == 0 { var err error diff --git a/exporter/oci/export.go b/exporter/oci/export.go index c79e0e441c719..0d86e7e7e6280 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -27,11 +27,12 @@ import ( type ExporterVariant string const ( - keyImageName = "name" - keyLayerCompression = "compression" - VariantOCI = "oci" - VariantDocker = "docker" - ociTypes = "oci-mediatypes" + keyImageName = "name" + keyLayerCompression = "compression" + VariantOCI = "oci" + VariantDocker = "docker" + ociTypes = "oci-mediatypes" + keyLayerCompressionAll = "compression-all" ) type Opt struct { @@ -53,8 +54,9 @@ func New(opt Opt) (exporter.Exporter, error) { func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) { var ot *bool i := &imageExporterInstance{ - imageExporter: e, - layerCompression: compression.Default, + imageExporter: e, + layerCompression: compression.Default, + layerCompressionAll: compression.Any, } for k, v := range opt { switch k { @@ -69,6 +71,15 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp default: return nil, errors.Errorf("unsupported layer compression type: %v", v) } + case keyLayerCompressionAll: + switch v { + case "gzip": + i.layerCompressionAll = compression.Gzip + case "uncompressed": + i.layerCompressionAll = compression.Uncompressed + default: + return nil, errors.Errorf("unsupported layer compression type: %v", v) + } case ociTypes: ot = new(bool) if v == "" { @@ -97,10 +108,11 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp type imageExporterInstance struct { *imageExporter - meta map[string][]byte - name string - ociTypes bool - layerCompression compression.Type + meta map[string][]byte + name string + ociTypes bool + layerCompression compression.Type + layerCompressionAll compression.Type } func (e *imageExporterInstance) Name() string { @@ -125,7 +137,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, } defer done(context.TODO()) - desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, sessionID) + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.layerCompressionAll, sessionID) if err != nil { return nil, err } diff --git a/util/compression/compression.go b/util/compression/compression.go index 654b056335d5c..a65ac9686e1b9 100644 --- a/util/compression/compression.go +++ b/util/compression/compression.go @@ -23,6 +23,9 @@ const ( // Gzip is used for blob data. Gzip + // Any means any compression. + Any + // UnknownCompression means not supported yet. UnknownCompression Type = -1 ) @@ -35,6 +38,8 @@ func (ct Type) String() string { return "uncompressed" case Gzip: return "gzip" + case Any: + return "any" default: return "unknown" } diff --git a/vendor/github.com/containerd/containerd/images/converter/converter.go b/vendor/github.com/containerd/containerd/images/converter/converter.go new file mode 100644 index 0000000000000..441e0169efc18 --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/converter/converter.go @@ -0,0 +1,126 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package converter provides image converter +package converter + +import ( + "context" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" +) + +type convertOpts struct { + layerConvertFunc ConvertFunc + docker2oci bool + indexConvertFunc ConvertFunc + platformMC platforms.MatchComparer +} + +// Opt is an option for Convert() +type Opt func(*convertOpts) error + +// WithLayerConvertFunc specifies the function that converts layers. +func WithLayerConvertFunc(fn ConvertFunc) Opt { + return func(copts *convertOpts) error { + copts.layerConvertFunc = fn + return nil + } +} + +// WithDockerToOCI converts Docker media types into OCI ones. +func WithDockerToOCI(v bool) Opt { + return func(copts *convertOpts) error { + copts.docker2oci = true + return nil + } +} + +// WithPlatform specifies the platform. +// Defaults to all platforms. +func WithPlatform(p platforms.MatchComparer) Opt { + return func(copts *convertOpts) error { + copts.platformMC = p + return nil + } +} + +// WithIndexConvertFunc specifies the function that converts manifests and index (manifest lists). +// Defaults to DefaultIndexConvertFunc. +func WithIndexConvertFunc(fn ConvertFunc) Opt { + return func(copts *convertOpts) error { + copts.indexConvertFunc = fn + return nil + } +} + +// Client is implemented by *containerd.Client . +type Client interface { + WithLease(ctx context.Context, opts ...leases.Opt) (context.Context, func(context.Context) error, error) + ContentStore() content.Store + ImageService() images.Store +} + +// Convert converts an image. +func Convert(ctx context.Context, client Client, dstRef, srcRef string, opts ...Opt) (*images.Image, error) { + var copts convertOpts + for _, o := range opts { + if err := o(&copts); err != nil { + return nil, err + } + } + if copts.platformMC == nil { + copts.platformMC = platforms.All + } + if copts.indexConvertFunc == nil { + copts.indexConvertFunc = DefaultIndexConvertFunc(copts.layerConvertFunc, copts.docker2oci, copts.platformMC) + } + + ctx, done, err := client.WithLease(ctx) + if err != nil { + return nil, err + } + defer done(ctx) + + cs := client.ContentStore() + is := client.ImageService() + srcImg, err := is.Get(ctx, srcRef) + if err != nil { + return nil, err + } + + dstDesc, err := copts.indexConvertFunc(ctx, cs, srcImg.Target) + if err != nil { + return nil, err + } + + dstImg := srcImg + dstImg.Name = dstRef + if dstDesc != nil { + dstImg.Target = *dstDesc + } + var res images.Image + if dstRef != srcRef { + _ = is.Delete(ctx, dstRef) + res, err = is.Create(ctx, dstImg) + } else { + res, err = is.Update(ctx, dstImg) + } + return &res, err +} diff --git a/vendor/github.com/containerd/containerd/images/converter/default.go b/vendor/github.com/containerd/containerd/images/converter/default.go new file mode 100644 index 0000000000000..13dd513d8254a --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/converter/default.go @@ -0,0 +1,442 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package converter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +// ConvertFunc returns a converted content descriptor. +// When the content was not converted, ConvertFunc returns nil. +type ConvertFunc func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) + +// DefaultIndexConvertFunc is the default convert func used by Convert. +func DefaultIndexConvertFunc(layerConvertFunc ConvertFunc, docker2oci bool, platformMC platforms.MatchComparer) ConvertFunc { + c := &defaultConverter{ + layerConvertFunc: layerConvertFunc, + docker2oci: docker2oci, + platformMC: platformMC, + diffIDMap: make(map[digest.Digest]digest.Digest), + } + return c.convert +} + +type defaultConverter struct { + layerConvertFunc ConvertFunc + docker2oci bool + platformMC platforms.MatchComparer + diffIDMap map[digest.Digest]digest.Digest // key: old diffID, value: new diffID + diffIDMapMu sync.RWMutex +} + +// convert dispatches desc.MediaType and calls c.convert{Layer,Manifest,Index,Config}. +// +// Also converts media type if c.docker2oci is set. +func (c *defaultConverter) convert(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + newDesc *ocispec.Descriptor + err error + ) + if images.IsLayerType(desc.MediaType) { + newDesc, err = c.convertLayer(ctx, cs, desc) + } else if images.IsManifestType(desc.MediaType) { + newDesc, err = c.convertManifest(ctx, cs, desc) + } else if images.IsIndexType(desc.MediaType) { + newDesc, err = c.convertIndex(ctx, cs, desc) + } else if images.IsConfigType(desc.MediaType) { + newDesc, err = c.convertConfig(ctx, cs, desc) + } + if err != nil { + return nil, err + } + if images.IsDockerType(desc.MediaType) { + if c.docker2oci { + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.MediaType = ConvertDockerMediaTypeToOCI(newDesc.MediaType) + } else if (newDesc == nil && len(desc.Annotations) != 0) || (newDesc != nil && len(newDesc.Annotations) != 0) { + // Annotations is supported only on OCI manifest. + // We need to remove annotations for Docker media types. + if newDesc == nil { + newDesc = copyDesc(desc) + } + newDesc.Annotations = nil + } + } + logrus.WithField("old", desc).WithField("new", newDesc).Debugf("converted") + return newDesc, nil +} + +func copyDesc(desc ocispec.Descriptor) *ocispec.Descriptor { + descCopy := desc + return &descCopy +} + +// convertLayer converts image image layers if c.layerConvertFunc is set. +// +// c.layerConvertFunc can be nil, e.g., for converting Docker media types to OCI ones. +func (c *defaultConverter) convertLayer(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if c.layerConvertFunc != nil { + return c.layerConvertFunc(ctx, cs, desc) + } + return nil, nil +} + +// convertManifest converts image manifests. +// +// - clears `.mediaType` if the target format is OCI +// +// - records diff ID changes in c.diffIDMap +func (c *defaultConverter) convertManifest(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + manifest DualManifest + modified bool + ) + labels, err := readJSON(ctx, cs, &manifest, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if images.IsDockerType(manifest.MediaType) && c.docker2oci { + manifest.MediaType = "" + modified = true + } + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, l := range manifest.Layers { + i := i + l := l + oldDiffID, err := images.GetDiffID(ctx, cs, l) + if err != nil { + return nil, err + } + eg.Go(func() error { + newL, err := c.convert(ctx2, cs, l) + if err != nil { + return err + } + if newL != nil { + mu.Lock() + // update GC labels + ClearGCLabels(labels, l.Digest) + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i) + labels[labelKey] = newL.Digest.String() + manifest.Layers[i] = *newL + modified = true + mu.Unlock() + + // diffID changes if the tar entries were modified. + // diffID stays same if only the compression type was changed. + // When diffID changed, add a map entry so that we can update image config. + newDiffID, err := images.GetDiffID(ctx, cs, *newL) + if err != nil { + return err + } + if newDiffID != oldDiffID { + c.diffIDMapMu.Lock() + c.diffIDMap[oldDiffID] = newDiffID + c.diffIDMapMu.Unlock() + } + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + newConfig, err := c.convert(ctx, cs, manifest.Config) + if err != nil { + return nil, err + } + if newConfig != nil { + ClearGCLabels(labels, manifest.Config.Digest) + labels["containerd.io/gc.ref.content.config"] = newConfig.Digest.String() + manifest.Config = *newConfig + modified = true + } + + if modified { + return writeJSON(ctx, cs, &manifest, desc, labels) + } + return nil, nil +} + +// convertIndex converts image index. +// +// - clears `.mediaType` if the target format is OCI +// +// - clears manifest entries that do not match c.platformMC +func (c *defaultConverter) convertIndex(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + index DualIndex + modified bool + ) + labels, err := readJSON(ctx, cs, &index, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if images.IsDockerType(index.MediaType) && c.docker2oci { + index.MediaType = "" + modified = true + } + + newManifests := make([]ocispec.Descriptor, len(index.Manifests)) + newManifestsToBeRemoved := make(map[int]struct{}) // slice index + var mu sync.Mutex + eg, ctx2 := errgroup.WithContext(ctx) + for i, mani := range index.Manifests { + i := i + mani := mani + labelKey := fmt.Sprintf("containerd.io/gc.ref.content.m.%d", i) + eg.Go(func() error { + if mani.Platform != nil && !c.platformMC.Match(*mani.Platform) { + mu.Lock() + ClearGCLabels(labels, mani.Digest) + newManifestsToBeRemoved[i] = struct{}{} + modified = true + mu.Unlock() + return nil + } + newMani, err := c.convert(ctx2, cs, mani) + if err != nil { + return err + } + mu.Lock() + if newMani != nil { + ClearGCLabels(labels, mani.Digest) + labels[labelKey] = newMani.Digest.String() + // NOTE: for keeping manifest order, we specify `i` index explicitly + newManifests[i] = *newMani + modified = true + } else { + newManifests[i] = mani + } + mu.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + if modified { + var newManifestsClean []ocispec.Descriptor + for i, m := range newManifests { + if _, ok := newManifestsToBeRemoved[i]; !ok { + newManifestsClean = append(newManifestsClean, m) + } + } + index.Manifests = newManifestsClean + return writeJSON(ctx, cs, &index, desc, labels) + } + return nil, nil +} + +// convertConfig converts image config contents. +// +// - updates `.rootfs.diff_ids` using c.diffIDMap . +// +// - clears legacy `.config.Image` and `.container_config.Image` fields if `.rootfs.diff_ids` was updated. +func (c *defaultConverter) convertConfig(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + var ( + cfg DualConfig + cfgAsOCI ocispec.Image // read only, used for parsing cfg + modified bool + ) + + labels, err := readJSON(ctx, cs, &cfg, desc) + if err != nil { + return nil, err + } + if labels == nil { + labels = make(map[string]string) + } + if _, err := readJSON(ctx, cs, &cfgAsOCI, desc); err != nil { + return nil, err + } + + if rootfs := cfgAsOCI.RootFS; rootfs.Type == "layers" { + rootfsModified := false + c.diffIDMapMu.RLock() + for i, oldDiffID := range rootfs.DiffIDs { + if newDiffID, ok := c.diffIDMap[oldDiffID]; ok && newDiffID != oldDiffID { + rootfs.DiffIDs[i] = newDiffID + rootfsModified = true + } + } + c.diffIDMapMu.RUnlock() + if rootfsModified { + rootfsB, err := json.Marshal(rootfs) + if err != nil { + return nil, err + } + cfg["rootfs"] = (*json.RawMessage)(&rootfsB) + modified = true + } + } + + if modified { + // cfg may have dummy value for legacy `.config.Image` and `.container_config.Image` + // We should clear the ID if we changed the diff IDs. + if _, err := clearDockerV1DummyID(cfg); err != nil { + return nil, err + } + return writeJSON(ctx, cs, &cfg, desc, labels) + } + return nil, nil +} + +// clearDockerV1DummyID clears the dummy values for legacy `.config.Image` and `.container_config.Image`. +// Returns true if the cfg was modified. +func clearDockerV1DummyID(cfg DualConfig) (bool, error) { + var modified bool + f := func(k string) error { + if configX, ok := cfg[k]; ok && configX != nil { + var configField map[string]*json.RawMessage + if err := json.Unmarshal(*configX, &configField); err != nil { + return err + } + delete(configField, "Image") + b, err := json.Marshal(configField) + if err != nil { + return err + } + cfg[k] = (*json.RawMessage)(&b) + modified = true + } + return nil + } + if err := f("config"); err != nil { + return modified, err + } + if err := f("container_config"); err != nil { + return modified, err + } + return modified, nil +} + +// ObjectWithMediaType represents an object with a MediaType field +type ObjectWithMediaType struct { + // MediaType appears on Docker manifests and manifest lists. + // MediaType does not appear on OCI manifests and index + MediaType string `json:"mediaType,omitempty"` +} + +// DualManifest covers Docker manifest and OCI manifest +type DualManifest struct { + ocispec.Manifest + ObjectWithMediaType +} + +// DualIndex covers Docker manifest list and OCI index +type DualIndex struct { + ocispec.Index + ObjectWithMediaType +} + +// DualConfig covers Docker config (v1.0, v1.1, v1.2) and OCI config. +// Unmarshalled as map[string]*json.RawMessage to retain unknown fields on remarshalling. +type DualConfig map[string]*json.RawMessage + +func readJSON(ctx context.Context, cs content.Store, x interface{}, desc ocispec.Descriptor) (map[string]string, error) { + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + labels := info.Labels + b, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, x); err != nil { + return nil, err + } + return labels, nil +} + +func writeJSON(ctx context.Context, cs content.Store, x interface{}, oldDesc ocispec.Descriptor, labels map[string]string) (*ocispec.Descriptor, error) { + b, err := json.Marshal(x) + if err != nil { + return nil, err + } + dgst := digest.SHA256.FromBytes(b) + ref := fmt.Sprintf("converter-write-json-%s", dgst.String()) + w, err := content.OpenWriter(ctx, cs, content.WithRef(ref)) + if err != nil { + return nil, err + } + if err := content.Copy(ctx, w, bytes.NewReader(b), int64(len(b)), dgst, content.WithLabels(labels)); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := oldDesc + newDesc.Size = int64(len(b)) + newDesc.Digest = dgst + return &newDesc, nil +} + +// ConvertDockerMediaTypeToOCI converts a media type string +func ConvertDockerMediaTypeToOCI(mt string) string { + switch mt { + case images.MediaTypeDockerSchema2ManifestList: + return ocispec.MediaTypeImageIndex + case images.MediaTypeDockerSchema2Manifest: + return ocispec.MediaTypeImageManifest + case images.MediaTypeDockerSchema2LayerGzip: + return ocispec.MediaTypeImageLayerGzip + case images.MediaTypeDockerSchema2LayerForeignGzip: + return ocispec.MediaTypeImageLayerNonDistributableGzip + case images.MediaTypeDockerSchema2Layer: + return ocispec.MediaTypeImageLayer + case images.MediaTypeDockerSchema2LayerForeign: + return ocispec.MediaTypeImageLayerNonDistributable + case images.MediaTypeDockerSchema2Config: + return ocispec.MediaTypeImageConfig + default: + return mt + } +} + +// ClearGCLabels clears GC labels for the given digest. +func ClearGCLabels(labels map[string]string, dgst digest.Digest) { + for k, v := range labels { + if v == dgst.String() && strings.HasPrefix(k, "containerd.io/gc.ref.content") { + delete(labels, k) + } + } +} diff --git a/vendor/github.com/containerd/containerd/images/converter/uncompress/uncompress.go b/vendor/github.com/containerd/containerd/images/converter/uncompress/uncompress.go new file mode 100644 index 0000000000000..b77294cb15363 --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/converter/uncompress/uncompress.go @@ -0,0 +1,122 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package uncompress + +import ( + "compress/gzip" + "context" + "fmt" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter" + "github.com/containerd/containerd/labels" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +var _ converter.ConvertFunc = LayerConvertFunc + +// LayerConvertFunc converts tar.gz layers into uncompressed tar layers. +// Media type is changed, e.g., "application/vnd.oci.image.layer.v1.tar+gzip" -> "application/vnd.oci.image.layer.v1.tar" +func LayerConvertFunc(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + if !images.IsLayerType(desc.MediaType) || IsUncompressedType(desc.MediaType) { + // No conversion. No need to return an error here. + return nil, nil + } + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + readerAt, err := cs.ReaderAt(ctx, desc) + if err != nil { + return nil, err + } + defer readerAt.Close() + sr := io.NewSectionReader(readerAt, 0, desc.Size) + newR, err := gzip.NewReader(sr) + if err != nil { + return nil, err + } + defer newR.Close() + ref := fmt.Sprintf("convert-uncompress-from-%s", desc.Digest) + w, err := cs.Writer(ctx, content.WithRef(ref)) + if err != nil { + return nil, err + } + defer w.Close() + + // Reset the writing position + // Old writer possibly remains without aborted + // (e.g. conversion interrupted by a signal) + if err := w.Truncate(0); err != nil { + return nil, err + } + + n, err := io.Copy(w, newR) + if err != nil { + return nil, err + } + if err := newR.Close(); err != nil { + return nil, err + } + // no need to retain "containerd.io/uncompressed" label, but retain other labels ("containerd.io/distribution.source.*") + labelsMap := info.Labels + delete(labelsMap, labels.LabelUncompressed) + if err = w.Commit(ctx, 0, "", content.WithLabels(labelsMap)); err != nil && !errdefs.IsAlreadyExists(err) { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + newDesc := desc + newDesc.Digest = w.Digest() + newDesc.Size = n + newDesc.MediaType = convertMediaType(newDesc.MediaType) + return &newDesc, nil +} + +// IsUncompressedType returns whether the provided media type is considered +// an uncompressed layer type +func IsUncompressedType(mt string) bool { + switch mt { + case + images.MediaTypeDockerSchema2Layer, + images.MediaTypeDockerSchema2LayerForeign, + ocispec.MediaTypeImageLayer, + ocispec.MediaTypeImageLayerNonDistributable: + return true + default: + return false + } +} + +func convertMediaType(mt string) string { + switch mt { + case images.MediaTypeDockerSchema2LayerGzip: + return images.MediaTypeDockerSchema2Layer + case images.MediaTypeDockerSchema2LayerForeignGzip: + return images.MediaTypeDockerSchema2LayerForeign + case ocispec.MediaTypeImageLayerGzip: + return ocispec.MediaTypeImageLayer + case ocispec.MediaTypeImageLayerNonDistributableGzip: + return ocispec.MediaTypeImageLayerNonDistributable + default: + return mt + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0c42927c7a1d5..5a57028aa39a0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -72,6 +72,8 @@ github.com/containerd/containerd/gc github.com/containerd/containerd/identifiers github.com/containerd/containerd/images github.com/containerd/containerd/images/archive +github.com/containerd/containerd/images/converter +github.com/containerd/containerd/images/converter/uncompress github.com/containerd/containerd/labels github.com/containerd/containerd/leases github.com/containerd/containerd/leases/proxy