Skip to content

Commit

Permalink
Support creating and pushing OCI types
Browse files Browse the repository at this point in the history 
Adds image exporter option "oci-mediatypes"
Ensures that the images created in the content store
have the correct type which matches the manifest.
Sets the correct media type on the descriptor in push from
reading the type specified in the manifest.
Removes use of distribution manifest packages.

Signed-off-by: Derek McGowan <[email protected]>
  • Loading branch information
@dmcgowan
dmcgowan committed Jun 27, 2018
1 parent 19612b9 commit 9c89633
Show file tree
Hide file tree
Showing 17 changed files with 71 additions and 893 deletions.
6 changes: 4 additions & 2 deletions cache/remotecache/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ import (

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/docker/distribution/manifest"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/push"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe

// own type because oci type can't be pushed and docker type doesn't have annotations
type manifestList struct {
manifest.Versioned
specs.Versioned

MediaType string `json:"mediaType,omitempty"`

// Manifests references platform specific manifests.
Manifests []ocispec.Descriptor `json:"manifests"`
Expand Down
9 changes: 6 additions & 3 deletions client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/continuity/fs/fstest"
"github.com/docker/distribution/manifest/schema2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/testutil/httpserver"
Expand Down Expand Up @@ -612,11 +611,15 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target())
require.NoError(t, err)

var mfst schema2.Manifest
var mfst = struct {
MediaType string `json:"mediaType,omitempty"`
ocispec.Manifest
}{}

err = json.Unmarshal(dt, &mfst)
require.NoError(t, err)

require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
require.Equal(t, images.MediaTypeDockerSchema2Manifest, mfst.MediaType)
require.Equal(t, 2, len(mfst.Layers))

dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest})
Expand Down
14 changes: 13 additions & 1 deletion exporter/containerimage/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const (
keyPush = "push"
keyInsecure = "registry.insecure"
exporterImageConfig = "containerimage.config"
ociTypes = "oci-mediatypes"
)

type Opt struct {
Expand Down Expand Up @@ -69,6 +70,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
i.insecure = b
case exporterImageConfig:
i.config = []byte(v)
case ociTypes:
if v == "" {
i.ociTypes = true
continue
}
b, err := strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.ociTypes = b
default:
logrus.Warnf("image exporter: unknown option %s", k)
}
Expand All @@ -81,6 +92,7 @@ type imageExporterInstance struct {
targetName string
push bool
insecure bool
ociTypes bool
config []byte
}

Expand All @@ -92,7 +104,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
if config, ok := opt[exporterImageConfig]; ok {
e.config = config
}
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
if err != nil {
return nil, err
}
Expand Down
59 changes: 41 additions & 18 deletions exporter/containerimage/writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import (

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2"
"github.com/containerd/containerd/images"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
Expand All @@ -41,7 +41,7 @@ type ImageWriter struct {
opt WriterOpt
}

func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte) (*ocispec.Descriptor, error) {
func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte, oci bool) (*ocispec.Descriptor, error) {
layersDone := oneOffProgress(ctx, "exporting layers")
diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true)
if err != nil {
Expand All @@ -68,17 +68,39 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
return nil, err
}

configDigest := digest.FromBytes(config)

mfst := schema2.Manifest{
Config: distribution.Descriptor{
Digest: configDigest,
Size: int64(len(config)),
MediaType: schema2.MediaTypeImageConfig,
var (
configDigest = digest.FromBytes(config)
manifestType = ocispec.MediaTypeImageManifest
configType = ocispec.MediaTypeImageConfig
layerType = ocispec.MediaTypeImageLayerGzip
)

// Use docker media types for older Docker versions and registries
if !oci {
manifestType = images.MediaTypeDockerSchema2Manifest
configType = images.MediaTypeDockerSchema2Config
layerType = images.MediaTypeDockerSchema2LayerGzip
}

mfst := struct {
// MediaType is reserved in the OCI spec but
// excluded from go types.
MediaType string `json:"mediaType,omitempty"`

ocispec.Manifest
}{
MediaType: manifestType,
Manifest: ocispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
Config: ocispec.Descriptor{
Digest: configDigest,
Size: int64(len(config)),
MediaType: configType,
},
},
}
mfst.SchemaVersion = 2
mfst.MediaType = schema2.MediaTypeManifest

labels := map[string]string{
"containerd.io/gc.ref.content.0": configDigest.String(),
Expand All @@ -89,15 +111,15 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
if err != nil {
return nil, errors.Wrapf(err, "could not find blob %s from contentstore", dp.Blobsum)
}
mfst.Layers = append(mfst.Layers, distribution.Descriptor{
mfst.Layers = append(mfst.Layers, ocispec.Descriptor{
Digest: dp.Blobsum,
Size: info.Size,
MediaType: schema2.MediaTypeLayer,
MediaType: layerType,
})
labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = dp.Blobsum.String()
}

mfstJSON, err := json.Marshal(mfst)
mfstJSON, err := json.MarshalIndent(mfst, "", " ")
if err != nil {
return nil, errors.Wrap(err, "failed to marshal manifest")
}
Expand All @@ -115,8 +137,9 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
mfstDone(nil)

configDesc := ocispec.Descriptor{
Digest: configDigest,
Size: int64(len(config)),
Digest: configDigest,
Size: int64(len(config)),
MediaType: configType,
}
configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())

Expand All @@ -133,7 +156,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
return &ocispec.Descriptor{
Digest: mfstDigest,
Size: int64(len(mfstJSON)),
MediaType: ocispec.MediaTypeImageManifest,
MediaType: manifestType,
}, nil
}

Expand Down
2 changes: 1 addition & 1 deletion exporter/oci/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
if config, ok := opt[exporterImageConfig]; ok {
e.config = config
}
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.opt.Variant == VariantOCI)
if err != nil {
return nil, err
}
Expand Down
6 changes: 5 additions & 1 deletion util/imageutil/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
}

var mfst struct {
Config json.RawMessage `json:"config"`
MediaType string `json:"mediaType"`
Config json.RawMessage `json:"config"`
}

if err := json.Unmarshal(p, &mfst); err != nil {
return "", err
}

if mfst.MediaType != "" {
return mfst.MediaType, nil
}
if mfst.Config != nil {
return images.MediaTypeDockerSchema2Manifest, nil
}
Expand Down
Loading

0 comments on commit 9c89633

Please sign in to comment.