Skip to content

Commit

Permalink
prepare dynamic format recognition for object set serialization files…
Browse files Browse the repository at this point in the history 
…ystem formats (#97)

* prepare dynamic format recognition for object set serialization filesystem formats

Switch from a static set of data for configure the filesystem format
to a potentially dynamic format specification supporting format
discovery based on the actual state of the filesystem.

The static structure describing fixed values for various features
is replaced by an interface with an additional discover method.

* fix lint and introduce OCI format handling

* make format configurable

* format read/write tests

* review

* rebase and format
  • Loading branch information
@mandelsoft
mandelsoft authored Oct 12, 2022
1 parent b9c2d5a commit 9f733c1
Show file tree
Hide file tree
Showing 47 changed files with 1,032 additions and 457 deletions.
4 changes: 2 additions & 2 deletions cmds/helminstaller/testhelper/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ func NewTestEnv(opts ...env.Option) *TestEnv {
}
}

func (e *TestEnv) ApplyOption(opts *accessio.Options) {
e.Builder.ApplyOption(opts)
func (e *TestEnv) ApplyOption(opts accessio.Options) error {
return e.Builder.ApplyOption(opts)
}

func (e *TestEnv) ConfigContext() config.Context {
Expand Down
6 changes: 5 additions & 1 deletion cmds/ocm/commands/ocicmds/artefacts/download/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ var _ = Describe("Test Environment", func() {
/tmp/res: downloaded
`))
Expect(env.DirExists(OUT)).To(BeTrue())
Expect(env.ReadFile(OUT + "/" + artefactset.ArtefactSetDescriptorFileName)).To(Equal([]byte("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"size\":342,\"annotations\":{\"cloud.gardener.ocm/tags\":\"v1\"}}],\"annotations\":{\"cloud.gardener.ocm/main\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}}")))
tags := ""
if artefactset.IsOCIDefaultFormat() {
tags = ",\"org.opencontainers.image.ref.name\":\"v1\""
}
Expect(env.ReadFile(OUT + "/" + artefactset.DefaultArtefactSetDescriptorFileName)).To(Equal([]byte("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"size\":342,\"annotations\":{\"cloud.gardener.ocm/tags\":\"v1\"" + tags + "}}],\"annotations\":{\"cloud.gardener.ocm/main\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}}")))
})
})
47 changes: 43 additions & 4 deletions cmds/ocm/commands/ocmcmds/components/sign/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/open-component-model/ocm/cmds/ocm/testhelper"
. "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper"
. "github.com/open-component-model/ocm/pkg/testutils"

"github.com/mandelsoft/vfs/pkg/vfs"
Expand All @@ -46,9 +47,6 @@ const COMPONENTA = "github.com/mandelsoft/test"
const COMPONENTB = "github.com/mandelsoft/ref"
const OUT = "/tmp/res"
const OCIPATH = "/tmp/oci"
const OCINAMESPACE = "ocm/value"
const OCINAMESPACE2 = "ocm/ref"
const OCIVERSION = "v2.0"
const OCIHOST = "alias"

const SIGNATURE = "test"
Expand Down Expand Up @@ -78,6 +76,45 @@ var _ = Describe("access method", func() {
})

Context("valid", func() {
BeforeEach(func() {
FakeOCIRepo(env.Builder, OCIPATH, OCIHOST)

env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() {
OCIManifest1(env.Builder)
OCIManifest2(env.Builder)
})

env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() {
env.Component(COMPONENTA, func() {
env.Version(VERSION, func() {
env.Provider(PROVIDER)
env.Resource("testdata", "", "PlainText", metav1.LocalRelation, func() {
env.BlobStringData(mime.MIME_TEXT, "testdata")
})
env.Resource("value", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() {
env.Access(
ociartefact.New(oci.StandardOCIRef(OCIHOST+".alias", OCINAMESPACE, OCIVERSION)),
)
env.Label("transportByValue", true)
})
env.Resource("ref", "", resourcetypes.OCI_IMAGE, metav1.LocalRelation, func() {
env.Access(
ociartefact.New(oci.StandardOCIRef(OCIHOST+".alias", OCINAMESPACE2, OCIVERSION)),
)
})
})
})
env.Component(COMPONENTB, func() {
env.Version(VERSION, func() {
env.Provider(PROVIDER)
env.Resource("otherdata", "", "PlainText", metav1.LocalRelation, func() {
env.BlobStringData(mime.MIME_TEXT, "otherdata")
})
env.Reference("ref", COMPONENTA, VERSION)
})
})
})
})

It("sign component archive", func() {
prepareEnv(env, ARCH, ARCH)
Expand Down Expand Up @@ -203,7 +240,9 @@ Error: {signing: failed resolving component reference ref[github.com/mandelsoft/
})

func prepareEnv(env *TestEnv, componentAArchive, componentBArchive string) {
env.OCIContext().SetAlias(OCIHOST, ctfoci.NewRepositorySpec(accessobj.ACC_READONLY, OCIPATH, accessio.PathFileSystem(env.FileSystem())))
spec, err := ctfoci.NewRepositorySpec(accessobj.ACC_READONLY, OCIPATH, accessio.PathFileSystem(env.FileSystem()))
Expect(err).To(Succeed())
env.OCIContext().SetAlias(OCIHOST, spec)

env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() {
env.Namespace(OCINAMESPACE, func() {
Expand Down
36 changes: 9 additions & 27 deletions cmds/ocm/commands/ocmcmds/components/transfer/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/open-component-model/ocm/cmds/ocm/testhelper"
. "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper"
. "github.com/open-component-model/ocm/pkg/testutils"

"github.com/open-component-model/ocm/pkg/common/accessio"
"github.com/open-component-model/ocm/pkg/common/accessobj"
"github.com/open-component-model/ocm/pkg/contexts/oci"
"github.com/open-component-model/ocm/pkg/contexts/oci/artdesc"
"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artefactset"
ctfoci "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ctf"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartefact"
metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1"
Expand All @@ -45,9 +45,6 @@ const COMPONENT = "github.com/mandelsoft/test"
const COMPONENT2 = "github.com/mandelsoft/test2"
const OUT = "/tmp/res"
const OCIPATH = "/tmp/oci"
const OCINAMESPACE = "ocm/value"
const OCINAMESPACE2 = "ocm/ref"
const OCIVERSION = "v2.0"
const OCIHOST = "alias"

func Check(env *TestEnv, ldesc *artdesc.Descriptor, out string) {
Expand All @@ -68,11 +65,13 @@ func CheckComponent(env *TestEnv, ldesc *artdesc.Descriptor, tgt ocm.Repository)

data, err := json.Marshal(comp.GetDescriptor().Resources[2].Access)
Expect(err).To(Succeed())
Expect(string(data)).To(Equal("{\"localReference\":\"sha256:f6a519fb1d0c8cef5e8d7811911fc7cb170462bbce19d6df067dae041250de7f\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/ref:v2.0\",\"type\":\"localBlob\"}"))
hash := HashManifest2(artefactset.DefaultArtefactSetDescriptorFileName)
Expect(string(data)).To(Equal("{\"localReference\":\"" + hash + "\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/ref:v2.0\",\"type\":\"localBlob\"}"))

data, err = json.Marshal(comp.GetDescriptor().Resources[1].Access)
Expect(err).To(Succeed())
Expect(string(data)).To(Equal("{\"localReference\":\"sha256:018520b2b249464a83e370619f544957b7936dd974468a128545eab88a0f53ed\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}"))
hash = HashManifest1(artefactset.DefaultArtefactSetDescriptorFileName)
Expect(string(data)).To(Equal("{\"localReference\":\"" + hash + "\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+tar+gzip\",\"referenceName\":\"ocm/value:v2.0\",\"type\":\"localBlob\"}"))

racc, err := comp.GetResourceByIndex(1)
Expect(err).To(Succeed())
Expand All @@ -99,29 +98,12 @@ var _ = Describe("Test Environment", func() {
_ = ldesc
BeforeEach(func() {
env = NewTestEnv()
env.OCIContext().SetAlias(OCIHOST, ctfoci.NewRepositorySpec(accessobj.ACC_READONLY, OCIPATH, accessio.PathFileSystem(env.FileSystem())))

FakeOCIRepo(env.Builder, OCIPATH, OCIHOST)

env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() {
env.Namespace(OCINAMESPACE, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
ldesc = env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "manifestlayer")
})
})
})
env.Namespace(OCINAMESPACE2, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "otherlayer")
})
})
})
ldesc = OCIManifest1(env.Builder)
OCIManifest2(env.Builder)
})

env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() {
Expand Down
29 changes: 4 additions & 25 deletions cmds/ocm/commands/ocmcmds/components/verify/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/open-component-model/ocm/cmds/ocm/testhelper"
. "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper"
. "github.com/open-component-model/ocm/pkg/contexts/ocm/signing"
. "github.com/open-component-model/ocm/pkg/testutils"

Expand All @@ -31,7 +32,6 @@ import (
"github.com/open-component-model/ocm/pkg/common/accessobj"
"github.com/open-component-model/ocm/pkg/contexts/datacontext"
"github.com/open-component-model/ocm/pkg/contexts/oci"
ctfoci "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ctf"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartefact"
"github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr"
Expand All @@ -50,9 +50,6 @@ const COMPONENTA = "github.com/mandelsoft/test"
const COMPONENTB = "github.com/mandelsoft/ref"
const OUT = "/tmp/res"
const OCIPATH = "/tmp/oci"
const OCINAMESPACE = "ocm/value"
const OCINAMESPACE2 = "ocm/ref"
const OCIVERSION = "v2.0"
const OCIHOST = "alias"

const SIGNATURE = "test"
Expand Down Expand Up @@ -82,29 +79,11 @@ var _ = Describe("access method", func() {
Expect(err).To(Succeed())
Expect(vfs.WriteFile(env.FileSystem(), PRIVKEY, data, os.ModePerm)).To(Succeed())

env.OCIContext().SetAlias(OCIHOST, ctfoci.NewRepositorySpec(accessobj.ACC_READONLY, OCIPATH, accessio.PathFileSystem(env.FileSystem())))
FakeOCIRepo(env.Builder, OCIPATH, OCIHOST)

env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() {
env.Namespace(OCINAMESPACE, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "manifestlayer")
})
})
})
env.Namespace(OCINAMESPACE2, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "otherlayer")
})
})
})
OCIManifest1(env.Builder)
OCIManifest2(env.Builder)
})

env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() {
Expand Down
30 changes: 5 additions & 25 deletions cmds/ocm/commands/ocmcmds/ctf/transfer/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/open-component-model/ocm/cmds/ocm/testhelper"
. "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper"
. "github.com/open-component-model/ocm/pkg/testutils"

"github.com/open-component-model/ocm/pkg/common/accessio"
"github.com/open-component-model/ocm/pkg/common/accessobj"
"github.com/open-component-model/ocm/pkg/contexts/oci"
"github.com/open-component-model/ocm/pkg/contexts/oci/artdesc"
ctfoci "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ctf"
"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartefact"
metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1"
ctfocm "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf"
Expand All @@ -41,9 +41,6 @@ const VERSION = "v1"
const COMPONENT = "github.com/mandelsoft/test"
const OUT = "/tmp/res"
const OCIPATH = "/tmp/oci"
const OCINAMESPACE = "ocm/value"
const OCINAMESPACE2 = "ocm/ref"
const OCIVERSION = "v2.0"
const OCIHOST = "alias"

var _ = Describe("Test Environment", func() {
Expand All @@ -53,29 +50,12 @@ var _ = Describe("Test Environment", func() {
_ = ldesc
BeforeEach(func() {
env = NewTestEnv()
env.OCIContext().SetAlias(OCIHOST, ctfoci.NewRepositorySpec(accessobj.ACC_READONLY, OCIPATH, accessio.PathFileSystem(env.FileSystem())))

FakeOCIRepo(env.Builder, OCIPATH, OCIHOST)

env.OCICommonTransport(OCIPATH, accessio.FormatDirectory, func() {
env.Namespace(OCINAMESPACE, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
ldesc = env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "manifestlayer")
})
})
})
env.Namespace(OCINAMESPACE2, func() {
env.Manifest(OCIVERSION, func() {
env.Config(func() {
env.BlobStringData(mime.MIME_JSON, "{}")
})
env.Layer(func() {
env.BlobStringData(mime.MIME_TEXT, "otherlayer")
})
})
})
OCIManifest1(env.Builder)
OCIManifest2(env.Builder)
})

env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() {
Expand Down
4 changes: 2 additions & 2 deletions cmds/ocm/testhelper/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ func NewTestEnv(opts ...env.Option) *TestEnv {
}
}

func (e *TestEnv) ApplyOption(opts *accessio.Options) {
e.Builder.ApplyOption(opts)
func (e *TestEnv) ApplyOption(opts accessio.Options) error {
return e.Builder.ApplyOption(opts)
}

func (e *TestEnv) ConfigContext() config.Context {
Expand Down
5 changes: 3 additions & 2 deletions pkg/common/accessio/format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ func (f FileFormat) String() string {
return string(f)
}

func (o FileFormat) ApplyOption(options *Options) {
func (o FileFormat) ApplyOption(options Options) error {
if o != "" {
options.FileFormat = &o
options.SetFileFormat(o)
}
return nil
}

const (
Expand Down
Loading

0 comments on commit 9f733c1

Please sign in to comment.