diff --git a/cmd/otk-resolve-containers/export_test.go b/cmd/otk-resolve-containers/export_test.go new file mode 100644 index 0000000000..77c5f1a937 --- /dev/null +++ b/cmd/otk-resolve-containers/export_test.go @@ -0,0 +1,5 @@ +package main + +var ( + Run = run +) diff --git a/cmd/otk-resolve-containers/main.go b/cmd/otk-resolve-containers/main.go new file mode 100644 index 0000000000..0e4e0b9cef --- /dev/null +++ b/cmd/otk-resolve-containers/main.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" +) + +// All otk external inputs are nested under a top-level "tree" +type Tree struct { + Tree Input `json:"tree"` +} + +// Input represents the user-provided inputs that will be used to resolve a +// container image. +type Input struct { + // The architecture of the container images to resolve. + Arch string `json:"arch"` + + // List of container refs to resolve. + Containers []blueprint.Container `json:"containers"` +} + +// Output contains everything needed to write a manifest that requires pulling +// container images. +type Output struct { + Const OutputConst `json:"const"` +} + +type OutputConst struct { + Containers []ContainerInfo `json:"containers"` +} + +type ContainerInfo struct { + Source string `json:"source"` + + // Digest of the manifest at the source. + Digest string `json:"digest"` + + // Container image identifier. + ImageID string `json:"imageid"` + + // Name to use inside the image. + LocalName string `json:"local-name"` + + // Digest of the list manifest at the source + ListDigest string `json:"list-digest,omitempty"` + + // The architecture of the image + Arch string `json:"arch"` + + TLSVerify *bool `json:"tls-verify,omitempty"` +} + +func run(r io.Reader, w io.Writer) error { + var inputTree Tree + if err := json.NewDecoder(r).Decode(&inputTree); err != nil { + return err + } + + resolver := container.NewResolver(inputTree.Tree.Arch) + + for _, bpSpec := range inputTree.Tree.Containers { + srcSpec := container.SourceSpec{ + Source: bpSpec.Source, + Name: bpSpec.Name, + TLSVerify: bpSpec.TLSVerify, + } + resolver.Add(srcSpec) + } + + containerSpecs, err := resolver.Finish() + if err != nil { + return err + } + containerInfos := make([]ContainerInfo, len(containerSpecs)) + for idx := range containerSpecs { + spec := containerSpecs[idx] + containerInfos[idx] = ContainerInfo{ + Source: spec.Source, + Digest: spec.Digest, + ImageID: spec.ImageID, + LocalName: spec.LocalName, + ListDigest: spec.ListDigest, + Arch: spec.Arch.String(), + TLSVerify: spec.TLSVerify, + } + } + + output := map[string]Output{ + "tree": { + Const: OutputConst{ + Containers: containerInfos, + }, + }, + } + outputJson, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("cannot marshal response: %w", err) + } + fmt.Fprintf(w, "%s\n", outputJson) + return nil +} + +func main() { + if err := run(os.Stdin, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err.Error()) + os.Exit(1) + } +} diff --git a/cmd/otk-resolve-containers/main_test.go b/cmd/otk-resolve-containers/main_test.go new file mode 100644 index 0000000000..97b8044e74 --- /dev/null +++ b/cmd/otk-resolve-containers/main_test.go @@ -0,0 +1,251 @@ +package main_test + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/osbuild/images/internal/common" + "github.com/osbuild/images/internal/testregistry" + "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/blueprint" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + resolver "github.com/osbuild/images/cmd/otk-resolve-containers" +) + +const ( + owner = "osbuild" + reponame = "testcontainer" +) + +func createTestRegistry() (*testregistry.Registry, []string) { + registry := testregistry.New() + repo := registry.AddRepo(fmt.Sprintf("%s/%s", owner, reponame)) + ref := registry.GetRef(fmt.Sprintf("%s/%s", owner, reponame)) + + // add 10 images, all in the same repository with the same content + // (rootLayer), but each with a different tag and comment + refs := make([]string, 10) + for idx := 0; idx < len(refs); idx++ { + checksum := repo.AddImage( + []testregistry.Blob{testregistry.NewDataBlobFromBase64(testregistry.RootLayer)}, + []string{"amd64", "ppc64le"}, + fmt.Sprintf("image %d", idx), + time.Time{}) + + tag := fmt.Sprintf("tag-%d", idx) + repo.AddTag(checksum, tag) + refs[idx] = fmt.Sprintf("%s:%s", ref, tag) + } + return registry, refs +} + +func TestResolver(t *testing.T) { + registry, refs := createTestRegistry() + defer registry.Close() + + inpContainers := make([]blueprint.Container, len(refs)) + for idx, ref := range refs { + inpContainers[idx] = blueprint.Container{ + Source: ref, + Name: fmt.Sprintf("test/localhost/%s", ref), // add a prefix for the local name to override the source + TLSVerify: common.ToPtr(false), + LocalStorage: false, + } + } + + for _, containerArch := range []string{"amd64", "ppc64le"} { + t.Run(containerArch, func(t *testing.T) { + + require := require.New(t) + assert := assert.New(t) + + input := map[string]interface{}{ + "arch": containerArch, + "containers": inpContainers, + } + inputReq, err := json.Marshal(map[string]map[string]interface{}{ + "tree": input, + }) + require.NoError(err) + + inpBuf := bytes.NewBuffer(inputReq) + outBuf := &bytes.Buffer{} + + assert.NoError(resolver.Run(inpBuf, outBuf)) + + var output map[string]resolver.Output + require.NoError(json.Unmarshal(outBuf.Bytes(), &output)) + + outputContainers := output["tree"].Const.Containers + + assert.Len(outputContainers, len(refs)) + + expectedOutput := make([]resolver.ContainerInfo, len(refs)) + for idx, ref := range refs { + // resolve directly with the registry and convert to ContainerInfo to + // compare with output. + spec, err := registry.Resolve(ref, arch.FromString(containerArch)) + assert.NoError(err) + expectedOutput[idx] = resolver.ContainerInfo{ + Source: spec.Source, + Digest: spec.Digest, + ImageID: spec.ImageID, + // registry.Resolve() copies the ref to the local name but the + // resolver will add the user-defined local name instead + LocalName: fmt.Sprintf("test/localhost/%s", ref), + ListDigest: spec.ListDigest, + Arch: spec.Arch.String(), + TLSVerify: spec.TLSVerify, + } + } + + // NOTE: the order of containers in the resolver's output is stable but is + // not the same as the order of the inputs. + assert.ElementsMatch(outputContainers, expectedOutput) + }) + } +} + +func TestResolverUnhappy(t *testing.T) { + registry, refs := createTestRegistry() + defer registry.Close() + + type testCase struct { + source string + arch string + tlsverify bool + errSubstr string + } + + testCases := map[string]testCase{ + "bad-registry": { + source: "127.0.0.2:1990/org/repo:tag", + arch: "amd64", + errSubstr: "127.0.0.2:1990: connect: connection refused", + }, + "bad-repo": { + // modify the container path of an existing ref + source: strings.Replace(refs[0], owner, "notosbuild", 1), + arch: "amd64", + errSubstr: fmt.Sprintf("notosbuild/%s: StatusCode: 404", reponame), + }, + "bad-repo-containername": { + // modify the container path of an existing ref + source: strings.Replace(refs[0], reponame, "container-does-not-exist", 1), + arch: "amd64", + errSubstr: fmt.Sprintf("%s/container-does-not-exist: StatusCode: 404", owner), + }, + "bad-tag": { + // modify the tag of an existing ref + source: strings.Replace(refs[0], "tag", "not-a-tag", 1), + arch: "amd64", + errSubstr: "error getting manifest: reading manifest not-a-tag-0", + }, + "bad-arch": { + source: refs[0], + arch: "s390x", + errSubstr: "no image found in manifest list for architecture \"s390x\"", + }, + "tls-fail": { + source: refs[0], + arch: "amd64", + tlsverify: true, + errSubstr: "failed to verify certificate: x509: certificate signed by unknown authority", + }, + } + + for name := range testCases { + tc := testCases[name] + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + input := map[string]interface{}{ + "arch": tc.arch, + "containers": []blueprint.Container{ + { + Source: tc.source, + TLSVerify: &tc.tlsverify, + }, + }, + } + inputReq, err := json.Marshal(map[string]map[string]interface{}{ + "tree": input, + }) + require.NoError(err) + inpBuf := bytes.NewBuffer(inputReq) + outBuf := &bytes.Buffer{} + assert.ErrorContains(resolver.Run(inpBuf, outBuf), tc.errSubstr) + }) + } +} + +func TestResolverRawJSON(t *testing.T) { + require := require.New(t) + + registry, refs := createTestRegistry() + defer registry.Close() + + inpContainers := make([]blueprint.Container, len(refs)) + for idx, ref := range refs { + inpContainers[idx] = blueprint.Container{ + Source: ref, + Name: fmt.Sprintf("test/localhost/%s", ref), // add a prefix for the local name to override the source + TLSVerify: common.ToPtr(false), + LocalStorage: false, + } + } + + for _, containerArch := range []string{"amd64", "ppc64le"} { + t.Run(containerArch, func(t *testing.T) { + cntName := fmt.Sprintf("test/localhost/%s", refs[0]) + inpBuf := bytes.NewBufferString(fmt.Sprintf(`{ +"tree":{ + "arch":"%s", + "containers":[ + { + "source":"%s", + "name":"%s", + "tls-verify":false + } + ] + } +}`, containerArch, refs[0], cntName)) + outBuf := &bytes.Buffer{} + + err := resolver.Run(inpBuf, outBuf) + require.NoError(err) + + // resolve directly with the registry and convert to a raw JSON + // string to compare with output + spec, err := registry.Resolve(refs[0], arch.FromString(containerArch)) + require.NoError(err) + + expected := fmt.Sprintf(`{ + "tree": { + "const": { + "containers": [ + { + "source": "%s", + "digest": "%s", + "imageid": "%s", + "local-name": "%s", + "list-digest": "%s", + "arch": "%s", + "tls-verify": %v + } + ] + } + } +} +`, spec.Source, spec.Digest, spec.ImageID, fmt.Sprintf("test/localhost/%s", refs[0]), spec.ListDigest, spec.Arch, *spec.TLSVerify) + require.Equal(expected, outBuf.String()) + }) + } +} diff --git a/pkg/container/container_test.go b/internal/testregistry/testregistry.go similarity index 93% rename from pkg/container/container_test.go rename to internal/testregistry/testregistry.go index cda3752c76..e1ef065934 100644 --- a/pkg/container/container_test.go +++ b/internal/testregistry/testregistry.go @@ -1,4 +1,4 @@ -package container_test +package testregistry import ( "bytes" @@ -22,7 +22,17 @@ import ( "github.com/osbuild/images/pkg/container" ) -const rootLayer = `H4sIAAAJbogA/+SWUYqDMBCG53lP4V5g9x8dzRX2Bvtc0VIhEIhKe/wSKxgU6ktjC/O9hMzAQDL8 +/* +A base64 encoded gzipped tarball with the following contents: + + -rw-r--r-- root/root 12 2021-09-17 12:32 hello.txt (Contents: "Hello World") + drwxr-xr-x root/root 0 1970-01-01 01:00 subdir/ + -rw-r--r-- root/root 8 2021-09-17 12:32 subdir/file.txt (Contents: "osbuild") + -rw-r--r-- root/root 7 2021-09-17 12:32 world.txt (Contents: "hello!") + +Can be used with [NewDataBlobFromBase64] to create a data blob for [Repo.AddImage]. +*/ +const RootLayer = `H4sIAAAJbogA/+SWUYqDMBCG53lP4V5g9x8dzRX2Bvtc0VIhEIhKe/wSKxgU6ktjC/O9hMzAQDL8 /8yltdb9DLeB0gEGKhHCg/UJsBAL54zKFBAC54ZzyrCUSMfYDydPgHfu6R/s5VePilOfzF/of/bv vG2+lqhyFNGPddP53yjyegCBKcuNROZ77AmBoP+CmbIyqpEM5fqf+3/ubJtsCuz7P1b+L1Du/4f5 v+vrsVPu/Vq9P3ANk//d+x/MZv8TKNf/Qfqf9v9v5fLXK3/lKEc5ypm4AwAA//8DAE6E6nIAEgAA @@ -289,7 +299,7 @@ func (reg *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } -func NewTestRegistry() *Registry { +func New() *Registry { reg := &Registry{ repos: make(map[string]*Repo), diff --git a/pkg/container/client_test.go b/pkg/container/client_test.go index f31c55f4d2..0db80f3c4f 100644 --- a/pkg/container/client_test.go +++ b/pkg/container/client_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/osbuild/images/internal/testregistry" "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/container" ) @@ -16,12 +17,12 @@ import ( func TestClientResolve(t *testing.T) { - registry := NewTestRegistry() + registry := testregistry.New() defer registry.Close() repo := registry.AddRepo("library/osbuild") listDigest := repo.AddImage( - []Blob{NewDataBlobFromBase64(rootLayer)}, + []testregistry.Blob{testregistry.NewDataBlobFromBase64(testregistry.RootLayer)}, []string{"amd64", "ppc64le"}, "cool container", time.Time{}) diff --git a/pkg/container/resolver_test.go b/pkg/container/resolver_test.go index 0f24b5a14d..6f1fd2fecd 100644 --- a/pkg/container/resolver_test.go +++ b/pkg/container/resolver_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/osbuild/images/internal/common" + "github.com/osbuild/images/internal/testregistry" "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/container" ) @@ -24,7 +25,7 @@ var forceLocal = flag.Bool( func TestResolver(t *testing.T) { - registry := NewTestRegistry() + registry := testregistry.New() defer registry.Close() repo := registry.AddRepo("library/osbuild") ref := registry.GetRef("library/osbuild") @@ -32,7 +33,7 @@ func TestResolver(t *testing.T) { refs := make([]string, 10) for i := 0; i < len(refs); i++ { checksum := repo.AddImage( - []Blob{NewDataBlobFromBase64(rootLayer)}, + []testregistry.Blob{testregistry.NewDataBlobFromBase64(testregistry.RootLayer)}, []string{"amd64", "ppc64le"}, fmt.Sprintf("image %d", i), time.Time{}) @@ -84,7 +85,7 @@ func TestResolverFail(t *testing.T) { assert.Error(t, err) assert.Len(t, specs, 0) - registry := NewTestRegistry() + registry := testregistry.New() defer registry.Close() resolver.Add(container.SourceSpec{