From 8d9348b3248067b5f79a7180df63c3f5b25a38e1 Mon Sep 17 00:00:00 2001 From: Tobias Furuholm Date: Mon, 26 Jun 2017 09:08:04 +0200 Subject: [PATCH 1/3] Add support for Docker Image Manifest V 2, Schema 2 --- docker/docker.go | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 711a300..315eeb1 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -38,6 +38,26 @@ type FsLayer struct { BlobSum string } +// ImageV1 represents a Manifest V 2, Schema 1 Docker Image +type imageV1 struct { + FsLayers []fsLayer +} + +// FsLayer represents a layer in a Manifest V 2, Schema 1 Docker Image +type fsLayer struct { + BlobSum string +} + +// imageV2 represents Manifest V 2, Schema 2 Docker Image +type imageV2 struct { + Layers []layer +} + +// Layer represents a layer in a Manifest V 2, Schema 2 Docker Image +type layer struct { + Digest string +} + const dockerHub = "registry-1.docker.io" var tokenRe = regexp.MustCompile(`Bearer realm="(.*?)",service="(.*?)",scope="(.*?)"`) @@ -150,9 +170,27 @@ func (i *Image) Pull() error { } } defer resp.Body.Close() - if err = json.NewDecoder(resp.Body).Decode(i); err != nil { - fmt.Fprintln(os.Stderr, "Decode error") - return err + contentType := resp.Header.Get("Content-Type") + if contentType == "application/vnd.docker.distribution.manifest.v2+json" { + var imageV2 imageV2 + if err = json.NewDecoder(resp.Body).Decode(&imageV2); err != nil { + fmt.Fprintln(os.Stderr, "Image V2 decode error") + return err + } + i.FsLayers = make([]FsLayer, len(imageV2.Layers)) + for idx := range imageV2.Layers { + i.FsLayers[idx].BlobSum = imageV2.Layers[idx].Digest + } + } else { + var imageV1 imageV1 + if err = json.NewDecoder(resp.Body).Decode(&imageV1); err != nil { + fmt.Fprintln(os.Stderr, "ImageV1 decode error") + return err + } + i.FsLayers = make([]FsLayer, len(imageV1.FsLayers)) + for idx := range imageV1.FsLayers { + i.FsLayers[idx].BlobSum = imageV1.FsLayers[idx].BlobSum + } } return nil } @@ -210,8 +248,8 @@ func (i *Image) pullReq() (*http.Response, error) { req.Header.Set("Authorization", i.Token) } - req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") - req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.list.v2+json") + // Prefer v2 manifests + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json") resp, err := i.client.Do(req) if err != nil { From c88dd8c7a088e3cd2afaf35b8ad0aeb9833ae7e9 Mon Sep 17 00:00:00 2001 From: Tobias Furuholm Date: Fri, 30 Jun 2017 09:57:01 +0200 Subject: [PATCH 2/3] Add unit test for pulling docker image manifest v2, schema v2 --- docker/docker.go | 2 +- docker/docker_test.go | 24 ++++- .../testdata/registry-response-schemav2.json | 91 +++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 docker/testdata/registry-response-schemav2.json diff --git a/docker/docker.go b/docker/docker.go index 315eeb1..af344c3 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -248,7 +248,7 @@ func (i *Image) pullReq() (*http.Response, error) { req.Header.Set("Authorization", i.Token) } - // Prefer v2 manifests + // Prefer manifest schema v2 req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json") resp, err := i.client.Do(req) diff --git a/docker/docker_test.go b/docker/docker_test.go index ddcb454..73471d0 100644 --- a/docker/docker_test.go +++ b/docker/docker_test.go @@ -83,7 +83,7 @@ func TestNewImage(t *testing.T) { } -func TestPull(t *testing.T) { +func TestPullManifestSchemaV1(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { resp, err := ioutil.ReadFile("testdata/registry-response.json") if err != nil { @@ -103,3 +103,25 @@ func TestPull(t *testing.T) { t.Fatal("Can't pull fsLayers") } } + +func TestPullManifestSchemaV2(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp, err := ioutil.ReadFile("testdata/registry-response-schemav2.json") + w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json") + if err != nil { + t.Fatalf("Can't load registry test response %s", err.Error()) + } + fmt.Fprintln(w, string(resp)) + })) + defer ts.Close() + + image, err := NewImage("docker-registry.domain.com/nginx:1b29e1531c", "", "", false, false) + image.Registry = ts.URL + err = image.Pull() + if err != nil { + t.Fatalf("Can't pull image: %s", err) + } + if len(image.FsLayers) == 0 { + t.Fatal("Can't pull fsLayers") + } +} diff --git a/docker/testdata/registry-response-schemav2.json b/docker/testdata/registry-response-schemav2.json new file mode 100644 index 0000000..ecb63a3 --- /dev/null +++ b/docker/testdata/registry-response-schemav2.json @@ -0,0 +1,91 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 9160, + "digest": "sha256:b0025729d38162597006aa4f82aae09b7c5a1e7240790c0ebb1afe89f51f0aa9" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 52614808, + "digest": "sha256:9f0706ba7422412cd468804fee456786f88bed94bf9aea6dde2a47f770d19d27" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 19264368, + "digest": "sha256:d3942a742d221ef22a0a335c4eebf09e15a36dcfb224b5a2d0cdcc405f374ccb" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 568520, + "digest": "sha256:2b95a7bc6bf9459b773705f47f4d76c50337997a71fffb22f775ad8906f5c8d0" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 215, + "digest": "sha256:7bd307c6c6e7953d9349b4808b361ff7468f0f325b2a662cd48305c7d66b2034" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 240, + "digest": "sha256:ba7da8b0113502d19a00da6d1271fcfda426cd53dcd9698e9ca3e8f06857168f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 130, + "digest": "sha256:74169d04cf0da1c003a94b9ee2fb68acd4fa186f465acb53e28352a8787132dc" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 54074870, + "digest": "sha256:08cc0e2943324a0640f6990e4f0789b5d3946b133331c28ba52f3809e5c289f9" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 289620, + "digest": "sha256:d2f5746bc4d33fea165d7c9288d8f8ec9ccb341897c0100f321f869f17aebaa0" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1371083, + "digest": "sha256:50e1b94735ab86baec2dc8220afab29a999931e50dc51cee6ba348a7b0dbfffd" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 192, + "digest": "sha256:5d412b734a0bfd5fd9f6e1afdc62fc3c7e21a1c24d6180d497ad150d109b74ab" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 500660, + "digest": "sha256:43be7cdfb670e616080dda90e92b6225c235899487856f177e075362a01d55b8" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1450, + "digest": "sha256:ad33767a537ebc823449469e7493e7a2a363f2f38729f3a3c92062c5c9bfedb0" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 225, + "digest": "sha256:7604efdb01e4f9fb23501a61da6ca7d1fed5e30a5c323954903a8d5dfd318df7" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 95908290, + "digest": "sha256:30f317eb0b898602a92269c92b9da36852d4cce44fd3c58f1c2c09bfc9907ed8" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 2229, + "digest": "sha256:6c03d301bfca4ae6b600306c533dd9ff952f56fdbcaf3bd0d4378739fbbe4428" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 304, + "digest": "sha256:1c5d5548a00e4f6a7e592df79cece5ed86ee00e8617bf6b822a0b457f8b55355" + } + ] +} From 26bd4eadad7baaca38bd383bf4650360d44e1454 Mon Sep 17 00:00:00 2001 From: Tobias Furuholm Date: Fri, 30 Jun 2017 10:11:36 +0200 Subject: [PATCH 3/3] Refactor docker layer digest/blobsum extraction --- docker/docker.go | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index af344c3..7eb1d4f 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -170,29 +170,41 @@ func (i *Image) Pull() error { } } defer resp.Body.Close() + digests, err := extractLayerDigests(resp) + if err != nil { + return err + } + i.FsLayers = make([]FsLayer, len(digests)) + for idx := range digests { + i.FsLayers[idx].BlobSum = digests[idx] + } + return err +} + +func extractLayerDigests(resp *http.Response) (digests []string, err error) { contentType := resp.Header.Get("Content-Type") if contentType == "application/vnd.docker.distribution.manifest.v2+json" { var imageV2 imageV2 if err = json.NewDecoder(resp.Body).Decode(&imageV2); err != nil { fmt.Fprintln(os.Stderr, "Image V2 decode error") - return err + return nil, err } - i.FsLayers = make([]FsLayer, len(imageV2.Layers)) - for idx := range imageV2.Layers { - i.FsLayers[idx].BlobSum = imageV2.Layers[idx].Digest + digests = make([]string, len(imageV2.Layers)) + for i := range imageV2.Layers { + digests[i] = imageV2.Layers[i].Digest } } else { var imageV1 imageV1 if err = json.NewDecoder(resp.Body).Decode(&imageV1); err != nil { fmt.Fprintln(os.Stderr, "ImageV1 decode error") - return err + return nil, err } - i.FsLayers = make([]FsLayer, len(imageV1.FsLayers)) - for idx := range imageV1.FsLayers { - i.FsLayers[idx].BlobSum = imageV1.FsLayers[idx].BlobSum + digests = make([]string, len(imageV1.FsLayers)) + for i := range imageV1.FsLayers { + digests[i] = imageV1.FsLayers[i].BlobSum } } - return nil + return digests, nil } func (i *Image) requestToken(resp *http.Response) (string, error) {