From d8aa7027f899ea40ecd1c5adc11b57881cbb9c43 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 23 Aug 2024 20:28:22 -0500 Subject: [PATCH] feat: Add support for OCI image manifest and index --- Cargo.toml | 2 +- src/errors.rs | 2 + src/mediatypes.rs | 14 ++++ src/v2/blobs.rs | 2 +- src/v2/config.rs | 4 + src/v2/manifest/manifest_schema2.rs | 5 +- src/v2/manifest/mod.rs | 27 ++++--- src/v2/mod.rs | 73 +++++++++++++++++-- .../api_error_fixture_with_detail.json | 16 ++++ .../api_error_fixture_without_detail.json | 8 ++ .../fixtures/manifest_oci_image_manifest.json | 16 ++++ tests/manifest.rs | 7 ++ tests/mock/base_client.rs | 45 ++++++++++++ tests/net/docker_io/mod.rs | 51 +++++++++++++ 14 files changed, 249 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/api_error_fixture_with_detail.json create mode 100644 tests/fixtures/api_error_fixture_without_detail.json create mode 100644 tests/fixtures/manifest_oci_image_manifest.json diff --git a/Cargo.toml b/Cargo.toml index 6584aee9..c8a77f0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ log = "0.4" mime = "0.3" regex-lite = "0.1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["raw_value"] } serde_ignored = "0.1" strum = { version = "0.26", features = ["derive"] } tar = "0.4" diff --git a/src/errors.rs b/src/errors.rs index 1a7d8f6d..268f14f5 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,6 +3,8 @@ #[non_exhaustive] #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("Api Error: {0}")] + Api(#[from] crate::v2::ApiErrors), #[error("base64 decode error")] Base64Decode(#[from] base64::DecodeError), #[error("header parse error")] diff --git a/src/mediatypes.rs b/src/mediatypes.rs index 9063c88d..69c4ecdd 100644 --- a/src/mediatypes.rs +++ b/src/mediatypes.rs @@ -33,6 +33,16 @@ pub enum MediaTypes { #[strum(serialize = "application/vnd.docker.container.image.v1+json")] #[strum(props(Sub = "vnd.docker.container.image.v1+json"))] ContainerConfigV1, + + /// OCI Manifest + #[strum(serialize = "application/vnd.oci.image.manifest.v1+json")] + #[strum(props(Sub = "vnd.oci.image.manifest.v1+json"))] + OciImageManifest, + // OCI Image index + #[strum(serialize = "application/vnd.oci.image.index.v1+json")] + #[strum(props(Sub = "vnd.oci.image.index.v1+json"))] + OciImageIndexV1, + /// Generic JSON #[strum(serialize = "application/json")] #[strum(props(Sub = "json"))] @@ -45,12 +55,16 @@ impl MediaTypes { match (mtype.type_(), mtype.subtype(), mtype.suffix()) { (mime::APPLICATION, mime::JSON, _) => Ok(MediaTypes::ApplicationJson), (mime::APPLICATION, subt, Some(suff)) => match (subt.to_string().as_str(), suff.to_string().as_str()) { + // Docker ("vnd.docker.distribution.manifest.v1", "json") => Ok(MediaTypes::ManifestV2S1), ("vnd.docker.distribution.manifest.v1", "prettyjws") => Ok(MediaTypes::ManifestV2S1Signed), ("vnd.docker.distribution.manifest.v2", "json") => Ok(MediaTypes::ManifestV2S2), ("vnd.docker.distribution.manifest.list.v2", "json") => Ok(MediaTypes::ManifestList), ("vnd.docker.image.rootfs.diff.tar.gzip", _) => Ok(MediaTypes::ImageLayerTgz), ("vnd.docker.container.image.v1", "json") => Ok(MediaTypes::ContainerConfigV1), + // OCI + ("vnd.oci.image.manifest.v1", "json") => Ok(MediaTypes::OciImageManifest), + ("vnd.oci.image.index.v1", "json") => Ok(MediaTypes::OciImageIndexV1), _ => Err(crate::Error::UnknownMimeType(mtype.clone())), }, _ => Err(crate::Error::UnknownMimeType(mtype.clone())), diff --git a/src/v2/blobs.rs b/src/v2/blobs.rs index 774bb556..b393d4a7 100644 --- a/src/v2/blobs.rs +++ b/src/v2/blobs.rs @@ -50,7 +50,7 @@ impl Client { } Ok(BlobResponse::new(resp, ContentDigest::try_new(digest)?)) } - Err(_) if status.is_client_error() => Err(Error::Client { status }), + Err(_) if status.is_client_error() => Err(ApiErrors::from(resp).await), Err(_) if status.is_server_error() => Err(Error::Server { status }), Err(_) => { error!("Received unexpected HTTP status '{}'", status); diff --git a/src/v2/config.rs b/src/v2/config.rs index d91be52a..d2170f8e 100644 --- a/src/v2/config.rs +++ b/src/v2/config.rs @@ -109,6 +109,8 @@ impl Config { (MediaTypes::ManifestV2S2, Some(0.5)), (MediaTypes::ManifestV2S1Signed, Some(0.4)), (MediaTypes::ManifestList, Some(0.5)), + (MediaTypes::OciImageManifest, Some(0.5)), + (MediaTypes::OciImageIndexV1, Some(0.5)), ], // GCR incorrectly parses `q` parameters, so we use special Accept for it. // Bug: https://issuetracker.google.com/issues/159827510. @@ -117,6 +119,8 @@ impl Config { (MediaTypes::ManifestV2S2, None), (MediaTypes::ManifestV2S1Signed, None), (MediaTypes::ManifestList, None), + (MediaTypes::OciImageManifest, None), + (MediaTypes::OciImageIndexV1, None), ], }, }; diff --git a/src/v2/manifest/manifest_schema2.rs b/src/v2/manifest/manifest_schema2.rs index 81602840..ae2286d0 100644 --- a/src/v2/manifest/manifest_schema2.rs +++ b/src/v2/manifest/manifest_schema2.rs @@ -2,7 +2,8 @@ use log::trace; use reqwest::Method; use serde::{Deserialize, Serialize}; -use crate::errors::{Error, Result}; +use crate::errors::Result; +pub use crate::v2::ApiErrors; /// Manifest version 2 schema 2. /// @@ -103,7 +104,7 @@ impl ManifestSchema2Spec { trace!("GET {:?}: {}", url, &status); if !status.is_success() { - return Err(Error::UnexpectedHttpStatus(status)); + return Err(ApiErrors::from(r).await); } let config_blob = r.json::().await?; diff --git a/src/v2/manifest/mod.rs b/src/v2/manifest/mod.rs index 785b7e2f..01415f87 100644 --- a/src/v2/manifest/mod.rs +++ b/src/v2/manifest/mod.rs @@ -51,7 +51,7 @@ impl Client { match status { StatusCode::OK => {} - _ => return Err(Error::UnexpectedHttpStatus(status)), + _ => return Err(ApiErrors::from(res).await), } let headers = res.headers(); @@ -73,7 +73,7 @@ impl Client { res.json::().await.map(Manifest::S1Signed)?, content_digest, )), - mediatypes::MediaTypes::ManifestV2S2 => { + mediatypes::MediaTypes::ManifestV2S2 | mediatypes::MediaTypes::OciImageManifest => { let m = res.json::().await?; Ok(( m.fetch_config_blob(client_spare0, name.to_string()) @@ -82,7 +82,9 @@ impl Client { content_digest, )) } - mediatypes::MediaTypes::ManifestList => Ok((res.json::().await.map(Manifest::ML)?, content_digest)), + mediatypes::MediaTypes::ManifestList | mediatypes::MediaTypes::OciImageIndexV1 => { + Ok((res.json::().await.map(Manifest::ML)?, content_digest)) + } unsupported => Err(Error::UnsupportedMediaType(unsupported)), } } @@ -109,7 +111,7 @@ impl Client { match status { StatusCode::OK => {} - _ => return Err(Error::UnexpectedHttpStatus(status)), + _ => return Err(ApiErrors::from(res).await), } let headers = res.headers(); @@ -169,7 +171,7 @@ impl Client { Ok(Some(media_type)) } StatusCode::NOT_FOUND => Ok(None), - _ => Err(Error::UnexpectedHttpStatus(status)), + _ => Err(ApiErrors::from(r).await), } } } @@ -326,25 +328,28 @@ mod tests { use super::*; use crate::v2::Client; - #[test_case("not-gcr.io" => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.4,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5"; "Not gcr registry")] - #[test_case("gcr.io" => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.v1+prettyjws,application/vnd.docker.distribution.manifest.list.v2+json"; "gcr.io")] - #[test_case("foobar.gcr.io" => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.v1+prettyjws,application/vnd.docker.distribution.manifest.list.v2+json"; "Custom gcr.io registry")] + #[test_case("not-gcr.io" => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.4,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5,application/vnd.oci.image.manifest.v1+json; q=0.5,application/vnd.oci.image.index.v1+json; q=0.5"; "Not gcr registry")] + #[test_case("gcr.io" => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.v1+prettyjws,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json"; "gcr.io")] + #[test_case("foobar.gcr.io" => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.v1+prettyjws,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json"; "Custom gcr.io registry")] fn gcr_io_accept_headers(registry: &str) -> String { let client_builder = Client::configure().registry(registry); let client = client_builder.build().unwrap(); let header_map = build_accept_headers(&client.accepted_types); header_map.get(header::ACCEPT).unwrap().to_str().unwrap().to_string() } - #[test_case(None => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.4,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5"; "Default settings")] + + #[test_case(None => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.4,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5,application/vnd.oci.image.manifest.v1+json; q=0.5,application/vnd.oci.image.index.v1+json; q=0.5"; "Default settings")] #[test_case(Some(vec![ (MediaTypes::ManifestV2S2, Some(0.5)), (MediaTypes::ManifestV2S1Signed, Some(0.2)), (MediaTypes::ManifestList, Some(0.5)), - ]) => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.2,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5"; "Custom accept types with weight")] + (MediaTypes::OciImageManifest, Some(0.2)), + ]) => "application/vnd.docker.distribution.manifest.v2+json; q=0.5,application/vnd.docker.distribution.manifest.v1+prettyjws; q=0.2,application/vnd.docker.distribution.manifest.list.v2+json; q=0.5,application/vnd.oci.image.manifest.v1+json; q=0.2"; "Custom accept types with weight")] #[test_case(Some(vec![ (MediaTypes::ManifestV2S2, None), (MediaTypes::ManifestList, None), - ]) => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json"; "Custom accept types, no weight")] + (MediaTypes::OciImageIndexV1, None), + ]) => "application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.index.v1+json"; "Custom accept types, no weight")] fn custom_accept_headers(accept_headers: Option)>>) -> String { let registry = "https://example.com"; diff --git a/src/v2/mod.rs b/src/v2/mod.rs index 01d7ed4e..2fd1f537 100644 --- a/src/v2/mod.rs +++ b/src/v2/mod.rs @@ -25,12 +25,17 @@ //! # } //! ``` +use std::fmt; + use futures::prelude::*; use log::trace; -use reqwest::{Method, StatusCode, Url}; +use reqwest::{Method, Response, StatusCode, Url}; use serde::{Deserialize, Serialize}; -use crate::{errors::*, mediatypes::MediaTypes}; +use crate::{ + errors::{self, *}, + mediatypes::MediaTypes, +}; mod config; pub use self::config::Config; @@ -128,13 +133,65 @@ impl Client { } #[derive(Debug, Default, Deserialize, Serialize)] -struct ApiError { +pub struct ApiError { code: String, - message: String, - detail: String, + message: Option, + detail: Option>, } -#[derive(Debug, Default, Deserialize, Serialize)] -struct Errors { - errors: Vec, +#[derive(Debug, Default, Deserialize, Serialize, thiserror::Error)] +pub struct ApiErrors { + errors: Option>, +} + +impl ApiError { + /// Return the API error code. + pub fn code(&self) -> &str { + &self.code + } + + pub fn message(&self) -> Option<&str> { + self.message.as_deref() + } +} +impl fmt::Display for ApiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "({})", self.code)?; + if let Some(message) = &self.message { + write!(f, ", message: {}", message)?; + } + if let Some(detail) = &self.detail { + write!(f, ", detail: {}", detail)?; + } + Ok(()) + } +} + +impl ApiErrors { + /// Create a new ApiErrors from a API Json response. + /// Returns an ApiError if the content is a valid per + /// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes + pub async fn from(r: Response) -> errors::Error { + match r.json::().await { + Ok(e) => errors::Error::Api(e), + Err(e) => errors::Error::Reqwest(e), + } + } + + /// Returns the errors returned by the API. + pub fn errors(&self) -> &Option> { + &self.errors + } +} + +impl fmt::Display for ApiErrors { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.errors.is_none() { + return Ok(()); + } + for error in self.errors.as_ref().unwrap().iter() { + write!(f, "({})", error)? + } + Ok(()) + } } diff --git a/tests/fixtures/api_error_fixture_with_detail.json b/tests/fixtures/api_error_fixture_with_detail.json new file mode 100644 index 00000000..84e8996e --- /dev/null +++ b/tests/fixtures/api_error_fixture_with_detail.json @@ -0,0 +1,16 @@ +{ + "errors": [ + { + "code": "UNAUTHORIZED", + "message": "authentication required", + "detail": [ + { + "Type": "repository", + "Class": "", + "Name": "some/image", + "Action": "some_action" + } + ] + } + ] +} diff --git a/tests/fixtures/api_error_fixture_without_detail.json b/tests/fixtures/api_error_fixture_without_detail.json new file mode 100644 index 00000000..0b673b9f --- /dev/null +++ b/tests/fixtures/api_error_fixture_without_detail.json @@ -0,0 +1,8 @@ +{ + "errors": [ + { + "code": "UNAUTHORIZED", + "message": "authentication required" + } + ] +} diff --git a/tests/fixtures/manifest_oci_image_manifest.json b/tests/fixtures/manifest_oci_image_manifest.json new file mode 100644 index 00000000..d6490abc --- /dev/null +++ b/tests/fixtures/manifest_oci_image_manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 2297, + "digest": "sha256:7324f32f94760ec1dc237858203ea520fc4e6dfbd0bc018f392e54b1392ac722" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 28028344, + "digest": "sha256:b2afc8f0dccbc5496c814ae03ac3fff7e86393abd18b2d2910a9c489bfe64311" + } + ] +} diff --git a/tests/manifest.rs b/tests/manifest.rs index 384b7c37..7be263a6 100644 --- a/tests/manifest.rs +++ b/tests/manifest.rs @@ -63,6 +63,13 @@ fn test_manifest_v2s2() -> Result<(), Box> { Ok(()) } +#[test] +fn test_deserialize_oci_image_manifest() { + let f = fs::File::open("tests/fixtures/manifest_oci_image_manifest.json").expect("Missing fixture"); + let bufrd = io::BufReader::new(f); + let _manif: docker_registry::v2::manifest::ManifestSchema2Spec = serde_json::from_reader(bufrd).unwrap(); +} + #[test] fn test_deserialize_manifest_list_v2() { let f = fs::File::open("tests/fixtures/manifest_list_v2.json").expect("Missing fixture"); diff --git a/tests/mock/base_client.rs b/tests/mock/base_client.rs index b8969fb0..78f0f66c 100644 --- a/tests/mock/base_client.rs +++ b/tests/mock/base_client.rs @@ -84,6 +84,51 @@ async fn test_base_custom_useragent() { assert!(res); } +/// Test that we properly deserialize API error payload and can access error contents. +#[test_case::test_case("tests/fixtures/api_error_fixture_with_detail.json".to_string() ; "API error with detail")] +#[test_case::test_case("tests/fixtures/api_error_fixture_without_detail.json".to_string() ; "API error without detail")] +fn test_base_api_error(fixture: String) { + let ua = "custom-ua/1.0"; + let image = "fake/image"; + let version = "fakeversion"; + + let mut server = mockito::Server::new(); + let addr = server.host_with_port(); + + let mock = server + .mock("GET", format!("/v2/{}/manifests/{}", image, version).as_str()) + .match_header("user-agent", ua) + .with_status(404) + .with_header(API_VERSION_K, API_VERSION_V) + .with_body_from_file(fixture) + .create(); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let dclient = docker_registry::v2::Client::configure() + .registry(&addr) + .insecure_registry(true) + .user_agent(Some(ua.to_string())) + .username(None) + .password(None) + .build() + .unwrap(); + + let futcheck = dclient.get_manifest(image, version); + + let res = runtime.block_on(futcheck); + assert!(res.is_err()); + + assert!(matches!(res, Err(docker_registry::errors::Error::Api(_)))); + if let docker_registry::errors::Error::Api(e) = res.unwrap_err() { + assert_eq!(e.errors().as_ref().unwrap()[0].code(), "UNAUTHORIZED"); + assert_eq!( + e.errors().as_ref().unwrap()[0].message().unwrap(), + "authentication required" + ); + } + mock.assert(); +} + mod test_custom_root_certificate { use std::{ error::Error, diff --git a/tests/net/docker_io/mod.rs b/tests/net/docker_io/mod.rs index f95b162c..dfa4fdd1 100644 --- a/tests/net/docker_io/mod.rs +++ b/tests/net/docker_io/mod.rs @@ -82,3 +82,54 @@ fn test_dockerio_anonymous_auth() { let res = runtime.block_on(futcheck); assert!(res.is_ok()); } + +/// Check that when requesting an image that does not exist +/// we get an Api error. +#[test] +fn test_dockerio_anonymous_non_existent_image() { + let runtime = Runtime::new().unwrap(); + let image = "bad/image"; + let version = "latest"; + let login_scope = format!("repository:{}:pull", image); + let scopes = vec![login_scope.as_str()]; + let dclient_future = docker_registry::v2::Client::configure() + .registry(REGISTRY) + .insecure_registry(false) + .username(None) + .password(None) + .build() + .unwrap() + .authenticate(scopes.as_slice()); + + let dclient = runtime.block_on(dclient_future).unwrap(); + let futcheck = dclient.get_manifest(image, version); + + let res = runtime.block_on(futcheck); + assert!(res.is_err()); + assert!(matches!(res, Err(docker_registry::errors::Error::Api(_)))); +} + +/// Test that we can deserialize OCI image manifest, as is +/// returned for s390x/ubuntu image. +#[test] +fn test_dockerio_anonymous_auth_oci_manifest() { + let runtime = Runtime::new().unwrap(); + let image = "s390x/ubuntu"; + let version = "latest"; + let login_scope = format!("repository:{}:pull", image); + let scopes = vec![login_scope.as_str()]; + let dclient_future = docker_registry::v2::Client::configure() + .registry(REGISTRY) + .insecure_registry(false) + .username(None) + .password(None) + .build() + .unwrap() + .authenticate(scopes.as_slice()); + + let dclient = runtime.block_on(dclient_future).unwrap(); + let futcheck = dclient.get_manifest(image, version); + + let res = runtime.block_on(futcheck); + assert!(res.is_ok()); +}