Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow deserialized WAPM JSON even if some version have some NULL infos #3946

Merged
merged 3 commits into from
Jun 2, 2023
Merged

Allow deserialized WAPM JSON even if some version have some NULL infos #3946

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7918d58
Allow deserialized WAPM JSON even if some version have some NULL Mani…
ptitSeb Jun 2, 2023
0688791
Adding a test for missing response fields
Jun 2, 2023
690ee5e
Handle missing fields more gracefully
Jun 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 144 additions & 37 deletions lib/wasi/src/runtime/resolver/wapm_source.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ impl Source for WapmSource {
PackageSpecifier::Registry { full_name, version } => (full_name, version),
_ => return Ok(Vec::new()),
};

#[derive(serde::Serialize)]
struct Body {
query: String,
Expand All @@ -51,6 +50,7 @@ impl Source for WapmSource {
query: WAPM_WEBC_QUERY_ALL.replace("$NAME", full_name),
};
let body = serde_json::to_string(&body)?;
tracing::trace!(%body, "Sending GraphQL query");

let request = HttpRequest {
url: self.registry_endpoint.clone(),
Expand All @@ -69,6 +69,11 @@ impl Source for WapmSource {
}

let body = response.body.unwrap_or_default();
tracing::trace!(
body=?String::from_utf8_lossy(&body),
"Received a response from GraphQL",
);

let response: WapmWebQuery =
serde_json::from_slice(&body).context("Unable to deserialize the response")?;

Expand All @@ -80,10 +85,20 @@ impl Source for WapmSource {
};

for pkg_version in versions {
tracing::trace!(?pkg_version, "checking package version");

let version = Version::parse(&pkg_version.version)?;
if version_constraint.matches(&version) {
let summary = decode_summary(pkg_version)?;
summaries.push(summary);
match decode_summary(pkg_version) {
Ok(summary) => summaries.push(summary),
Err(e) => {
tracing::debug!(
version=%version,
error=&*e,
"Skipping version because its metadata couldn't be parsed"
);
}
}
}
}

Expand All @@ -109,17 +124,21 @@ fn decode_summary(pkg_version: WapmWebQueryGetPackageVersion) -> Result<PackageS
..
} = pkg_version;

let manifest = manifest.context("missing Manifest")?;
let hash = pirita_sha256_hash.context("missing sha256")?;
let url = pirita_download_url.context("missing download url")?;

let manifest: Manifest = serde_json::from_slice(manifest.as_bytes())
.context("Unable to deserialize the manifest")?;

let mut webc_sha256 = [0_u8; 32];
hex::decode_to_slice(&pirita_sha256_hash, &mut webc_sha256)?;
hex::decode_to_slice(&hash, &mut webc_sha256)?;
let webc_sha256 = WebcHash::from_bytes(webc_sha256);

Ok(PackageSummary {
pkg: PackageInfo::from_manifest(&manifest)?,
dist: DistributionInfo {
webc: pirita_download_url.parse()?,
webc: url.parse().context("Unable to parse the download URL")?,
webc_sha256,
},
})
Expand Down Expand Up @@ -161,20 +180,22 @@ pub struct WapmWebQueryGetPackageVersion {
pub version: String,
/// A JSON string containing a [`Manifest`] definition.
#[serde(rename = "piritaManifest")]
pub manifest: String,
pub manifest: Option<String>,
pub distribution: WapmWebQueryGetPackageVersionDistribution,
}

#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct WapmWebQueryGetPackageVersionDistribution {
#[serde(rename = "piritaDownloadUrl")]
pub pirita_download_url: String,
pub pirita_download_url: Option<String>,
#[serde(rename = "piritaSha256Hash")]
pub pirita_sha256_hash: String,
pub pirita_sha256_hash: Option<String>,
}

#[cfg(test)]
mod tests {
use std::sync::Mutex;

use http::{HeaderMap, StatusCode};

use crate::{
Expand All @@ -184,54 +205,59 @@ mod tests {

use super::*;

// You can check the response with:
// curl https://registry.wapm.io/graphql \
// -H "Content-Type: application/json" \
// -X POST \
// -d '@wasmer_pack_cli_request.json' > wasmer_pack_cli_response.json
const WASMER_PACK_CLI_REQUEST: &[u8] = br#"{"query": "{\n getPackage(name: \"wasmer/wasmer-pack-cli\") {\n versions {\n version\n piritaManifest\n distribution {\n piritaDownloadUrl\n piritaSha256Hash\n }\n }\n }\n}"}"#;
const WASMER_PACK_CLI_RESPONSE: &[u8] = br#"{"data":{"getPackage":{"versions":[{"version":"0.7.0","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:FesCIAS6URjrIAAyy4G5u5HjJjGQBLGmnafjHPHRvqo=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"/home/consulting/Documents/wasmer/wasmer-pack/crates/cli/../../README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.7.0\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.7.0-0e384e88-ab70-11ed-b0ed-b22ba48456e7.webc","piritaSha256Hash":"d085869201aa602673f70abbd5e14e5a6936216fa93314c5b103cda3da56e29e"}},{"version":"0.6.0","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:CzzhNaav3gjBkCJECGbk7e+qAKurWbcIAzQvEqsr2Co=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"/home/consulting/Documents/wasmer/wasmer-pack/crates/cli/../../README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.6.0\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.6.0-654a2ed8-875f-11ed-90e2-c6aeb50490de.webc","piritaSha256Hash":"7e1add1640d0037ff6a726cd7e14ea36159ec2db8cb6debd0e42fa2739bea52b"}},{"version":"0.5.3","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:qdiJVfpi4icJXdR7Y5US/pJ4PjqbAq9PkU+obMZIMlE=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"/home/runner/work/wasmer-pack/wasmer-pack/crates/cli/../../README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.5.3\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.5.3-4a2b9764-728c-11ed-9fe4-86bf77232c64.webc","piritaSha256Hash":"44fdcdde23d34175887243d7c375e4e4a7e6e2cd1ae063ebffbede4d1f68f14a"}},{"version":"0.5.2","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:xiwrUFAo+cU1xW/IE6MVseiyjNGHtXooRlkYKiOKzQc=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"/home/consulting/Documents/wasmer/wasmer-pack/crates/cli/../../README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.5.2\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.5.2.webc","piritaSha256Hash":"d1dbc8168c3a2491a7158017a9c88df9e0c15bed88ebcd6d9d756e4b03adde95"}},{"version":"0.5.1","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:TliPwutfkFvRite/3/k3OpLqvV0EBKGwyp3L5UjCuEI=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"/home/runner/work/wasmer-pack/wasmer-pack/crates/cli/../../README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.5.1\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.5.1.webc","piritaSha256Hash":"c42924619660e2befd69b5c72729388985dcdcbf912d51a00015237fec3e1ade"}},{"version":"0.5.0","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:6UD7NS4KtyNYa3TcnKOvd+kd3LxBCw+JQ8UWRpMXeC0=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.5.0\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.5.0.webc","piritaSha256Hash":"d30ca468372faa96469163d2d1546dd34be9505c680677e6ab86a528a268e5f5"}},{"version":"0.5.0-rc.1","piritaManifest":"{\"atoms\": {\"wasmer-pack\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:ThybHIc2elJEcDdQiq5ffT1TVaNs70+WAqoKw4Tkh3E=\"}}, \"package\": {\"wapm\": {\"name\": \"wasmer/wasmer-pack-cli\", \"readme\": {\"path\": \"README.md\", \"volume\": \"metadata\"}, \"license\": \"MIT\", \"version\": \"0.5.0-rc.1\", \"homepage\": \"https://wasmer.io/\", \"repository\": \"https://github.com/wasmerio/wasmer-pack\", \"description\": \"A code generator that lets you treat WebAssembly modules like native dependencies.\"}}, \"commands\": {\"wasmer-pack\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"wasmer-pack\", \"package\": \"wasmer/wasmer-pack-cli\", \"main_args\": null}}}}, \"entrypoint\": \"wasmer-pack\"}","distribution":{"piritaDownloadUrl":"https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.5.0-rc.1.webc","piritaSha256Hash":"0cd5d6e4c33c92c52784afed3a60c056953104d719717948d4663ff2521fe2bb"}}]}}}"#;

#[derive(Debug, Default)]
struct DummyClient;
#[derive(Debug)]
struct DummyClient {
requests: Mutex<Vec<HttpRequest>>,
responses: Mutex<Vec<HttpResponse>>,
}

impl DummyClient {
fn new(responses: Vec<HttpResponse>) -> Self {
DummyClient {
requests: Mutex::new(Vec::new()),
responses: Mutex::new(responses),
}
}

fn take_requests(&self) -> Vec<HttpRequest> {
std::mem::take(&mut *self.requests.lock().unwrap())
}
}

impl HttpClient for DummyClient {
fn request(
&self,
request: HttpRequest,
) -> futures::future::BoxFuture<'_, Result<HttpResponse, anyhow::Error>> {
// You can check the response with:
// curl https://registry.wapm.io/graphql \
// -H "Content-Type: application/json" \
// -X POST \
// -d '@wasmer_pack_cli_request.json' > wasmer_pack_cli_response.json
assert_eq!(request.method, http::Method::POST);
assert_eq!(request.url.as_str(), WapmSource::WAPM_PROD_ENDPOINT);
assert_eq!(request.headers.len(), 2);
assert_eq!(request.headers["User-Agent"], USER_AGENT);
assert_eq!(request.headers["Content-Type"], "application/json");

let body: serde_json::Value =
serde_json::from_slice(request.body.as_deref().unwrap()).unwrap();
let expected_body: serde_json::Value =
serde_json::from_slice(WASMER_PACK_CLI_REQUEST).unwrap();
assert_eq!(body, expected_body);

Box::pin(async {
Ok(HttpResponse {
body: Some(WASMER_PACK_CLI_RESPONSE.to_vec()),
redirected: false,
status: StatusCode::OK,
headers: HeaderMap::new(),
})
})
self.requests.lock().unwrap().push(request);
let response = self.responses.lock().unwrap().remove(0);
Box::pin(async { Ok(response) })
}
}

#[tokio::test]
async fn run_known_query() {
let client = Arc::new(DummyClient::default());
let response = HttpResponse {
body: Some(WASMER_PACK_CLI_RESPONSE.to_vec()),
redirected: false,
status: StatusCode::OK,
headers: HeaderMap::new(),
};
let client = Arc::new(DummyClient::new(vec![response]));
let registry_endpoint = WapmSource::WAPM_PROD_ENDPOINT.parse().unwrap();
let request = PackageSpecifier::Registry {
full_name: "wasmer/wasmer-pack-cli".to_string(),
version: "^0.6".parse().unwrap(),
};
let source = WapmSource::new(registry_endpoint, client);
let source = WapmSource::new(registry_endpoint, client.clone());

let summaries = source.query(&request).await.unwrap();

Expand Down Expand Up @@ -288,5 +314,86 @@ mod tests {
}
}]
);
let requests = client.take_requests();
assert_eq!(requests.len(), 1);
let request = &requests[0];
assert_eq!(request.method, http::Method::POST);
assert_eq!(request.url.as_str(), WapmSource::WAPM_PROD_ENDPOINT);
assert_eq!(request.headers.len(), 2);
assert_eq!(request.headers["User-Agent"], USER_AGENT);
assert_eq!(request.headers["Content-Type"], "application/json");
let body: serde_json::Value =
serde_json::from_slice(request.body.as_deref().unwrap()).unwrap();
let expected_body: serde_json::Value =
serde_json::from_slice(WASMER_PACK_CLI_REQUEST).unwrap();
assert_eq!(body, expected_body);
}

/// For the full context, see #3946 on GitHub or the original conversation
/// [on
/// Slack](https://wasmerio.slack.com/archives/C03MX4KL6KH/p1685706988500919).
#[tokio::test]
async fn skip_package_versions_with_missing_fields() {
let body = serde_json::json! {
{
"data": {
"getPackage": {
"versions": [
{
"version": "0.2.0",
"piritaManifest": "{\"atoms\": {\"cowsay\": {\"kind\": \"https://webc.org/kind/wasm\", \"signature\": \"sha256:DPmhiSNXCg5261eTUi3BIvAc/aJttGj+nD+bGhQkVQo=\"}}, \"package\": {\"wapm\": {\"name\": \"cowsay\", \"readme\": {\"path\": \"README.md\", \"volume\": \"metadata\"}, \"version\": \"0.2.0\", \"repository\": \"https://github.com/wapm-packages/cowsay\", \"description\": \"cowsay is a program that generates ASCII pictures of a cow with a message\"}}, \"commands\": {\"cowsay\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"cowsay\", \"package\": null, \"main_args\": null}}}, \"cowthink\": {\"runner\": \"https://webc.org/runner/wasi/command@unstable_\", \"annotations\": {\"wasi\": {\"atom\": \"cowsay\", \"package\": null, \"main_args\": null}}}}}",
"distribution": {
"piritaDownloadUrl": "https://storage.googleapis.com/wapm-registry-prod/packages/_/cowsay/cowsay-0.2.0.webc",
"piritaSha256Hash": "9586938a0a89219dafe4ae97a901c56d4b3e2a9941520d1309ae880c9a1868c9"
}
},
{
"version": "0.1.3",
"piritaManifest": null,
"distribution": {
"piritaDownloadUrl": "https://example.com/",
"piritaSha256Hash": "1234asdf"
}
},
{
"version": "0.1.2",
"piritaManifest": "{}",
"distribution": {
"piritaDownloadUrl": null,
"piritaSha256Hash": "1234asdf"
}
},
{
"version": "0.1.3",
"piritaManifest": "{}",
"distribution": {
"piritaDownloadUrl": "https://example.com/",
"piritaSha256Hash":null
}
}
]
}
}
}

};
let response = HttpResponse {
body: Some(serde_json::to_vec(&body).unwrap()),
redirected: false,
status: StatusCode::OK,
headers: HeaderMap::new(),
};
let client = Arc::new(DummyClient::new(vec![response]));
let registry_endpoint = WapmSource::WAPM_PROD_ENDPOINT.parse().unwrap();
let request = PackageSpecifier::Registry {
full_name: "_/cowsay".to_string(),
version: semver::VersionReq::STAR,
};
let source = WapmSource::new(registry_endpoint, client.clone());

let summaries = source.query(&request).await.unwrap();

assert_eq!(summaries.len(), 1);
assert_eq!(summaries[0].pkg.version.to_string(), "0.2.0");
}
}