From 0b40f479719a23741ccbea3e8160692e7a68d647 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Wed, 11 Oct 2023 13:29:44 -0700 Subject: [PATCH 1/4] Replace cargo registry server's GIT index with sparse index --- cargo-registry/src/dummy_git_index.rs | 18 +----------- cargo-registry/src/main.rs | 40 ++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/cargo-registry/src/dummy_git_index.rs b/cargo-registry/src/dummy_git_index.rs index ae5def46b082bb..f83b6a44d00041 100644 --- a/cargo-registry/src/dummy_git_index.rs +++ b/cargo-registry/src/dummy_git_index.rs @@ -1,6 +1,5 @@ use { git2::{IndexAddOption, Repository}, - serde::{Deserialize, Serialize}, std::{ fs::{self, create_dir_all}, io::ErrorKind, @@ -9,27 +8,12 @@ use { }, }; -#[derive(Debug, Default, Deserialize, Serialize)] -struct RegistryConfig { - dl: String, - api: Option, -} - pub struct DummyGitIndex {} impl DummyGitIndex { - pub fn create_or_update_git_repo(root_dir: PathBuf, server_url: &str) { + pub fn create_or_update_git_repo(root_dir: PathBuf, expected_config: &str) { create_dir_all(&root_dir).expect("Failed to create root directory"); - let expected_config = serde_json::to_string(&RegistryConfig { - dl: format!( - "{}/api/v1/crates/{{crate}}/{{version}}/download", - server_url - ), - api: Some(server_url.to_string()), - }) - .expect("Failed to create expected config"); - let config_path = root_dir.join("config.json"); let config_written = if let Ok(config) = fs::read_to_string(&config_path) { if config != expected_config { diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index d225ca8b112f3e..d0c55c3ea5c32e 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -12,6 +12,7 @@ use { }, hyper_staticfile::Static, log::*, + serde::{Deserialize, Serialize}, std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, @@ -25,6 +26,12 @@ mod publisher; const PATH_PREFIX: &str = "/api/v1/crates"; +#[derive(Debug, Default, Deserialize, Serialize)] +struct RegistryConfig { + dl: String, + api: Option, +} + pub struct CargoRegistryService {} impl CargoRegistryService { @@ -43,13 +50,17 @@ impl CargoRegistryService { .unwrap() } - fn success_response() -> hyper::Response { + fn success_response_str(value: &str) -> hyper::Response { hyper::Response::builder() .status(hyper::StatusCode::OK) - .body(hyper::Body::from("")) + .body(hyper::Body::from(value.to_string())) .unwrap() } + fn success_response() -> hyper::Response { + Self::success_response_str("") + } + async fn handle_publish_request( request: hyper::Request, client: Arc, @@ -241,9 +252,11 @@ impl CargoRegistryService { } async fn handler( + config: String, request: hyper::Request, client: Arc, ) -> Result, Error> { + info!("Request is {:?}", &request); let path = request.uri().path(); if path.starts_with("/git") { return Static::new("/tmp/dummy-git") @@ -257,6 +270,10 @@ impl CargoRegistryService { }); } + if path == "/config.json" { + return Ok(Self::success_response_str(&config)); + } + if !path.starts_with(PATH_PREFIX) { return Ok(Self::error_response( hyper::StatusCode::BAD_REQUEST, @@ -308,13 +325,28 @@ async fn main() { let client = Arc::new(Client::new().expect("Failed to get RPC Client instance")); let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), client.port); - DummyGitIndex::create_or_update_git_repo(PathBuf::from("/tmp/dummy-git"), &client.server_url); + + let registry_config = RegistryConfig { + dl: format!( + "{}/api/v1/crates/{{crate}}/{{version}}/download", + client.server_url + ), + api: Some(client.server_url.to_string()), + }; + let registry_config_json = + serde_json::to_string(®istry_config).expect("Failed to create registry config"); + + DummyGitIndex::create_or_update_git_repo( + PathBuf::from("/tmp/dummy-git"), + ®istry_config_json, + ); let registry_service = make_service_fn(move |_| { let client_inner = client.clone(); + let config = registry_config_json.clone(); async move { Ok::<_, Error>(service_fn(move |request| { - CargoRegistryService::handler(request, client_inner.clone()) + CargoRegistryService::handler(config.clone(), request, client_inner.clone()) })) } }); From 9e92a31121bdedda97a727a9af952c8272f8416c Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Wed, 11 Oct 2023 13:36:07 -0700 Subject: [PATCH 2/4] Remove GIT index support --- cargo-registry/src/dummy_git_index.rs | 106 -------------------------- cargo-registry/src/main.rs | 23 +----- 2 files changed, 4 insertions(+), 125 deletions(-) delete mode 100644 cargo-registry/src/dummy_git_index.rs diff --git a/cargo-registry/src/dummy_git_index.rs b/cargo-registry/src/dummy_git_index.rs deleted file mode 100644 index f83b6a44d00041..00000000000000 --- a/cargo-registry/src/dummy_git_index.rs +++ /dev/null @@ -1,106 +0,0 @@ -use { - git2::{IndexAddOption, Repository}, - std::{ - fs::{self, create_dir_all}, - io::ErrorKind, - path::PathBuf, - process::Command, - }, -}; - -pub struct DummyGitIndex {} - -impl DummyGitIndex { - pub fn create_or_update_git_repo(root_dir: PathBuf, expected_config: &str) { - create_dir_all(&root_dir).expect("Failed to create root directory"); - - let config_path = root_dir.join("config.json"); - let config_written = if let Ok(config) = fs::read_to_string(&config_path) { - if config != expected_config { - fs::write(config_path, expected_config).expect("Failed to update config"); - true - } else { - false - } - } else { - fs::write(config_path, expected_config).expect("Failed to write config"); - true - }; - - #[cfg(unix)] - use std::os::unix::fs::symlink; - #[cfg(windows)] - use std::os::windows::fs::symlink_dir as symlink; - - let new_symlink = match symlink(".", root_dir.join("index")) { - Ok(()) => true, - Err(ref err) if err.kind() == ErrorKind::AlreadyExists => false, - Err(err) => panic!("Failed to create a symlink: {}", err), - }; - - let new_git_symlink = match symlink(".git", root_dir.join("git")) { - Ok(()) => true, - Err(ref err) if err.kind() == ErrorKind::AlreadyExists => false, - Err(err) => panic!("Failed to create git symlink: {}", err), - }; - - let repository = Repository::init(&root_dir).expect("Failed to GIT init"); - - let empty = repository - .is_empty() - .expect("Failed to check if GIT repo is empty"); - - if empty || config_written || new_symlink || new_git_symlink { - let mut index = repository.index().expect("cannot get the Index file"); - index - .add_all( - ["config.json", "index"].iter(), - IndexAddOption::DEFAULT, - None, - ) - .expect("Failed to add modified files to git index"); - index.write().expect("Failed to update the git index"); - - let tree = index - .write_tree() - .and_then(|tree_id| repository.find_tree(tree_id)) - .expect("Failed to get tree"); - - let signature = repository.signature().expect("Failed to get signature"); - - if empty { - repository.commit( - Some("HEAD"), - &signature, - &signature, - "Created new repo", - &tree, - &[], - ) - } else { - let oid = repository - .refname_to_id("HEAD") - .expect("Failed to get HEAD ref"); - let parent = repository - .find_commit(oid) - .expect("Failed to find parent commit"); - - repository.commit( - Some("HEAD"), - &signature, - &signature, - "Updated GIT repo", - &tree, - &[&parent], - ) - } - .expect("Failed to commit the changes"); - } - - Command::new("git") - .current_dir(&root_dir) - .arg("update-server-info") - .status() - .expect("git update-server-info failed"); - } -} diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index d0c55c3ea5c32e..0f99919585bd25 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -2,7 +2,6 @@ use { crate::{ client::Client, - dummy_git_index::DummyGitIndex, publisher::{Error, Publisher}, }, hyper::{ @@ -10,18 +9,15 @@ use { service::{make_service_fn, service_fn}, Method, Server, }, - hyper_staticfile::Static, log::*, serde::{Deserialize, Serialize}, std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, - path::PathBuf, sync::Arc, }, }; mod client; -mod dummy_git_index; mod publisher; const PATH_PREFIX: &str = "/api/v1/crates"; @@ -256,18 +252,12 @@ impl CargoRegistryService { request: hyper::Request, client: Arc, ) -> Result, Error> { - info!("Request is {:?}", &request); let path = request.uri().path(); if path.starts_with("/git") { - return Static::new("/tmp/dummy-git") - .serve(request) - .await - .or_else(|_| { - Ok(Self::error_response( - hyper::StatusCode::BAD_REQUEST, - "Failed to serve git index", - )) - }); + return Ok(Self::error_response( + hyper::StatusCode::BAD_REQUEST, + "This registry server does not support GIT index. Please use sparse index.", + )); } if path == "/config.json" { @@ -336,11 +326,6 @@ async fn main() { let registry_config_json = serde_json::to_string(®istry_config).expect("Failed to create registry config"); - DummyGitIndex::create_or_update_git_repo( - PathBuf::from("/tmp/dummy-git"), - ®istry_config_json, - ); - let registry_service = make_service_fn(move |_| { let client_inner = client.clone(); let config = registry_config_json.clone(); From c9db6b38223eb91f9d43b1892ecf42de62a0ad58 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Wed, 11 Oct 2023 15:49:56 -0700 Subject: [PATCH 3/4] handler for crate download request processing --- cargo-registry/src/main.rs | 176 +++++++++++++++++++++++++++++++++++-- 1 file changed, 168 insertions(+), 8 deletions(-) diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index 0f99919585bd25..2f7bd96412ebf6 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -247,6 +247,42 @@ impl CargoRegistryService { ) } + fn get_crate_name_from_path(path: &str) -> Option<&str> { + let (path, crate_name) = path.rsplit_once('/')?; + + match crate_name.len() { + 0 => false, + 1 => path == "/1", + 2 => path == "/2", + 3 => { + let first_char = crate_name.chars().next()?; + path == format!("/3/{}", first_char) + } + _ => { + let (first_two_char, rest) = crate_name.split_at(2); + let (next_two_char, _) = rest.split_at(2); + path == format!("/{}/{}", first_two_char, next_two_char) + } + } + .then_some(crate_name) + } + + fn handle_crate_dl_request( + request: &hyper::Request, + ) -> hyper::Response { + let Some(crate_name) = Self::get_crate_name_from_path(request.uri().path()) else { + return Self::error_response( + hyper::StatusCode::BAD_REQUEST, + "Invalid path for the request", + ); + }; + + // Fetch the program and return a crate buffer. + info!("Received a request to fetch {:?}", crate_name); + + Self::success_response() + } + async fn handler( config: String, request: hyper::Request, @@ -265,10 +301,7 @@ impl CargoRegistryService { } if !path.starts_with(PATH_PREFIX) { - return Ok(Self::error_response( - hyper::StatusCode::BAD_REQUEST, - "Invalid path for the request", - )); + return Ok(Self::handle_crate_dl_request(&request)); } let Some((path, endpoint)) = path.rsplit_once('/') else { @@ -317,10 +350,7 @@ async fn main() { let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), client.port); let registry_config = RegistryConfig { - dl: format!( - "{}/api/v1/crates/{{crate}}/{{version}}/download", - client.server_url - ), + dl: format!("{}/api/v1/crates", client.server_url), api: Some(client.server_url.to_string()), }; let registry_config_json = @@ -341,3 +371,133 @@ async fn main() { let _ = server.await; } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_get_crate_name_from_path() { + assert_eq!(CargoRegistryService::get_crate_name_from_path(""), None); + assert_eq!(CargoRegistryService::get_crate_name_from_path("/"), None); + + // Single character crate name + assert_eq!(CargoRegistryService::get_crate_name_from_path("/a"), None); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/1/a"), + Some("a") + ); + assert_eq!(CargoRegistryService::get_crate_name_from_path("/2/a"), None); + assert_eq!(CargoRegistryService::get_crate_name_from_path("/a/a"), None); + + // Two character crate name + assert_eq!(CargoRegistryService::get_crate_name_from_path("/ab"), None); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/1/ab"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/2/ab"), + Some("ab") + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/ab"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/ab"), + None + ); + + // Three character crate name + assert_eq!(CargoRegistryService::get_crate_name_from_path("/abc"), None); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/1/abc"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/2/abc"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/abc"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/a/abc"), + Some("abc") + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/abc"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/c/abc"), + None + ); + + // Four character crate name + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/1/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/2/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/a/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/4/abcd"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/cd/abcd"), + Some("abcd") + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/cd/abc"), + None + ); + + // More character crate name + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/1/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/2/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/3/a/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/4/abcdefgh"), + None + ); + assert_eq!( + CargoRegistryService::get_crate_name_from_path("/ab/cd/abcdefgh"), + Some("abcdefgh") + ); + } +} From 5db3ba901136bbec58484367a765f3e8ea49a333 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Thu, 12 Oct 2023 10:12:53 -0700 Subject: [PATCH 4/4] restructure the index code --- cargo-registry/src/main.rs | 293 +++++-------------------- cargo-registry/src/response_builder.rs | 27 +++ cargo-registry/src/sparse_index.rs | 158 +++++++++++++ 3 files changed, 238 insertions(+), 240 deletions(-) create mode 100644 cargo-registry/src/response_builder.rs create mode 100644 cargo-registry/src/sparse_index.rs diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index 2f7bd96412ebf6..60227fa32a9962 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -10,7 +10,6 @@ use { Method, Server, }, log::*, - serde::{Deserialize, Serialize}, std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, @@ -20,43 +19,14 @@ use { mod client; mod publisher; -const PATH_PREFIX: &str = "/api/v1/crates"; +mod response_builder; +mod sparse_index; -#[derive(Debug, Default, Deserialize, Serialize)] -struct RegistryConfig { - dl: String, - api: Option, -} +const PATH_PREFIX: &str = "/api/v1/crates"; pub struct CargoRegistryService {} impl CargoRegistryService { - fn error_response(status: hyper::StatusCode, msg: &str) -> hyper::Response { - error!("{}", msg); - hyper::Response::builder() - .status(status) - .body(hyper::Body::from( - serde_json::json!({ - "errors" : [ - {"details": msg} - ] - }) - .to_string(), - )) - .unwrap() - } - - fn success_response_str(value: &str) -> hyper::Response { - hyper::Response::builder() - .status(hyper::StatusCode::OK) - .body(hyper::Body::from(value.to_string())) - .unwrap() - } - - fn success_response() -> hyper::Response { - Self::success_response_str("") - } - async fn handle_publish_request( request: hyper::Request, client: Arc, @@ -70,7 +40,7 @@ impl CargoRegistryService { tokio::task::spawn_blocking(move || Publisher::publish_crate(data, client)) .await else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::INTERNAL_SERVER_ERROR, "Internal error. Failed to wait for program deployment", ); @@ -78,15 +48,15 @@ impl CargoRegistryService { if result.is_ok() { info!("Published the crate successfully. {:?}", result); - Self::success_response() + response_builder::success_response() } else { - Self::error_response( + response_builder::error_response( hyper::StatusCode::BAD_REQUEST, format!("Failed to publish the crate. {:?}", result).as_str(), ) } } - Err(_) => Self::error_response( + Err(_) => response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to receive the crate data from the client.", ), @@ -106,20 +76,20 @@ impl CargoRegistryService { _request: &hyper::Request, ) -> hyper::Response { let Some((path, _crate_name, _version)) = Self::get_crate_name_and_version(path) else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to parse the request.", ); }; if path.len() != PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) @@ -130,20 +100,20 @@ impl CargoRegistryService { _request: &hyper::Request, ) -> hyper::Response { let Some((path, _crate_name, _version)) = Self::get_crate_name_and_version(path) else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to parse the request.", ); }; if path.len() != PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) @@ -158,20 +128,20 @@ impl CargoRegistryService { _request: &hyper::Request, ) -> hyper::Response { let Some((path, _crate_name)) = Self::get_crate_name(path) else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to parse the request.", ); }; if path.len() != PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) @@ -182,20 +152,20 @@ impl CargoRegistryService { _request: &hyper::Request, ) -> hyper::Response { let Some((path, _crate_name)) = Self::get_crate_name(path) else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to parse the request.", ); }; if path.len() != PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) @@ -206,20 +176,20 @@ impl CargoRegistryService { _request: &hyper::Request, ) -> hyper::Response { let Some((path, _crate_name)) = Self::get_crate_name(path) else { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Failed to parse the request.", ); }; if path.len() != PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) @@ -235,77 +205,44 @@ impl CargoRegistryService { // full path started with PATH_PREFIX. So it's sufficient to check that provided // path is smaller than PATH_PREFIX. if path.len() >= PATH_PREFIX.len() { - return Self::error_response( + return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Request length is incorrect", ); } - Self::error_response( + response_builder::error_response( hyper::StatusCode::NOT_IMPLEMENTED, "This command is not implemented yet", ) } - fn get_crate_name_from_path(path: &str) -> Option<&str> { - let (path, crate_name) = path.rsplit_once('/')?; - - match crate_name.len() { - 0 => false, - 1 => path == "/1", - 2 => path == "/2", - 3 => { - let first_char = crate_name.chars().next()?; - path == format!("/3/{}", first_char) - } - _ => { - let (first_two_char, rest) = crate_name.split_at(2); - let (next_two_char, _) = rest.split_at(2); - path == format!("/{}/{}", first_two_char, next_two_char) - } - } - .then_some(crate_name) - } - - fn handle_crate_dl_request( - request: &hyper::Request, - ) -> hyper::Response { - let Some(crate_name) = Self::get_crate_name_from_path(request.uri().path()) else { - return Self::error_response( - hyper::StatusCode::BAD_REQUEST, - "Invalid path for the request", - ); - }; - - // Fetch the program and return a crate buffer. - info!("Received a request to fetch {:?}", crate_name); - - Self::success_response() - } - async fn handler( - config: String, + index: sparse_index::RegistryIndex, request: hyper::Request, client: Arc, ) -> Result, Error> { let path = request.uri().path(); if path.starts_with("/git") { - return Ok(Self::error_response( + return Ok(response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "This registry server does not support GIT index. Please use sparse index.", )); } - if path == "/config.json" { - return Ok(Self::success_response_str(&config)); + if path.starts_with(index.index_root.as_str()) { + return Ok(index.handler(request)); } if !path.starts_with(PATH_PREFIX) { - return Ok(Self::handle_crate_dl_request(&request)); + return Ok(response_builder::error_response( + hyper::StatusCode::BAD_REQUEST, + "Invalid path for the request", + )); } let Some((path, endpoint)) = path.rsplit_once('/') else { - return Ok(Self::error_response( + return Ok(response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Invalid endpoint in the path", )); @@ -315,7 +252,7 @@ impl CargoRegistryService { Method::PUT => match endpoint { "new" => { if path.len() != PATH_PREFIX.len() { - Self::error_response( + response_builder::error_response( hyper::StatusCode::BAD_REQUEST, "Invalid length of the request.", ) @@ -325,19 +262,31 @@ impl CargoRegistryService { } "unyank" => Self::handle_unyank_request(path, &request), "owners" => Self::handle_add_owners_request(path, &request), - _ => Self::error_response(hyper::StatusCode::METHOD_NOT_ALLOWED, "Unknown request"), + _ => response_builder::error_response( + hyper::StatusCode::METHOD_NOT_ALLOWED, + "Unknown request", + ), }, Method::GET => match endpoint { "crates" => Self::handle_get_crates_request(path, &request), "owners" => Self::handle_get_owners_request(path, &request), - _ => Self::error_response(hyper::StatusCode::METHOD_NOT_ALLOWED, "Unknown request"), + _ => response_builder::error_response( + hyper::StatusCode::METHOD_NOT_ALLOWED, + "Unknown request", + ), }, Method::DELETE => match endpoint { "yank" => Self::handle_yank_request(path, &request), "owners" => Self::handle_delete_owners_request(path, &request), - _ => Self::error_response(hyper::StatusCode::METHOD_NOT_ALLOWED, "Unknown request"), + _ => response_builder::error_response( + hyper::StatusCode::METHOD_NOT_ALLOWED, + "Unknown request", + ), }, - _ => Self::error_response(hyper::StatusCode::METHOD_NOT_ALLOWED, "Unknown request"), + _ => response_builder::error_response( + hyper::StatusCode::METHOD_NOT_ALLOWED, + "Unknown request", + ), }) } } @@ -348,20 +297,14 @@ async fn main() { let client = Arc::new(Client::new().expect("Failed to get RPC Client instance")); let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), client.port); - - let registry_config = RegistryConfig { - dl: format!("{}/api/v1/crates", client.server_url), - api: Some(client.server_url.to_string()), - }; - let registry_config_json = - serde_json::to_string(®istry_config).expect("Failed to create registry config"); + let index = sparse_index::RegistryIndex::new("/index", &client.server_url); let registry_service = make_service_fn(move |_| { let client_inner = client.clone(); - let config = registry_config_json.clone(); + let index = index.clone(); async move { Ok::<_, Error>(service_fn(move |request| { - CargoRegistryService::handler(config.clone(), request, client_inner.clone()) + CargoRegistryService::handler(index.clone(), request, client_inner.clone()) })) } }); @@ -371,133 +314,3 @@ async fn main() { let _ = server.await; } - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_get_crate_name_from_path() { - assert_eq!(CargoRegistryService::get_crate_name_from_path(""), None); - assert_eq!(CargoRegistryService::get_crate_name_from_path("/"), None); - - // Single character crate name - assert_eq!(CargoRegistryService::get_crate_name_from_path("/a"), None); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/1/a"), - Some("a") - ); - assert_eq!(CargoRegistryService::get_crate_name_from_path("/2/a"), None); - assert_eq!(CargoRegistryService::get_crate_name_from_path("/a/a"), None); - - // Two character crate name - assert_eq!(CargoRegistryService::get_crate_name_from_path("/ab"), None); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/1/ab"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/2/ab"), - Some("ab") - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/ab"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/ab"), - None - ); - - // Three character crate name - assert_eq!(CargoRegistryService::get_crate_name_from_path("/abc"), None); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/1/abc"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/2/abc"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/abc"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/a/abc"), - Some("abc") - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/abc"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/c/abc"), - None - ); - - // Four character crate name - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/1/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/2/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/a/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/4/abcd"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/cd/abcd"), - Some("abcd") - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/cd/abc"), - None - ); - - // More character crate name - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/1/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/2/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/3/a/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/4/abcdefgh"), - None - ); - assert_eq!( - CargoRegistryService::get_crate_name_from_path("/ab/cd/abcdefgh"), - Some("abcdefgh") - ); - } -} diff --git a/cargo-registry/src/response_builder.rs b/cargo-registry/src/response_builder.rs new file mode 100644 index 00000000000000..8a56e298f713ae --- /dev/null +++ b/cargo-registry/src/response_builder.rs @@ -0,0 +1,27 @@ +use {crate::response_builder, log::error}; + +pub(crate) fn error_response(status: hyper::StatusCode, msg: &str) -> hyper::Response { + error!("{}", msg); + hyper::Response::builder() + .status(status) + .body(hyper::Body::from( + serde_json::json!({ + "errors" : [ + {"details": msg} + ] + }) + .to_string(), + )) + .unwrap() +} + +pub(crate) fn success_response_str(value: &str) -> hyper::Response { + hyper::Response::builder() + .status(hyper::StatusCode::OK) + .body(hyper::Body::from(value.to_string())) + .unwrap() +} + +pub(crate) fn success_response() -> hyper::Response { + response_builder::success_response_str("") +} diff --git a/cargo-registry/src/sparse_index.rs b/cargo-registry/src/sparse_index.rs new file mode 100644 index 00000000000000..59b9b88985c445 --- /dev/null +++ b/cargo-registry/src/sparse_index.rs @@ -0,0 +1,158 @@ +use { + crate::response_builder, + log::info, + serde::{Deserialize, Serialize}, +}; + +#[derive(Debug, Default, Deserialize, Serialize)] +struct RegistryConfig { + dl: String, + api: Option, +} + +#[derive(Clone)] +pub struct RegistryIndex { + pub(crate) index_root: String, + config: String, +} + +impl RegistryIndex { + pub fn new(root: &str, server_url: &str) -> Self { + let registry_config = RegistryConfig { + dl: format!("{}/api/v1/crates", server_url), + api: Some(server_url.to_string()), + }; + let config = + serde_json::to_string(®istry_config).expect("Failed to create registry config"); + + info!("Registry index is available at {}{}/", server_url, root); + Self { + index_root: root.to_string(), + config, + } + } + + pub fn handler(&self, request: hyper::Request) -> hyper::Response { + let path = request.uri().path(); + let expected_root = self.index_root.as_str(); + if !path.starts_with(expected_root) { + return response_builder::error_response( + hyper::StatusCode::BAD_REQUEST, + "Invalid path for index", + ); + } + + let Some((_, path)) = path.split_once(expected_root) else { + return response_builder::error_response( + hyper::StatusCode::BAD_REQUEST, + "Invalid path for index", + ); + }; + + if path == "/config.json" { + return response_builder::success_response_str(&self.config); + } + + Self::handle_crate_lookup_request(path) + } + + fn get_crate_name_from_path(path: &str) -> Option<&str> { + let (path, crate_name) = path.rsplit_once('/')?; + + // The index for deployed crates follow the path naming described here + // https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + match crate_name.len() { + 0 => false, + 1 => path == "/1", + 2 => path == "/2", + 3 => { + let first_char = crate_name.chars().next()?; + path == format!("/3/{}", first_char) + } + _ => { + let (first_two_char, rest) = crate_name.split_at(2); + let (next_two_char, _) = rest.split_at(2); + path == format!("/{}/{}", first_two_char, next_two_char) + } + } + .then_some(crate_name) + } + + fn handle_crate_lookup_request(path: &str) -> hyper::Response { + let Some(crate_name) = Self::get_crate_name_from_path(path) else { + return response_builder::error_response( + hyper::StatusCode::BAD_REQUEST, + "Invalid path for the request", + ); + }; + + // Fetch the index information for the crate + info!("Received a request to fetch {:?}", crate_name); + + response_builder::success_response() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_get_crate_name_from_path() { + assert_eq!(RegistryIndex::get_crate_name_from_path(""), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/"), None); + + // Single character crate name + assert_eq!(RegistryIndex::get_crate_name_from_path("/a"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/1/a"), Some("a")); + assert_eq!(RegistryIndex::get_crate_name_from_path("/2/a"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/a/a"), None); + + // Two character crate name + assert_eq!(RegistryIndex::get_crate_name_from_path("/ab"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/1/ab"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/2/ab"), Some("ab")); + assert_eq!(RegistryIndex::get_crate_name_from_path("/3/ab"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/ab/ab"), None); + + // Three character crate name + assert_eq!(RegistryIndex::get_crate_name_from_path("/abc"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/1/abc"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/2/abc"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/3/abc"), None); + assert_eq!( + RegistryIndex::get_crate_name_from_path("/3/a/abc"), + Some("abc") + ); + assert_eq!(RegistryIndex::get_crate_name_from_path("/ab/abc"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/ab/c/abc"), None); + + // Four character crate name + assert_eq!(RegistryIndex::get_crate_name_from_path("/abcd"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/1/abcd"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/2/abcd"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/3/abcd"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/3/a/abcd"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/4/abcd"), None); + assert_eq!( + RegistryIndex::get_crate_name_from_path("/ab/cd/abcd"), + Some("abcd") + ); + assert_eq!(RegistryIndex::get_crate_name_from_path("/ab/cd/abc"), None); + + // More character crate name + assert_eq!(RegistryIndex::get_crate_name_from_path("/abcdefgh"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/1/abcdefgh"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/2/abcdefgh"), None); + assert_eq!(RegistryIndex::get_crate_name_from_path("/3/abcdefgh"), None); + assert_eq!( + RegistryIndex::get_crate_name_from_path("/3/a/abcdefgh"), + None + ); + assert_eq!(RegistryIndex::get_crate_name_from_path("/4/abcdefgh"), None); + assert_eq!( + RegistryIndex::get_crate_name_from_path("/ab/cd/abcdefgh"), + Some("abcdefgh") + ); + } +}