From dfe3134212d52811aa1ea33fdb19c2769f07be93 Mon Sep 17 00:00:00 2001 From: Pasha Podolsky Date: Thu, 15 Dec 2022 14:44:27 +0100 Subject: [PATCH] feat: support of Gateway Subdomain spec (#546) * [feat] Support subdomains * pr: fixes * pr: fix test * fix: remove mut in iroh/src/run.rs --- iroh-api/src/api.rs | 2 +- iroh-bitswap/src/lib.rs | 11 +- iroh-bitswap/src/network.rs | 8 +- iroh-gateway/src/bad_bits.rs | 17 +- iroh-gateway/src/client.rs | 15 +- iroh-gateway/src/config.rs | 9 + iroh-gateway/src/constants.rs | 2 + iroh-gateway/src/core.rs | 422 ++++++++++++++++++++++------- iroh-gateway/src/handler_params.rs | 127 +++++++++ iroh-gateway/src/handlers.rs | 402 +++++++++++++++------------ iroh-gateway/src/lib.rs | 2 + iroh-gateway/src/text.rs | 70 +++++ iroh-one/src/config.rs | 4 + iroh-resolver/Cargo.toml | 2 + iroh-resolver/src/resolver.rs | 62 ++++- iroh-resolver/tests/unixfs.rs | 2 +- iroh-rpc-types/src/addr.rs | 4 +- iroh-store/src/store.rs | 2 +- iroh-unixfs/src/balanced_tree.rs | 2 +- iroh-unixfs/src/builder.rs | 2 +- iroh-unixfs/src/hamt.rs | 2 +- iroh-unixfs/src/hamt/bitfield.rs | 6 +- iroh-util/src/lib.rs | 2 +- iroh-util/src/lock.rs | 4 +- iroh/Cargo.toml | 1 + stores/flatfs/src/flatfs.rs | 16 +- stores/flatfs/src/shard.rs | 5 +- xtask/src/main.rs | 4 +- 28 files changed, 883 insertions(+), 324 deletions(-) create mode 100644 iroh-gateway/src/handler_params.rs create mode 100644 iroh-gateway/src/text.rs diff --git a/iroh-api/src/api.rs b/iroh-api/src/api.rs index 1fcf4e15bf..ac7dc0757c 100644 --- a/iroh-api/src/api.rs +++ b/iroh-api/src/api.rs @@ -159,7 +159,7 @@ impl Api { self.client.check().await } - pub async fn watch(&self) -> LocalBoxStream<'static, iroh_rpc_client::StatusTable> { + pub async fn watch(&self) -> LocalBoxStream<'static, StatusTable> { self.client.clone().watch().await.boxed_local() } diff --git a/iroh-bitswap/src/lib.rs b/iroh-bitswap/src/lib.rs index c6ec22357d..f3bc7ca576 100644 --- a/iroh-bitswap/src/lib.rs +++ b/iroh-bitswap/src/lib.rs @@ -545,9 +545,9 @@ impl NetworkBehaviour for Bitswap { { // Do not bother trying to dial these for now. if let Err(err) = - response.send(Err(format!("dial:{}: undialable peer", id))) + response.send(Err(format!("dial:{id}: undialable peer"))) { - debug!("dial:{}: failed to send dial response {:?}", id, err) + debug!("dial:{id}: failed to send dial response {err:?}") } continue; } @@ -555,12 +555,9 @@ impl NetworkBehaviour for Bitswap { if self.pause_dialing { // already connected if let Err(err) = - response.send(Err(format!("dial:{}: dialing paused", id))) + response.send(Err(format!("dial:{id}: dialing paused"))) { - debug!( - "dial:{}: failed to send dial response {:?}", - id, err - ) + debug!("dial:{id}: failed to send dial response {err:?}",) } continue; } diff --git a/iroh-bitswap/src/network.rs b/iroh-bitswap/src/network.rs index a10301ea05..42d0e44bcb 100644 --- a/iroh-bitswap/src/network.rs +++ b/iroh-bitswap/src/network.rs @@ -344,13 +344,7 @@ impl Default for MessageSenderConfig { fn send_timeout(size: usize) -> Duration { let mut timeout = SEND_LATENCY; timeout += Duration::from_secs(size as u64 / MIN_SEND_RATE); - if timeout > MAX_SEND_TIMEOUT { - MAX_SEND_TIMEOUT - } else if timeout < MIN_SEND_TIMEOUT { - MIN_SEND_TIMEOUT - } else { - timeout - } + timeout.clamp(MIN_SEND_TIMEOUT, MAX_SEND_TIMEOUT) } #[derive(Debug)] diff --git a/iroh-gateway/src/bad_bits.rs b/iroh-gateway/src/bad_bits.rs index 5847e82714..a7c1eecdc8 100644 --- a/iroh-gateway/src/bad_bits.rs +++ b/iroh-gateway/src/bad_bits.rs @@ -1,7 +1,7 @@ use cid::Cid; use serde::{de, Deserialize, Deserializer, Serialize}; use sha2::{Digest, Sha256}; -use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; +use std::{collections::HashSet, sync::Arc, time::Duration}; use tokio::{sync::RwLock, task::JoinHandle}; use tracing::{debug, log::error}; @@ -42,12 +42,8 @@ impl BadBits { self.denylist = denylist; } - pub fn is_bad(&self, cid: &str, path: &str) -> bool { - let cid = match Cid::from_str(cid) { - Ok(cid) => cid, - Err(_) => return false, - }; - let hash = BadBits::to_anchor(cid, path); + pub fn is_bad(&self, cid: &Cid, path: &str) -> bool { + let hash = BadBits::to_anchor(*cid, path); self.denylist.contains(&hash) } @@ -106,15 +102,17 @@ pub fn spawn_bad_bits_updater(bad_bits: Arc>>) -> Option< #[cfg(test)] mod tests { - use crate::config::Config; + use std::str::FromStr; - use super::*; use hex_literal::hex; use http::StatusCode; use iroh_resolver::dns_resolver::Config as DnsResolverConfig; use iroh_rpc_client::{Client as RpcClient, Config as RpcClientConfig}; use iroh_unixfs::content_loader::{FullLoader, FullLoaderConfig, GatewayUrl}; + use super::*; + use crate::config::Config; + #[tokio::test] async fn bad_bits_anchor() { let cid = @@ -186,6 +184,7 @@ mod tests { channels: Some(1), }, ); + config.redirect_to_subdomain = false; config.set_default_headers(); let rpc_addr = "irpc://0.0.0.0:0".parse().unwrap(); diff --git a/iroh-gateway/src/client.rs b/iroh-gateway/src/client.rs index f4a224fecf..6c10bc7914 100644 --- a/iroh-gateway/src/client.rs +++ b/iroh-gateway/src/client.rs @@ -23,7 +23,7 @@ use tokio_util::io::ReaderStream; use tracing::{info, warn}; use crate::response::ResponseFormat; -use crate::{constants::RECURSION_LIMIT, handlers::GetParams}; +use crate::{constants::RECURSION_LIMIT, handler_params::GetParams}; #[derive(Debug, Clone)] pub struct Client { @@ -231,13 +231,24 @@ impl Client { } #[derive(Debug, Clone)] -pub struct Request { +pub struct IpfsRequest { pub format: ResponseFormat, pub cid: CidOrDomain, pub resolved_path: iroh_resolver::resolver::Path, pub query_file_name: String, pub download: bool, pub query_params: GetParams, + pub subdomain_mode: bool, +} + +impl IpfsRequest { + pub fn request_path_for_redirection(&self) -> String { + if self.subdomain_mode { + self.resolved_path.to_relative_string() + } else { + self.resolved_path.to_string() + } + } } async fn fetch_car_recursive( diff --git a/iroh-gateway/src/config.rs b/iroh-gateway/src/config.rs index 51f6badc13..4c7016f82c 100644 --- a/iroh-gateway/src/config.rs +++ b/iroh-gateway/src/config.rs @@ -48,6 +48,9 @@ pub struct Config { /// set of user provided headers to attach to all responses #[serde(with = "http_serde::header_map")] pub headers: HeaderMap, + /// Redirects to subdomains for path requests + #[serde(default)] + pub redirect_to_subdomain: bool, } impl Config { @@ -62,6 +65,7 @@ impl Config { indexer_endpoint: None, metrics: MetricsConfig::default(), use_denylist: false, + redirect_to_subdomain: false, } } @@ -116,6 +120,7 @@ impl Default for Config { indexer_endpoint: None, metrics: MetricsConfig::default(), use_denylist: false, + redirect_to_subdomain: false, }; t.set_default_headers(); t @@ -166,6 +171,10 @@ impl crate::handlers::StateConfig for Config { fn user_headers(&self) -> &HeaderMap { &self.headers } + + fn redirect_to_subdomain(&self) -> bool { + self.redirect_to_subdomain + } } fn collect_headers(headers: &HeaderMap) -> Result, ConfigError> { diff --git a/iroh-gateway/src/constants.rs b/iroh-gateway/src/constants.rs index af6ecbb56f..61db420b6b 100644 --- a/iroh-gateway/src/constants.rs +++ b/iroh-gateway/src/constants.rs @@ -1,6 +1,8 @@ use axum::http::{header::HeaderName, HeaderValue}; // Headers +pub static HEADER_X_FORWARDED_HOST: HeaderName = HeaderName::from_static("x-forwarded-host"); +pub static HEADER_X_FORWARDED_PROTO: HeaderName = HeaderName::from_static("x-forwarded-proto"); pub static HEADER_X_IPFS_PATH: HeaderName = HeaderName::from_static("x-ipfs-path"); pub static HEADER_X_CONTENT_TYPE_OPTIONS: HeaderName = HeaderName::from_static("x-content-type-options"); diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index 7cf6cdb9d8..f1d280e7b2 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -28,7 +28,7 @@ pub struct State { pub bad_bits: Arc>>, } -impl Core { +impl Core { pub async fn new( config: Arc, rpc_addr: GatewayAddr, @@ -120,8 +120,10 @@ mod tests { use std::net::SocketAddr; use super::*; + use axum::response::Response; use cid::Cid; use futures::{StreamExt, TryStreamExt}; + use hyper::Body; use iroh_rpc_client::Client as RpcClient; use iroh_rpc_client::Config as RpcClientConfig; use iroh_rpc_types::store::StoreAddr; @@ -134,6 +136,23 @@ mod tests { use crate::config::Config; + struct TestSetup { + gateway_addr: SocketAddr, + root_cid: Cid, + file_cids: Vec, + core_task: tokio::task::JoinHandle<()>, + store_task: tokio::task::JoinHandle<()>, + files: Vec<(String, Vec)>, + } + + impl TestSetup { + pub async fn shutdown(self) { + self.core_task.abort(); + self.store_task.abort(); + self.store_task.await.ok(); + } + } + async fn spawn_gateway( config: Arc, ) -> (SocketAddr, RpcClient, tokio::task::JoinHandle<()>) { @@ -185,14 +204,14 @@ mod tests { async fn put_directory_with_files( rpc_client: &RpcClient, dir: &str, - files: &[(&str, Vec)], + files: &[(String, Vec)], ) -> (Cid, Vec) { let store = rpc_client.try_store().unwrap(); let mut cids = vec![]; let mut dir_builder = DirectoryBuilder::new().name(dir); for (name, content) in files { let file = FileBuilder::new() - .name(*name) + .name(name) .content_bytes(content.clone()) .build() .await @@ -210,6 +229,62 @@ mod tests { (*cids.last().unwrap(), cids) } + async fn do_request( + method: &str, + authority: &str, + path_and_query: &str, + headers: Option<&[(&str, &str)]>, + ) -> Response { + let client = hyper::Client::new(); + let uri = hyper::Uri::builder() + .scheme("http") + .authority(authority) + .path_and_query(path_and_query) + .build() + .unwrap(); + let mut req = hyper::Request::builder().method(method).uri(uri); + if let Some(headers) = headers { + for header in headers { + req = req.header(header.0, header.1); + } + } + client + .request(req.body(hyper::Body::empty()).unwrap()) + .await + .unwrap() + } + + async fn setup_test(redirect_to_subdomains: bool) -> TestSetup { + let (store_client_addr, store_task) = spawn_store().await; + let mut config = Config::new( + 0, + RpcClientConfig { + gateway_addr: None, + p2p_addr: None, + store_addr: Some(store_client_addr), + channels: Some(1), + }, + ); + config.set_default_headers(); + config.redirect_to_subdomain = redirect_to_subdomains; + let (gateway_addr, rpc_client, core_task) = spawn_gateway(Arc::new(config)).await; + let dir = "demo"; + let files = vec![ + ("hello.txt".to_string(), b"ola".to_vec()), + ("world.txt".to_string(), b"mundo".to_vec()), + ]; + let (root_cid, file_cids) = put_directory_with_files(&rpc_client, dir, &files).await; + + TestSetup { + gateway_addr, + root_cid, + file_cids, + core_task, + store_task, + files, + } + } + #[tokio::test] async fn gateway_health() { let mut config = Config::new( @@ -243,47 +318,97 @@ mod tests { // TODO(b5) - refactor to return anyhow::Result<()> #[tokio::test] - async fn fetch_car_recursive() { - let (store_client_addr, store_task) = spawn_store().await; - let mut config = Config::new( - 0, - RpcClientConfig { - gateway_addr: None, - p2p_addr: None, - store_addr: Some(store_client_addr), - channels: Some(1), - }, + async fn test_fetch_car_recursive() { + let test_setup = setup_test(false).await; + + // request the root cid as a recursive car + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}?recursive=true", test_setup.root_cid), + Some(&[("accept", "application/vnd.ipld.car")]), + ) + .await; + assert_eq!(http::StatusCode::OK, res.status()); + + // read the response body into a car reader and map the entries + // to UnixFS nodes + let body = StreamReader::new( + res.into_body() + .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string())), + ); + let car_reader = iroh_car::CarReader::new(body).await.unwrap(); + let (nodes, cids): (Vec, Vec) = car_reader + .stream() + .map(|res| res.unwrap()) + .map(|(cid, bytes)| (UnixfsNode::decode(&cid, bytes.into()).unwrap(), cid)) + .unzip() + .await; + // match cids and content + assert_eq!(cids.len(), test_setup.file_cids.len()); + assert_eq!( + HashSet::<_>::from_iter(cids.iter()), + HashSet::from_iter(test_setup.file_cids.iter()) + ); + assert_eq!(cids[0], test_setup.root_cid); + assert_eq!(nodes.len(), test_setup.files.len() + 1); + assert!(nodes[0].is_dir()); + assert_eq!( + nodes[0] + .links() + .map(|link| link.unwrap().name.unwrap().to_string()) + .collect::>(), + test_setup + .files + .iter() + .map(|(name, _content)| name.to_string()) + .collect::>() ); - config.set_default_headers(); - let (addr, rpc_client, core_task) = spawn_gateway(Arc::new(config)).await; + for (i, node) in nodes[1..].iter().enumerate() { + assert_eq!(node, &UnixfsNode::Raw(test_setup.files[i].1.clone().into())); + } + test_setup.shutdown().await + } - let dir = "demo"; - let files = [ - ("hello.txt", b"ola".to_vec()), - ("world.txt", b"mundo".to_vec()), - ]; + #[tokio::test] + async fn test_head_request_to_file() { + let test_setup = setup_test(false).await; + + // request the root cid as a recursive car + let res = do_request( + "HEAD", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/{}", test_setup.root_cid, "world.txt"), + None, + ) + .await; + + assert_eq!(http::StatusCode::OK, res.status()); + assert!(res.headers().get("content-length").is_some()); + assert_eq!(res.headers().get("content-length").unwrap(), "5"); + + let (body, _) = res.into_body().into_future().await; + assert!(body.is_none()); + + test_setup.shutdown().await + } - // add a directory with two files to the store. - let (root_cid, all_cids) = put_directory_with_files(&rpc_client, dir, &files).await; + #[tokio::test] + async fn test_gateway_requests() { + let test_setup = setup_test(false).await; // request the root cid as a recursive car - let res = { - let client = hyper::Client::new(); - let uri = hyper::Uri::builder() - .scheme("http") - .authority(format!("localhost:{}", addr.port())) - .path_and_query(format!("/ipfs/{}?recursive=true", root_cid)) - .build() - .unwrap(); - let req = hyper::Request::builder() - .method("GET") - .header("accept", "application/vnd.ipld.car") - .uri(uri) - .body(hyper::Body::empty()) - .unwrap(); - client.request(req).await.unwrap() - }; + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + "/?recursive=true", + Some(&[ + ("accept", "application/vnd.ipld.car"), + ("host", &format!("{}.ipfs.localhost", test_setup.root_cid)), + ]), + ) + .await; assert_eq!(http::StatusCode::OK, res.status()); @@ -302,87 +427,200 @@ mod tests { .await; // match cids and content - assert_eq!(cids.len(), all_cids.len()); + assert_eq!(cids.len(), test_setup.file_cids.len()); assert_eq!( HashSet::<_>::from_iter(cids.iter()), - HashSet::from_iter(all_cids.iter()) + HashSet::from_iter(test_setup.file_cids.iter()) ); - assert_eq!(cids[0], root_cid); - assert_eq!(nodes.len(), files.len() + 1); + assert_eq!(cids[0], test_setup.root_cid); + assert_eq!(nodes.len(), test_setup.files.len() + 1); assert!(nodes[0].is_dir()); assert_eq!( nodes[0] .links() .map(|link| link.unwrap().name.unwrap().to_string()) .collect::>(), - files + test_setup + .files .iter() .map(|(name, _content)| name.to_string()) .collect::>() ); for (i, node) in nodes[1..].iter().enumerate() { - assert_eq!(node, &UnixfsNode::Raw(files[i].1.clone().into())); + assert_eq!(node, &UnixfsNode::Raw(test_setup.files[i].1.clone().into())); } - core_task.abort(); - core_task.await.unwrap_err(); - store_task.abort(); - store_task.await.unwrap_err(); + test_setup.shutdown().await } #[tokio::test] - async fn test_head_request_to_file() { - let (store_client_addr, store_task) = spawn_store().await; - let mut config = Config::new( - 0, - RpcClientConfig { - gateway_addr: None, - p2p_addr: None, - store_addr: Some(store_client_addr), - channels: Some(1), - }, + async fn test_gateway_redirection() { + let test_setup = setup_test(true).await; + + // Usual request + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/{}", test_setup.root_cid, "world.txt"), + None, + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://{}.ipfs.localhost:{}/world.txt", + test_setup.root_cid, + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap(), ); - config.set_default_headers(); - let (addr, rpc_client, core_task) = spawn_gateway(Arc::new(config)).await; + // No trailing slash + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}", test_setup.root_cid), + None, + ) + .await; - let dir = "demo"; - let files = [ - ("hello.txt", b"ola".to_vec()), - ("world.txt", b"mundo".to_vec()), - ]; + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://{}.ipfs.localhost:{}/", + test_setup.root_cid, + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); - // add a directory with two files to the store. - let (root_cid, _) = put_directory_with_files(&rpc_client, dir, &files).await; + // Trailing slash + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/", test_setup.root_cid), + None, + ) + .await; - // request the root cid as a recursive car - let res = { - let client = hyper::Client::new(); - let uri = hyper::Uri::builder() - .scheme("http") - .authority(format!("localhost:{}", addr.port())) - .path_and_query(format!("/ipfs/{}/{}", root_cid, "world.txt")) - .build() - .unwrap(); - let req = hyper::Request::builder() - .method("HEAD") - .uri(uri) - .body(hyper::Body::empty()) - .unwrap(); - client.request(req).await.unwrap() - }; + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://{}.ipfs.localhost:{}/", + test_setup.root_cid, + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); - assert_eq!(http::StatusCode::OK, res.status()); - assert!(res.headers().get("content-length").is_some()); - assert_eq!(res.headers().get("content-length").unwrap(), "5"); + // IPNS + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8", + None, + ) + .await; - let (body, _) = res.into_body().into_future().await; - assert!(body.is_none()); + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8.ipns.localhost:{}/", + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); - core_task.abort(); - core_task.await.unwrap_err(); - store_task.abort(); - store_task.await.unwrap_err(); + // Test that IPNS records are recoded to base36 + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + "/ipns/bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm", + None, + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://k2jvslbl2n1p4suo7yr973y3v7pfautpxba2jeb9fpmjil0l3lppcgi3.ipns.localhost:{}/", + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); + + // IPNS + DNSLink + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + "/ipns/en.wikipedia-on-ipfs.org", + None, + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://en-wikipedia--on--ipfs-org.ipns.localhost:{}/", + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); + + // IPNS + DNSLink + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + "/ipns/12D3KooWJHxkQKX8C5KAyqEPhn2ssT2in4TExyG9SXxi519tycL9", + None, + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "http://k51qzi5uqu5djbl2zsl8ooauuh7wb1ycesq93g72iym71shji1pbntl1vuyuk2.ipns.localhost:{}/", + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap() + ); + + // X-Forwarded-Proto + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/{}", test_setup.root_cid, "world.txt"), + Some(&[("x-forwarded-proto", "https")]), + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!( + "https://{}.ipfs.localhost:{}/world.txt", + test_setup.root_cid, + test_setup.gateway_addr.port() + ), + res.headers().get("Location").unwrap().to_str().unwrap(), + ); + + // X-Forwarded-Host + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/{}", test_setup.root_cid, "world.txt"), + Some(&[("x-forwarded-host", "ipfs.io")]), + ) + .await; + + assert_eq!(http::StatusCode::MOVED_PERMANENTLY, res.status()); + assert_eq!( + format!("http://{}.ipfs.ipfs.io/world.txt", test_setup.root_cid,), + res.headers().get("Location").unwrap().to_str().unwrap(), + ); + + test_setup.shutdown().await } } diff --git a/iroh-gateway/src/handler_params.rs b/iroh-gateway/src/handler_params.rs new file mode 100644 index 0000000000..9a78ecdd6b --- /dev/null +++ b/iroh-gateway/src/handler_params.rs @@ -0,0 +1,127 @@ +use cid::multibase; +use cid::multibase::Base; +use iroh_resolver::resolver::{CidOrDomain, Path, PathType}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct GetParams { + /// specifies the expected format of the response + pub format: Option, + /// specifies the desired filename of the response + pub filename: Option, + /// specifies whether the response should be of disposition inline or attachment + pub download: Option, + /// specifies whether the response should render a directory even if index.html is present + pub force_dir: Option, + /// uri query parameter for handling navigator.registerProtocolHandler Web API requests + pub uri: Option, + pub recursive: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct DefaultHandlerPathParams { + pub scheme: String, + pub cid_or_domain: String, + pub content_path: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct SubdomainHandlerPathParams { + pub content_path: Option, +} + +impl GetParams { + pub fn to_query_string(&self) -> String { + let q = serde_qs::to_string(self).unwrap(); + if q.is_empty() { + q + } else { + format!("?{}", q) + } + } +} + +pub fn recode_path_to_inlined_dns_link(path: &Path) -> String { + match path.root() { + CidOrDomain::Cid(cid) => match path.typ() { + PathType::Ipfs => cid.to_string(), + PathType::Ipns => multibase::encode(Base::Base36Lower, cid.to_bytes().as_slice()), + }, + CidOrDomain::Domain(domain) => domain.replace('-', "--").replace('.', "-"), + } +} + +pub fn inlined_dns_link_to_dns_link(dns_link: &str) -> String { + let dns_link = dns_link.chars().collect::>(); + // first char + mapping that replaces standalone dashes + last char + dns_link + .iter() + .take(1) + .chain( + dns_link + .iter() + .enumerate() + .skip(1) + .take(dns_link.len() - 2) + .map(|(i, ch)| { + if *ch == '-' && dns_link[i - 1] != '-' && dns_link[i + 1] != '-' { + &'.' + } else { + ch + } + }) + .chain(dns_link.iter().last()), + ) + .collect::() + .replace("--", "-") +} + +#[cfg(test)] +mod tests { + use iroh_resolver::resolver::Path; + + use crate::handler_params::inlined_dns_link_to_dns_link; + use crate::handler_params::recode_path_to_inlined_dns_link; + + fn just_domain(domain: &str) -> Path { + Path::from_parts("ipns", domain, "").unwrap() + } + + #[test] + fn test_dns_link_to_inlined_dns_link() { + assert_eq!( + recode_path_to_inlined_dns_link(&just_domain("google.com")), + "google-com", + ); + assert_eq!( + recode_path_to_inlined_dns_link(&just_domain("goo-gle.com")), + "goo--gle-com", + ); + assert_eq!( + recode_path_to_inlined_dns_link(&just_domain("goog-l.e.com")), + "goog--l-e-com", + ); + } + + #[test] + fn test_inlined_dns_link_to_dns_link() { + assert_eq!( + inlined_dns_link_to_dns_link(&recode_path_to_inlined_dns_link(&just_domain( + "google.com" + ))), + "google.com", + ); + assert_eq!( + inlined_dns_link_to_dns_link(&recode_path_to_inlined_dns_link(&just_domain( + "goo-gle.com" + ))), + "goo-gle.com", + ); + assert_eq!( + inlined_dns_link_to_dns_link(&recode_path_to_inlined_dns_link(&just_domain( + "goog-l.e.com" + ))), + "goog-l.e.com", + ); + } +} diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index 84a21231c1..901519b45b 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -1,14 +1,17 @@ use async_recursion::async_recursion; +use axum::extract::Host; +use axum::routing::any; use axum::{ body::Body, error_handling::HandleErrorLayer, - extract::{Extension, Path, Query}, + extract::{Extension, Path as AxumPath, Query}, http::{header::*, Request as HttpRequest, StatusCode}, middleware, response::IntoResponse, routing::{get, head}, BoxError, Router, }; +use cid::Cid; use futures::TryStreamExt; use handlebars::Handlebars; use http::Method; @@ -16,12 +19,10 @@ use iroh_metrics::{core::MRecorder, gateway::GatewayMetrics, inc, resolver::OutM use iroh_resolver::resolver::{CidOrDomain, UnixfsType}; use iroh_unixfs::{content_loader::ContentLoader, Link}; use iroh_util::human::format_bytes; -use serde::{Deserialize, Serialize}; use serde_json::{ json, value::{Map, Value as Json}, }; -use serde_qs; use std::{ collections::HashMap, fmt::Write, @@ -30,14 +31,20 @@ use std::{ time::{self, Duration}, }; -use tower::ServiceBuilder; +use iroh_resolver::Path; +use tower::{ServiceBuilder, ServiceExt}; use tower_http::{compression::CompressionLayer, trace::TraceLayer}; use tracing::info_span; use url::Url; use urlencoding::encode; +use crate::handler_params::{ + inlined_dns_link_to_dns_link, recode_path_to_inlined_dns_link, DefaultHandlerPathParams, + GetParams, SubdomainHandlerPathParams, +}; +use crate::text::IpfsSubdomain; use crate::{ - client::{FileResult, Request}, + client::{FileResult, IpfsRequest}, constants::*, core::State, error::GatewayError, @@ -46,6 +53,11 @@ use crate::{ templates::{icon_class_name, ICONS_STYLESHEET, STYLESHEET}, }; +enum RequestPreprocessingResult { + RespondImmediately(GatewayResponse), + ShouldRequestData(Box), +} + /// Trait describing what needs to be accessed on the configuration /// from the shared state. pub trait StateConfig: std::fmt::Debug + Sync + Send { @@ -53,20 +65,44 @@ pub trait StateConfig: std::fmt::Debug + Sync + Send { fn public_url_base(&self) -> &str; fn port(&self) -> u16; fn user_headers(&self) -> &HeaderMap; + fn redirect_to_subdomain(&self) -> bool; } -pub fn get_app_routes(state: &Arc>) -> Router { +pub fn get_app_routes(state: &Arc>) -> Router { let cors = crate::cors::cors_from_headers(state.config.user_headers()); // todo(arqu): ?uri=... https://github.com/ipfs/go-ipfs/pull/7802 - Router::new() - .route("/:scheme/:cid", get(get_handler::)) - .route("/:scheme/:cid/*cpath", get(get_handler::)) - .route("/:scheme/:cid/*cpath", head(head_handler::)) + let path_router = Router::new() + .route("/:scheme/:cid_or_domain", get(path_handler::)) + .route( + "/:scheme/:cid_or_domain/*content_path", + get(path_handler::), + ) + .route( + "/:scheme/:cid_or_domain/*content_path", + head(path_handler::), + ) .route("/health", get(health_check)) .route("/icons.css", get(stylesheet_icons)) .route("/style.css", get(stylesheet_main)) - .route("/info", get(info)) + .route("/info", get(info)); + + let subdomain_router = Router::new() + .route("/*content_path", get(subdomain_handler::)) + .route("/*content_path", head(subdomain_handler::)); + + Router::new() + .route( + "/*path", + any( + |Host(hostname): Host, request: hyper::Request| async move { + match IpfsSubdomain::try_from_str(&hostname) { + Some(_) => subdomain_router.oneshot(request).await, + None => path_router.oneshot(request).await, + } + }, + ), + ) .layer(cors) .layer(Extension(Arc::clone(state))) .layer( @@ -93,109 +129,60 @@ pub fn get_app_routes(state: &Arc, - /// specifies the desired filename of the response - filename: Option, - /// specifies whether the response should be of disposition inline or attachment - download: Option, - /// specifies whether the response should render a directory even if index.html is present - force_dir: Option, - /// uri query parameter for handling navigator.registerProtocolHandler Web API requests - uri: Option, - recursive: Option, -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -pub struct PathParams { - scheme: String, - cid: String, - cpath: Option, -} - -impl GetParams { - pub fn to_query_string(&self) -> String { - let q = serde_qs::to_string(self).unwrap(); - if q.is_empty() { - q - } else { - format!("?{}", q) - } - } -} - -enum RequestPreprocessingResult { - Response(GatewayResponse), - FurtherRequest(Box), -} - -async fn request_preprocessing( +async fn request_preprocessing( state: &Arc>, - path_params: &PathParams, + path: &Path, query_params: &GetParams, - request_headers: HeaderMap, + request_headers: &HeaderMap, response_headers: &mut HeaderMap, + subdomain_mode: bool, ) -> Result { - if path_params.scheme != SCHEME_IPFS && path_params.scheme != SCHEME_IPNS { + if path.typ().as_str() != SCHEME_IPFS && path.typ().as_str() != SCHEME_IPNS { return Err(GatewayError::new( StatusCode::BAD_REQUEST, "invalid scheme, must be ipfs or ipns", )); } - let cpath = path_params.cpath.as_deref().unwrap_or(""); + let content_path = path.to_relative_string(); let uri_param = query_params.uri.clone().unwrap_or_default(); if !uri_param.is_empty() { - return protocol_handler_redirect(uri_param, state) - .map(RequestPreprocessingResult::Response); + return protocol_handler_redirect(uri_param) + .map(RequestPreprocessingResult::RespondImmediately); } - service_worker_check(&request_headers, cpath.to_string(), state)?; - unsuported_header_check(&request_headers, state)?; + service_worker_check(request_headers, &content_path)?; + unsupported_header_check(request_headers)?; - if check_bad_bits(state, &path_params.cid, cpath).await { - return Err(GatewayError::new( - StatusCode::GONE, - "CID is in the denylist", - )); + if let Some(cid) = path.cid() { + if check_bad_bits(state, cid, &content_path).await { + return Err(GatewayError::new( + StatusCode::GONE, + "CID is in the denylist", + )); + } } - let full_content_path = format!("/{}/{}{}", path_params.scheme, path_params.cid, cpath); - let resolved_path: iroh_resolver::resolver::Path = full_content_path - .parse() - .map_err(|e: anyhow::Error| e.to_string()) - .map_err(|e| GatewayError::new(StatusCode::BAD_REQUEST, &e))?; // TODO: handle 404 or error - let resolved_cid = resolved_path.root(); - - if handle_only_if_cached(&request_headers, state, resolved_cid).await? { - return Ok(RequestPreprocessingResult::Response(GatewayResponse::new( - StatusCode::OK, - Body::empty(), - HeaderMap::new(), - ))); - } + let resolved_cid = path.root(); - if check_bad_bits(state, resolved_cid.to_string().as_str(), cpath).await { - return Err(GatewayError::new( - StatusCode::GONE, - "CID is in the denylist", + if handle_only_if_cached(request_headers, state, path.root()).await? { + return Ok(RequestPreprocessingResult::RespondImmediately( + GatewayResponse::new(StatusCode::OK, Body::empty(), HeaderMap::new()), )); } // parse query params - let format = get_response_format(&request_headers, &query_params.format) + let format = get_response_format(request_headers, &query_params.format) .map_err(|err| GatewayError::new(StatusCode::BAD_REQUEST, &err))?; - if let Some(resp) = etag_check(&request_headers, resolved_cid, &format, state) { - return Ok(RequestPreprocessingResult::Response(resp)); + if let Some(resp) = etag_check(request_headers, resolved_cid, &format) { + return Ok(RequestPreprocessingResult::RespondImmediately(resp)); } // init headers format.write_headers(response_headers); add_user_headers(response_headers, state.config.user_headers().clone()); - let hv = match HeaderValue::from_str(&full_content_path) { + let hv = match HeaderValue::from_str(&path.to_string()) { Ok(hv) => hv, Err(err) => { return Err(GatewayError::new( @@ -207,10 +194,10 @@ async fn request_preprocessing( response_headers.insert(&HEADER_X_IPFS_PATH, hv); // handle request and fetch data - let req = Request { + let req = IpfsRequest { format, - cid: resolved_path.root().clone(), - resolved_path, + cid: path.root().clone(), + resolved_path: path.clone(), query_file_name: query_params .filename .as_deref() @@ -218,79 +205,129 @@ async fn request_preprocessing( .to_string(), download: query_params.download.unwrap_or_default(), query_params: query_params.clone(), + subdomain_mode, }; - Ok(RequestPreprocessingResult::FurtherRequest(Box::new(req))) + Ok(RequestPreprocessingResult::ShouldRequestData(Box::new(req))) } -#[tracing::instrument(skip(state))] -pub async fn get_handler( - Extension(state): Extension>>, - Path(path_params): Path, - Query(query_params): Query, +pub async fn handler( + state: Arc>, + method: Method, + path: &Path, + query_params: &GetParams, + request_headers: &HeaderMap, http_req: HttpRequest, - request_headers: HeaderMap, + subdomain_mode: bool, ) -> Result { - inc!(GatewayMetrics::Requests); let start_time = time::Instant::now(); let mut response_headers = HeaderMap::new(); match request_preprocessing( &state, - &path_params, - &query_params, + path, + query_params, request_headers, &mut response_headers, + subdomain_mode, ) .await? { - RequestPreprocessingResult::Response(gateway_response) => Ok(gateway_response), - RequestPreprocessingResult::FurtherRequest(req) => { - if query_params.recursive.unwrap_or_default() { - serve_car_recursive(&req, state, response_headers, start_time).await - } else { - match req.format { - ResponseFormat::Raw => { - serve_raw(&req, state, response_headers, &http_req, start_time).await - } - ResponseFormat::Car => { - serve_car(&req, state, response_headers, start_time).await - } - ResponseFormat::Fs(_) => { - serve_fs(&req, state, response_headers, &http_req, start_time).await + RequestPreprocessingResult::RespondImmediately(gateway_response) => Ok(gateway_response), + RequestPreprocessingResult::ShouldRequestData(req) => match method { + Method::HEAD => { + let path_metadata = state + .client + .retrieve_path_metadata(req.resolved_path) + .await + .map_err(|e| GatewayError::new(StatusCode::INTERNAL_SERVER_ERROR, &e))?; + add_content_length_header(&mut response_headers, path_metadata.metadata().clone()); + Ok(GatewayResponse::empty(response_headers)) + } + Method::GET => { + if query_params.recursive.unwrap_or_default() { + serve_car_recursive(&req, state, response_headers, start_time).await + } else { + match req.format { + ResponseFormat::Raw => { + serve_raw(&req, state, response_headers, &http_req, start_time).await + } + ResponseFormat::Car => { + serve_car(&req, state, response_headers, start_time).await + } + ResponseFormat::Fs(_) => { + serve_fs(&req, state, response_headers, &http_req, start_time).await + } } } } - } + _ => Err(GatewayError::new( + StatusCode::METHOD_NOT_ALLOWED, + "method not allowed", + )), + }, } } #[tracing::instrument(skip(state))] -pub async fn head_handler( +pub async fn subdomain_handler( Extension(state): Extension>>, - Path(path_params): Path, + method: Method, + Host(host): Host, + AxumPath(path_params): AxumPath, Query(query_params): Query, request_headers: HeaderMap, + http_req: HttpRequest, ) -> Result { inc!(GatewayMetrics::Requests); - let mut response_headers = HeaderMap::new(); - match request_preprocessing( - &state, - &path_params, + let parsed_subdomain_url = IpfsSubdomain::try_from_str(&host) + .ok_or_else(|| GatewayError::new(StatusCode::BAD_REQUEST, "hostname is not compliant"))?; + let path = Path::from_parts( + parsed_subdomain_url.scheme, + &inlined_dns_link_to_dns_link(parsed_subdomain_url.cid_or_domain), + path_params.content_path.as_deref().unwrap_or(""), + ) + .map_err(|e| GatewayError::new(StatusCode::BAD_REQUEST, &e.to_string()))?; + handler( + state, + method, + &path, &query_params, - request_headers, - &mut response_headers, + &request_headers, + http_req, + true, ) - .await? - { - RequestPreprocessingResult::Response(gateway_response) => Ok(gateway_response), - RequestPreprocessingResult::FurtherRequest(req) => { - let path_metadata = state - .client - .retrieve_path_metadata(req.resolved_path) - .await - .map_err(|e| GatewayError::new(StatusCode::INTERNAL_SERVER_ERROR, &e))?; - add_content_length_header(&mut response_headers, path_metadata.metadata().clone()); - Ok(GatewayResponse::empty(response_headers)) - } + .await +} + +#[tracing::instrument(skip(state))] +pub async fn path_handler( + Extension(state): Extension>>, + method: Method, + Host(host): Host, + AxumPath(path_params): AxumPath, + Query(query_params): Query, + request_headers: HeaderMap, + http_req: HttpRequest, +) -> Result { + inc!(GatewayMetrics::Requests); + let path = Path::from_parts( + &path_params.scheme, + &path_params.cid_or_domain, + path_params.content_path.as_deref().unwrap_or(""), + ) + .map_err(|e| GatewayError::new(StatusCode::BAD_REQUEST, &e.to_string()))?; + if state.config.redirect_to_subdomain() { + Ok(redirect_path_handlers(&host, &path, &request_headers)) + } else { + handler( + state, + method, + &path, + &query_params, + &request_headers, + http_req, + false, + ) + .await } } @@ -336,10 +373,7 @@ pub async fn stylesheet_icons() -> (HeaderMap, &'static str) { } #[tracing::instrument()] -fn protocol_handler_redirect( - uri_param: String, - state: &State, -) -> Result { +fn protocol_handler_redirect(uri_param: String) -> Result { let u = match Url::parse(&uri_param) { Ok(u) => u, Err(e) => { @@ -369,14 +403,13 @@ fn protocol_handler_redirect( } #[tracing::instrument()] -fn service_worker_check( +fn service_worker_check( request_headers: &HeaderMap, - cpath: String, - state: &State, + content_path: &str, ) -> Result<(), GatewayError> { if request_headers.contains_key(&HEADER_SERVICE_WORKER) { let sw = request_headers.get(&HEADER_SERVICE_WORKER).unwrap(); - if sw.to_str().unwrap() == "script" && cpath.is_empty() { + if sw.to_str().unwrap() == "script" && content_path.is_empty() { return Err(GatewayError::new( StatusCode::BAD_REQUEST, "Service Worker not supported", @@ -387,10 +420,7 @@ fn service_worker_check( } #[tracing::instrument()] -fn unsuported_header_check( - request_headers: &HeaderMap, - state: &State, -) -> Result<(), GatewayError> { +fn unsupported_header_check(request_headers: &HeaderMap) -> Result<(), GatewayError> { if request_headers.contains_key(&HEADER_X_IPFS_GATEWAY_PREFIX) { return Err(GatewayError::new( StatusCode::BAD_REQUEST, @@ -400,6 +430,27 @@ fn unsuported_header_check( Ok(()) } +#[tracing::instrument()] +fn redirect_path_handlers(host: &str, path: &Path, request_headers: &HeaderMap) -> GatewayResponse { + let target_host = request_headers + .get(&HEADER_X_FORWARDED_HOST) + .map(|hv| hv.to_str().unwrap()) + .unwrap_or(host); + let target_proto = request_headers + .get(&HEADER_X_FORWARDED_PROTO) + .map(|hv| hv.to_str().unwrap()) + .unwrap_or("http"); + let content_path = path.to_relative_string(); + GatewayResponse::redirect_permanently(&format!( + "{}://{}.{}.{}/{}", + target_proto, + recode_path_to_inlined_dns_link(path), + path.typ().as_str(), + target_host, + content_path.strip_prefix('/').unwrap_or(&content_path) + )) +} + #[tracing::instrument()] async fn handle_only_if_cached( request_headers: &HeaderMap, @@ -432,12 +483,16 @@ async fn handle_only_if_cached( Ok(false) } -pub async fn check_bad_bits(state: &State, cid: &str, path: &str) -> bool { +pub async fn check_bad_bits( + state: &State, + cid: &Cid, + content_path: &str, +) -> bool { // check if cid is in the denylist if state.bad_bits.is_some() { let bad_bits = state.bad_bits.as_ref(); if let Some(bbits) = bad_bits { - if bbits.read().await.is_bad(cid, path) { + if bbits.read().await.is_bad(cid, content_path) { return true; } } @@ -446,11 +501,10 @@ pub async fn check_bad_bits(state: &State, cid: &str, path: } #[tracing::instrument()] -fn etag_check( +fn etag_check( request_headers: &HeaderMap, resolved_cid: &CidOrDomain, format: &ResponseFormat, - state: &State, ) -> Option { if request_headers.contains_key("If-None-Match") { let inm = request_headers @@ -471,12 +525,12 @@ fn etag_check( } #[tracing::instrument()] -async fn serve_raw( - req: &Request, +async fn serve_raw( + req: &IpfsRequest, state: Arc>, mut headers: HeaderMap, http_req: &HttpRequest, - start_time: std::time::Instant, + start_time: time::Instant, ) -> Result { let range: Option> = if http_req.headers().contains_key(RANGE) { parse_range_header(http_req.headers().get(RANGE).unwrap()) @@ -499,7 +553,7 @@ async fn serve_raw( set_content_disposition_headers(&mut headers, &file_name, DISPOSITION_ATTACHMENT); set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } add_cache_control_headers(&mut headers, metadata.clone()); @@ -529,11 +583,11 @@ async fn serve_raw( } #[tracing::instrument()] -async fn serve_car( - req: &Request, +async fn serve_car( + req: &IpfsRequest, state: Arc>, mut headers: HeaderMap, - start_time: std::time::Instant, + start_time: time::Instant, ) -> Result { // TODO: handle car versions // FIXME: we currently only retrieve full cids @@ -556,7 +610,7 @@ async fn serve_car( add_content_length_header(&mut headers, metadata.clone()); let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); set_etag_headers(&mut headers, etag); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } add_ipfs_roots_headers(&mut headers, metadata); @@ -570,11 +624,11 @@ async fn serve_car( } #[tracing::instrument()] -async fn serve_car_recursive( - req: &Request, +async fn serve_car_recursive( + req: &IpfsRequest, state: Arc>, mut headers: HeaderMap, - start_time: std::time::Instant, + start_time: time::Instant, ) -> Result { let body = state .client @@ -593,7 +647,7 @@ async fn serve_car_recursive( // add_cache_control_headers(&mut headers, metadata.clone()); let etag = format!("W/{}", get_etag(&req.cid, Some(req.format.clone()))); set_etag_headers(&mut headers, etag); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } // add_ipfs_roots_headers(&mut headers, metadata); @@ -602,26 +656,24 @@ async fn serve_car_recursive( #[tracing::instrument()] #[async_recursion] -async fn serve_fs( - req: &Request, +async fn serve_fs( + req: &IpfsRequest, state: Arc>, mut headers: HeaderMap, http_req: &HttpRequest, - start_time: std::time::Instant, + start_time: time::Instant, ) -> Result { let range: Option> = if http_req.headers().contains_key(RANGE) { parse_range_header(http_req.headers().get(RANGE).unwrap()) } else { None }; - // FIXME: we currently only retrieve full cids let (body, metadata) = state .client .get_file(req.resolved_path.clone(), start_time, range.clone()) .await .map_err(|e| GatewayError::new(StatusCode::INTERNAL_SERVER_ERROR, &e))?; - add_ipfs_roots_headers(&mut headers, metadata.clone()); match body { FileResult::Directory(res) => { @@ -652,7 +704,7 @@ async fn serve_fs( add_cache_control_headers(&mut headers, metadata.clone()); add_content_length_header(&mut headers, metadata.clone()); set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } let name = add_content_disposition_headers( @@ -670,7 +722,6 @@ async fn serve_fs( let content_sniffed_mime = body.get_mime(); add_content_type_headers(&mut headers, &name, content_sniffed_mime); } - if let Some(mut capped_range) = range { if let Some(size) = metadata.size { capped_range.end = std::cmp::min(capped_range.end, size); @@ -698,7 +749,7 @@ async fn serve_fs( add_cache_control_headers(&mut headers, metadata.clone()); add_content_length_header(&mut headers, metadata.clone()); set_etag_headers(&mut headers, get_etag(&req.cid, Some(req.format.clone()))); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } let name = add_content_disposition_headers( @@ -715,9 +766,9 @@ async fn serve_fs( } #[tracing::instrument()] -async fn serve_fs_dir( +async fn serve_fs_dir( dir_list: &[Link], - req: &Request, + req: &IpfsRequest, state: Arc>, mut headers: HeaderMap, http_req: &HttpRequest, @@ -734,7 +785,7 @@ async fn serve_fs_dir( if !req.resolved_path.has_trailing_slash() { let redirect_path = format!( "{}/{}", - req.resolved_path, + req.request_path_for_redirection(), req.query_params.to_query_string() ); return Ok(GatewayResponse::redirect_permanently(&redirect_path)); @@ -746,7 +797,7 @@ async fn serve_fs_dir( headers.insert(CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap()); set_etag_headers(&mut headers, get_dir_etag(&req.cid)); - if let Some(res) = etag_check(&headers, &req.cid, &req.format, &state) { + if let Some(res) = etag_check(&headers, &req.cid, &req.format) { return Ok(res); } @@ -823,8 +874,8 @@ async fn serve_fs_dir( // #[tracing::instrument()] pub async fn request_middleware( - request: axum::http::Request, - next: axum::middleware::Next, + request: http::Request, + next: middleware::Next, ) -> axum::response::Response { let method = request.method().clone(); let mut r = next.run(request).await; @@ -838,7 +889,6 @@ pub async fn request_middleware( #[tracing::instrument()] pub async fn middleware_error_handler( method: Method, - Extension(state): Extension>>, err: BoxError, ) -> impl IntoResponse { inc!(GatewayMetrics::FailCount); diff --git a/iroh-gateway/src/lib.rs b/iroh-gateway/src/lib.rs index dd5e336f0a..2fe3249715 100644 --- a/iroh-gateway/src/lib.rs +++ b/iroh-gateway/src/lib.rs @@ -6,9 +6,11 @@ pub mod constants; pub mod core; mod cors; mod error; +pub mod handler_params; pub mod handlers; pub mod headers; pub mod metrics; pub mod response; mod rpc; pub mod templates; +mod text; diff --git a/iroh-gateway/src/text.rs b/iroh-gateway/src/text.rs new file mode 100644 index 0000000000..c8db3013df --- /dev/null +++ b/iroh-gateway/src/text.rs @@ -0,0 +1,70 @@ +pub struct IpfsSubdomain<'a> { + pub cid_or_domain: &'a str, + pub scheme: &'a str, + pub hostname: &'a str, +} + +impl<'a> IpfsSubdomain<'a> { + pub(crate) fn try_from_str(value: &'a str) -> Option { + let mut value = value.splitn(3, '.'); + if let (Some(cid_or_domain), Some(schema), Some(hostname)) = + (value.next(), value.next(), value.next()) + { + if schema == "ipns" || schema == "ipfs" { + return Some(IpfsSubdomain { + cid_or_domain, + scheme: schema, + hostname, + }); + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn response_format_try_from() { + assert!(IpfsSubdomain::try_from_str("localhost:8080").is_none()); + assert!(IpfsSubdomain::try_from_str("localhost").is_none()); + assert!(IpfsSubdomain::try_from_str("ipfs.localhost").is_none()); + assert!(IpfsSubdomain::try_from_str("bafy.ipfs.localhost").is_some()); + assert!(IpfsSubdomain::try_from_str("ipfs-eth.ipns.localhost").is_some()); + assert!(IpfsSubdomain::try_from_str("bafy.ipfs.localhost:8080").is_some()); + assert!(IpfsSubdomain::try_from_str("bafy.ipnotfs.localhost").is_none()); + assert!(IpfsSubdomain::try_from_str("bafy.ipnotfs.localhost").is_none()); + + let complex_case_1 = IpfsSubdomain::try_from_str("bafy.ipfs.bafy.ipfs.com").unwrap(); + assert_eq!(complex_case_1.cid_or_domain, "bafy"); + assert_eq!(complex_case_1.scheme, "ipfs"); + assert_eq!(complex_case_1.hostname, "bafy.ipfs.com"); + + let complex_case_2 = IpfsSubdomain::try_from_str("bafy.ipfs.ipfs.com").unwrap(); + assert_eq!(complex_case_2.cid_or_domain, "bafy"); + assert_eq!(complex_case_2.scheme, "ipfs"); + assert_eq!(complex_case_2.hostname, "ipfs.com"); + + let complex_case_3 = IpfsSubdomain::try_from_str("bafy.ipfs.ipns.com").unwrap(); + assert_eq!(complex_case_3.cid_or_domain, "bafy"); + assert_eq!(complex_case_3.scheme, "ipfs"); + assert_eq!(complex_case_3.hostname, "ipns.com"); + + let complex_case_4 = IpfsSubdomain::try_from_str("bafy.ipns.ipfs.com").unwrap(); + assert_eq!(complex_case_4.cid_or_domain, "bafy"); + assert_eq!(complex_case_4.scheme, "ipns"); + assert_eq!(complex_case_4.hostname, "ipfs.com"); + + let complex_case_5 = IpfsSubdomain::try_from_str("bafy.ipns.ipns.com").unwrap(); + assert_eq!(complex_case_5.cid_or_domain, "bafy"); + assert_eq!(complex_case_5.scheme, "ipns"); + assert_eq!(complex_case_5.hostname, "ipns.com"); + + let complex_case_6 = IpfsSubdomain::try_from_str("bafy-mafy.ipfs.ipfs.com").unwrap(); + assert_eq!(complex_case_6.cid_or_domain, "bafy-mafy"); + assert_eq!(complex_case_6.scheme, "ipfs"); + assert_eq!(complex_case_6.hostname, "ipfs.com"); + } +} diff --git a/iroh-one/src/config.rs b/iroh-one/src/config.rs index bac1fad6b9..ced5c2b5d5 100644 --- a/iroh-one/src/config.rs +++ b/iroh-one/src/config.rs @@ -173,4 +173,8 @@ impl iroh_gateway::handlers::StateConfig for Config { fn user_headers(&self) -> &HeaderMap { &self.gateway.headers } + + fn redirect_to_subdomain(&self) -> bool { + self.gateway.redirect_to_subdomain + } } diff --git a/iroh-resolver/Cargo.toml b/iroh-resolver/Cargo.toml index c6fe0f3913..4c1f5b5e48 100644 --- a/iroh-resolver/Cargo.toml +++ b/iroh-resolver/Cargo.toml @@ -18,6 +18,7 @@ anyhow = { version = "1", features = ["backtrace"] } async-channel = "1.7.1" async-stream = "0.3.3" async-trait = "0.1.53" +bs58 = "0.4.0" bytes = "1.1.0" cid = "0.9" futures = "0.3.21" @@ -26,6 +27,7 @@ iroh-rpc-client = { version = "0.1.3", path = "../iroh-rpc-client", default-feat iroh-util = { version = "0.1.3", path = "../iroh-util", default-features = false } iroh-unixfs = { version = "0.1.3", path = "../iroh-unixfs" } libipld = "0.15.0" +libp2p = "0.50" serde = { version = "1", features = ["derive"] } tokio = { version = "1", features = ["fs"] } tracing = "0.1.34" diff --git a/iroh-resolver/src/resolver.rs b/iroh-resolver/src/resolver.rs index 846ceeda37..aa7e248e80 100644 --- a/iroh-resolver/src/resolver.rs +++ b/iroh-resolver/src/resolver.rs @@ -35,6 +35,25 @@ use crate::dns_resolver::{Config, DnsResolver}; pub const IROH_STORE: &str = "iroh-store"; +// ToDo: Remove this function +// Related issue: https://github.com/n0-computer/iroh/issues/593 +fn from_peer_id(id: &str) -> Option { + static MAX_INLINE_KEY_LENGTH: usize = 42; + let multihash = + libp2p::multihash::Multihash::from_bytes(&bs58::decode(id).into_vec().ok()?).ok()?; + match libp2p::multihash::Code::try_from(multihash.code()) { + Ok(libp2p::multihash::Code::Sha2_256) => { + Some(libipld::Multihash::from_bytes(&multihash.to_bytes()).unwrap()) + } + Ok(libp2p::multihash::Code::Identity) + if multihash.digest().len() <= MAX_INLINE_KEY_LENGTH => + { + Some(libipld::Multihash::from_bytes(&multihash.to_bytes()).unwrap()) + } + _ => None, + } +} + /// Represents an ipfs path. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Path { @@ -52,6 +71,40 @@ impl Path { } } + pub fn from_parts( + scheme: &str, + cid_or_domain: &str, + tail_path: &str, + ) -> Result { + let (typ, root) = if scheme.eq_ignore_ascii_case("ipns") { + let root = if let Ok(cid) = Cid::from_str(cid_or_domain) { + CidOrDomain::Cid(cid) + } else if let Some(multihash) = from_peer_id(cid_or_domain) { + CidOrDomain::Cid(Cid::new_v1(Codec::Libp2pKey.into(), multihash)) + // ToDo: Bring back commented "else if" instead of "else if" above + // Related issue: https://github.com/n0-computer/iroh/issues/593 + // } else if let Ok(peer_id) = PeerId::from_str(cid_or_domain) { + // CidOrDomain::Cid(Cid::new_v1(Codec::Libp2pKey.into(), *peer_id.as_ref())) + } else { + CidOrDomain::Domain(cid_or_domain.to_string()) + }; + (PathType::Ipns, root) + } else { + let root = Cid::from_str(cid_or_domain).context("invalid cid")?; + (PathType::Ipfs, CidOrDomain::Cid(root)) + }; + let tail = if tail_path != "/" { + tail_path + .split(&['/', '\\']) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect() + } else { + vec!["".to_string()] + }; + Ok(Path { typ, root, tail }) + } + pub fn typ(&self) -> PathType { self.typ } @@ -144,6 +197,7 @@ impl PathType { impl FromStr for Path { type Err = anyhow::Error; + // ToDo: Replace it with from_parts (or vice verse) fn from_str(s: &str) -> Result { let mut parts = s.split(&['/', '\\']).filter(|s| !s.is_empty()); @@ -508,7 +562,7 @@ impl AsyncSeek for OutPrettyReader { bytes_reader.pos = i as usize; } std::io::SeekFrom::Current(pos) => { - let mut i = std::cmp::min(data_len as i64 - 1, pos_current as i64 + pos); + let mut i = std::cmp::min(data_len as i64 - 1, pos_current + pos); i = std::cmp::max(0, i); bytes_reader.pos = i as usize; } @@ -898,7 +952,7 @@ impl Resolver { async fn resolve_ipld_path( &self, _cid: Cid, - codec: libipld::IpldCodec, + codec: IpldCodec, root: Ipld, path: &[String], ctx: &mut LoaderContext, @@ -907,7 +961,7 @@ impl Resolver { let mut codec = codec; for part in path.iter().filter(|s| !s.is_empty()) { - if let libipld::Ipld::Link(c) = current { + if let Ipld::Link(c) = current { (codec, current) = self.load_ipld_link(c, ctx).await?; } if codec == IpldCodec::DagPb { @@ -936,7 +990,7 @@ impl Resolver { #[tracing::instrument(skip(self))] async fn load_ipld_link(&self, cid: Cid, ctx: &mut LoaderContext) -> Result<(IpldCodec, Ipld)> { - let codec: libipld::IpldCodec = cid.codec().try_into()?; + let codec: IpldCodec = cid.codec().try_into()?; // resolve link and update if we have encountered a link let loaded_cid = self.load_cid(&cid, ctx).await?; diff --git a/iroh-resolver/tests/unixfs.rs b/iroh-resolver/tests/unixfs.rs index 26f0d6dda3..78ed046fe7 100644 --- a/iroh-resolver/tests/unixfs.rs +++ b/iroh-resolver/tests/unixfs.rs @@ -54,7 +54,7 @@ async fn test_dagger_testdata() -> Result<()> { }, Param { degree: 174, - chunker: Chunker::Rabin(Box::new(chunker::Rabin::default())), + chunker: Chunker::Rabin(Box::default()), }, ]; diff --git a/iroh-rpc-types/src/addr.rs b/iroh-rpc-types/src/addr.rs index 4843a7d3b5..bb1c1dfc9d 100644 --- a/iroh-rpc-types/src/addr.rs +++ b/iroh-rpc-types/src/addr.rs @@ -49,8 +49,8 @@ impl Addr { impl Display for Addr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Addr::Irpc(addr) => write!(f, "irpc://{}", addr), - Addr::IrpcLookup(addr) => write!(f, "irpc://{}", addr), + Addr::Irpc(addr) => write!(f, "irpc://{addr}"), + Addr::IrpcLookup(addr) => write!(f, "irpc://{addr}"), Addr::Mem(_, _) => write!(f, "mem"), #[allow(unreachable_patterns)] _ => unreachable!(), diff --git a/iroh-store/src/store.rs b/iroh-store/src/store.rs index 5bf2234851..2726f9f7f4 100644 --- a/iroh-store/src/store.rs +++ b/iroh-store/src/store.rs @@ -658,7 +658,7 @@ impl<'a> ReadStore<'a> { let meta = rkyv::check_archived_root::(&meta) .map_err(|e| anyhow!("{:?}", e))?; let multihash = cid::multihash::Multihash::from_bytes(&meta.multihash)?; - let c = cid::Cid::new_v1(meta.codec, multihash); + let c = Cid::new_v1(meta.codec, multihash); links.push(c); } None => { diff --git a/iroh-unixfs/src/balanced_tree.rs b/iroh-unixfs/src/balanced_tree.rs index 35752a01fc..4b05fa41ff 100644 --- a/iroh-unixfs/src/balanced_tree.rs +++ b/iroh-unixfs/src/balanced_tree.rs @@ -167,7 +167,7 @@ fn create_unixfs_node_from_links(links: Vec<(Cid, LinkInfo)>) -> Result Self { - self.chunker = Chunker::Rabin(Box::new(chunker::Rabin::default())); + self.chunker = Chunker::Rabin(Box::default()); self } diff --git a/iroh-unixfs/src/hamt.rs b/iroh-unixfs/src/hamt.rs index 4f38358fb5..36377b2881 100644 --- a/iroh-unixfs/src/hamt.rs +++ b/iroh-unixfs/src/hamt.rs @@ -312,7 +312,7 @@ fn hash_key(key: &[u8]) -> [u8; HASH_BIT_LENGTH] { fn log2(x: u32) -> u32 { assert!(x > 0); - u32::BITS as u32 - x.leading_zeros() - 1 + u32::BITS - x.leading_zeros() - 1 } #[cfg(test)] diff --git a/iroh-unixfs/src/hamt/bitfield.rs b/iroh-unixfs/src/hamt/bitfield.rs index d34cf930a5..fe26721067 100644 --- a/iroh-unixfs/src/hamt/bitfield.rs +++ b/iroh-unixfs/src/hamt/bitfield.rs @@ -47,21 +47,21 @@ impl Bitfield { pub fn clear_bit(&mut self, idx: u32) { let ai = idx / 64; let bi = idx % 64; - self.0[ai as usize] &= u64::MAX - (1 << bi as u32); + self.0[ai as usize] &= u64::MAX - (1 << bi); } pub fn test_bit(&self, idx: u32) -> bool { let ai = idx / 64; let bi = idx % 64; - self.0[ai as usize] & (1 << bi as u32) != 0 + self.0[ai as usize] & (1 << bi) != 0 } pub fn set_bit(&mut self, idx: u32) { let ai = idx / 64; let bi = idx % 64; - self.0[ai as usize] |= 1 << bi as u32; + self.0[ai as usize] |= 1 << bi; } pub fn count_ones(&self) -> usize { diff --git a/iroh-util/src/lib.rs b/iroh-util/src/lib.rs index 7bee91a84a..1922a640d7 100644 --- a/iroh-util/src/lib.rs +++ b/iroh-util/src/lib.rs @@ -248,7 +248,7 @@ pub fn increase_fd_limit() -> std::io::Result { if soft < MIN_NOFILE_LIMIT { return Err(std::io::Error::new( std::io::ErrorKind::Other, - format!("NOFILE limit too low: {}", soft), + format!("NOFILE limit too low: {soft}"), )); } Ok(soft) diff --git a/iroh-util/src/lock.rs b/iroh-util/src/lock.rs index d82d72f6ee..a7f32dd6a8 100644 --- a/iroh-util/src/lock.rs +++ b/iroh-util/src/lock.rs @@ -25,7 +25,7 @@ pub struct ProgramLock { impl ProgramLock { /// Create a new lock for the given program. This does not yet acquire the lock. pub fn new(prog_name: &str) -> Result { - let path = crate::iroh_data_path(&format!("{}.lock", prog_name)) + let path = crate::iroh_data_path(&format!("{prog_name}.lock")) .map_err(|e| LockError::InvalidPath { source: e })?; Ok(Self { path, @@ -165,7 +165,7 @@ impl Drop for ProgramLock { /// Report Process ID stored in a lock file pub fn read_lock_pid(prog_name: &str) -> Result { - let path = crate::iroh_data_path(&format!("{}.lock", prog_name))?; + let path = crate::iroh_data_path(&format!("{prog_name}.lock"))?; read_lock(&path) } diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 41e6805c75..edab91f2b3 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -30,6 +30,7 @@ iroh-api = { version = "0.1.3", path = "../iroh-api"} iroh-localops = { version = "0.1.3", path = "../iroh-localops" } iroh-metrics = { version = "0.1.3", path = "../iroh-metrics", default-features = false } iroh-util = { version = "0.1.3", path = "../iroh-util"} +iroh-unixfs = { version = "0.1.3", path = "../iroh-unixfs" } relative-path = { version = "1.7.2", optional = true } serde = { version = "1.0", features = ["derive"] } sysinfo = "0.26.4" diff --git a/stores/flatfs/src/flatfs.rs b/stores/flatfs/src/flatfs.rs index 139359fb22..41e541e6b7 100644 --- a/stores/flatfs/src/flatfs.rs +++ b/stores/flatfs/src/flatfs.rs @@ -65,11 +65,11 @@ impl Flatfs { let temp_filepath = filepath.with_extension(".temp"); let value = value.as_ref(); retry(|| fs::write(&temp_filepath, value)) - .with_context(|| format!("Failed to write {:?}", temp_filepath))?; + .with_context(|| format!("Failed to write {temp_filepath:?}"))?; // Rename after successfull write retry(|| fs::rename(&temp_filepath, &filepath)) - .with_context(|| format!("Failed to reaname: {:?} -> {:?}", temp_filepath, filepath))?; + .with_context(|| format!("Failed to reaname: {temp_filepath:?} -> {filepath:?}"))?; self.disk_usage .fetch_add(value.len() as u64, Ordering::SeqCst); @@ -83,7 +83,7 @@ impl Flatfs { let filepath = self.as_path(key); let value = retry(|| fs::read(&filepath)) - .with_context(|| format!("Failed to read {:?}", filepath))?; + .with_context(|| format!("Failed to read {filepath:?}"))?; Ok(value) } @@ -95,7 +95,7 @@ impl Flatfs { let metadata = filepath .metadata() - .with_context(|| format!("Failed to read metadata for {:?}", filepath))?; + .with_context(|| format!("Failed to read metadata for {filepath:?}"))?; Ok(metadata.len()) } @@ -107,11 +107,11 @@ impl Flatfs { let metadata = filepath .metadata() - .with_context(|| format!("Failed to read metadata for {:?}", filepath))?; + .with_context(|| format!("Failed to read metadata for {filepath:?}"))?; let filesize = metadata.len(); retry(|| fs::remove_file(&filepath)) - .with_context(|| format!("Failed to remove {:?}", filepath))?; + .with_context(|| format!("Failed to remove {filepath:?}"))?; self.disk_usage.fetch_sub(filesize, Ordering::SeqCst); @@ -300,7 +300,7 @@ impl Drop for Flatfs { fn write_disk_usage>(path: P, usage: u64) -> Result<()> { let disk_usage_path = path.as_ref().join(DISK_USAGE_CACHE); fs::write(&disk_usage_path, &usage.to_string()[..]) - .with_context(|| format!("Failed to write to {:?}", disk_usage_path))?; + .with_context(|| format!("Failed to write to {disk_usage_path:?}"))?; Ok(()) } @@ -309,7 +309,7 @@ fn calculate_disk_usage>(path: P) -> Result { let disk_usage_path = path.as_ref().join(DISK_USAGE_CACHE); if disk_usage_path.exists() { let usage: u64 = fs::read_to_string(&disk_usage_path) - .with_context(|| format!("Failed to read {:?}", disk_usage_path))? + .with_context(|| format!("Failed to read {disk_usage_path:?}"))? .parse()?; return Ok(usage); } diff --git a/stores/flatfs/src/shard.rs b/stores/flatfs/src/shard.rs index 69ee2273f5..d214bf0caa 100644 --- a/stores/flatfs/src/shard.rs +++ b/stores/flatfs/src/shard.rs @@ -31,14 +31,13 @@ impl Shard { pub fn write_to_file>(&self, path: P) -> Result<()> { let content = self.to_string(); let path = path.as_ref().join(FILE_NAME); - fs::write(&path, content) - .with_context(|| format!("Failed to write shard to {:?}", path))?; + fs::write(&path, content).with_context(|| format!("Failed to write shard to {path:?}"))?; Ok(()) } pub fn from_file>(path: P) -> Result { let path = path.as_ref().join(FILE_NAME); - let file = File::open(&path).with_context(|| format!("Failed to open file {:?}", path))?; + let file = File::open(&path).with_context(|| format!("Failed to open file {path:?}"))?; let mut content = String::with_capacity(50); // Protect agains invalid files and unknown formats. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1f874b8bbc..59c157f50f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -95,8 +95,8 @@ fn coverage() -> Result<()> { } fn dist() -> Result<()> { - let _ = fs::remove_dir_all(&dist_dir()); - fs::create_dir_all(&dist_dir())?; + let _ = fs::remove_dir_all(dist_dir()); + fs::create_dir_all(dist_dir())?; dist_binaries()?; dist_manpage()?;