From 96903e776e03c7f72155db3c2e105f33389cb06f Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Fri, 12 May 2023 10:43:33 +0200 Subject: [PATCH] feat: add configuration for derp regions * feat: add configuration for derp regions * add comment --- .gitignore | 1 + Cargo.lock | 50 ++++++++++++++++- Cargo.toml | 1 + example.config.toml | 16 ++++++ src/config.rs | 130 ++++++++++++++++++++++++++++++++++++++++++++ src/hp/derp/map.rs | 10 ++-- src/lib.rs | 1 + src/main.rs | 46 ++++++++++++---- src/main_util.rs | 12 +--- 9 files changed, 239 insertions(+), 28 deletions(-) create mode 100644 example.config.toml create mode 100644 src/config.rs diff --git a/.gitignore b/.gitignore index ea8c4bf7f3..fe8147e83b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +iroh.config.toml diff --git a/Cargo.lock b/Cargo.lock index d11579fd6e..46e2ba1638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -736,6 +736,23 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "config" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" +dependencies = [ + "async-trait", + "indexmap", + "lazy_static", + "nom", + "pathdiff", + "ron", + "serde", + "serde_json", + "toml 0.5.11", +] + [[package]] name = "console" version = "0.15.5" @@ -1798,6 +1815,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] @@ -1887,6 +1905,7 @@ dependencies = [ "build-data", "bytes", "clap", + "config", "console", "crypto_box", "curve25519-dalek", @@ -1949,7 +1968,7 @@ dependencies = [ "tokio-rustls 0.24.0", "tokio-stream", "tokio-util", - "toml", + "toml 0.7.3", "tracing", "tracing-futures", "tracing-subscriber", @@ -2541,6 +2560,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pem" version = "1.1.1" @@ -3285,6 +3310,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "ron" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" +dependencies = [ + "base64 0.13.1", + "bitflags", + "indexmap", + "serde", +] + [[package]] name = "rsa" version = "0.9.0-pre.0" @@ -3645,6 +3682,7 @@ version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -4207,6 +4245,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "indexmap", + "serde", +] + [[package]] name = "toml" version = "0.7.3" diff --git a/Cargo.toml b/Cargo.toml index 036a1533aa..9d64d78d16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ base64 = "0.21.0" blake3 = "1.3.3" bytes = { version = "1.4", features = ["serde"] } clap = { version = "4", features = ["derive"], optional = true } +config = { version = "0.13.1", default-features = false, features = ["toml", "preserve_order"] } console = { version = "0.15.5", optional = true } crypto_box = { version = "0.9.0-pre", features = ["serde", "chacha20"] } curve25519-dalek = "=4.0.0-rc.2" diff --git a/example.config.toml b/example.config.toml new file mode 100644 index 0000000000..02f1865156 --- /dev/null +++ b/example.config.toml @@ -0,0 +1,16 @@ +[[derp_regions]] +region_id = 1 +avoid = false +region_code = "default" + + +[[derp_regions.nodes]] +name = "default-1" +region_id = 1 +host_name = "foo.bar" +stun_only = false +stun_port = 1244 +ipv4 = { Some = "127.0.0.1" } +ipv6 = "None" +derp_port = 1 + diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000000..0907285255 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,130 @@ +//! Configuration for the iroh CLI. + +use std::{collections::HashMap, path::Path}; + +use anyhow::Result; +use config::{Environment, File, Value}; +use serde::{Deserialize, Serialize}; +use tracing::debug; + +use crate::hp::derp::{DerpMap, DerpNode, DerpRegion, UseIpv4, UseIpv6}; + +/// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory +pub const CONFIG_FILE_NAME: &str = "iroh.config.toml"; +/// ENV_PREFIX should be used along side the config field name to set a config field using +/// environment variables +/// For example, `IROH_PATH=/path/to/config` would set the value of the `Config.path` field +pub const ENV_PREFIX: &str = "IROH"; + +/// The configuration for the iroh cli. +#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] +#[serde(default)] +pub struct Config { + /// The regions for DERP to use. + pub derp_regions: Vec, +} + +impl Default for Config { + fn default() -> Self { + Self { + derp_regions: vec![default_derp_region()], + } + } +} + +impl Config { + /// Make a config using a default, files, environment variables, and commandline flags. + /// + /// Later items in the *file_paths* slice will have a higher priority than earlier ones. + /// + /// Environment variables are expected to start with the *env_prefix*. Nested fields can be + /// accessed using `.`, if your environment allows env vars with `.` + /// + /// Note: For the metrics configuration env vars, it is recommended to use the metrics + /// specific prefix `IROH_METRICS` to set a field in the metrics config. You can use the + /// above dot notation to set a metrics field, eg, `IROH_CONFIG_METRICS.SERVICE_NAME`, but + /// only if your environment allows it + pub fn load( + file_paths: &[Option<&Path>], + env_prefix: &str, + flag_overrides: HashMap, + ) -> Result + where + S: AsRef, + V: Into, + { + let mut builder = config::Config::builder(); + + // layer on config options from files + for path in file_paths.iter().flatten() { + if path.exists() { + let p = path.to_str().ok_or_else(|| anyhow::anyhow!("empty path"))?; + builder = builder.add_source(File::with_name(p)); + } + } + + // next, add any environment variables + builder = builder.add_source( + Environment::with_prefix(env_prefix) + .separator("__") + .try_parsing(true), + ); + + // finally, override any values + for (flag, val) in flag_overrides.into_iter() { + builder = builder.set_override(flag, val)?; + } + + let cfg = builder.build()?; + debug!("make_config:\n{:#?}\n", cfg); + let cfg = cfg.try_deserialize()?; + Ok(cfg) + } + + /// Constructs a `DerpMap` based on the current configuration. + pub fn derp_map(&self) -> Option { + if self.derp_regions.is_empty() { + return None; + } + + let mut regions = HashMap::new(); + for region in &self.derp_regions { + regions.insert(region.region_id, region.clone()); + } + + Some(DerpMap { regions }) + } +} + +fn default_derp_region() -> DerpRegion { + // The default derper run by number0. + let default_n0_derp = DerpNode { + name: "default-1".into(), + region_id: 1, + host_name: "derp.iroh.computer".into(), + stun_only: false, + stun_port: 3478, + ipv4: UseIpv4::Some("35.175.99.113".parse().unwrap()), + ipv6: UseIpv6::None, + derp_port: 3340, + stun_test_ip: None, + }; + DerpRegion { + region_id: 1, + nodes: vec![default_n0_derp], + avoid: false, + region_code: "default-1".into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_settings() { + let config = Config::load::(&[][..], "__FOO", Default::default()).unwrap(); + + assert_eq!(config.derp_regions.len(), 1); + } +} diff --git a/src/hp/derp/map.rs b/src/hp/derp/map.rs index 8743f416da..d87e79bafe 100644 --- a/src/hp/derp/map.rs +++ b/src/hp/derp/map.rs @@ -5,6 +5,8 @@ use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, }; +use serde::{Deserialize, Serialize}; + #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct DerpMap { pub regions: HashMap, @@ -54,7 +56,7 @@ impl DerpMap { } /// A geographic region running DERP relay node(s). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DerpRegion { /// A unique integer for a geographic region. pub region_id: usize, @@ -63,7 +65,7 @@ pub struct DerpRegion { pub region_code: String, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DerpNode { pub name: String, pub region_id: usize, @@ -82,7 +84,7 @@ pub struct DerpNode { pub derp_port: u16, } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum UseIpv4 { None, Disabled, @@ -96,7 +98,7 @@ impl UseIpv4 { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum UseIpv6 { None, Disabled, diff --git a/src/lib.rs b/src/lib.rs index 721b23459d..f68d3f1eb0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ // #![deny(missing_docs)] TODO: fix me before merging #![deny(rustdoc::broken_intra_doc_links)] pub mod blobs; +pub mod config; pub mod get; pub mod main_util; pub mod metrics; diff --git a/src/main.rs b/src/main.rs index d6d71fcbf1..aecae5d660 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,10 +12,11 @@ use indicatif::{ ProgressStyle, }; use iroh::blobs::{Blob, Collection}; +use iroh::config::{Config, CONFIG_FILE_NAME, ENV_PREFIX}; use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; use iroh::get::{get_data_path, get_missing_range, get_missing_ranges}; use iroh::hp::derp::DerpMap; -use iroh::main_util::pathbuf_from_name; +use iroh::main_util::{iroh_config_path, pathbuf_from_name}; use iroh::protocol::{GetRequest, RangeSpecSeq}; use iroh::provider::{Database, Provider, Ticket}; use iroh::rpc_protocol::*; @@ -26,7 +27,7 @@ use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; use tracing_subscriber::{prelude::*, EnvFilter}; -use iroh::main_util::{configure_derp_map, create_quinn_client, iroh_data_root, Blake3Cid}; +use iroh::main_util::{create_quinn_client, iroh_data_root, Blake3Cid}; use iroh::provider::FNAME_PATHS; use iroh::{get, provider, Hash, Keypair, PeerId}; @@ -63,6 +64,8 @@ struct Cli { #[cfg(feature = "metrics")] #[clap(long)] metrics_addr: Option, + #[clap(long)] + cfg: Option, } #[derive(Debug, Clone)] @@ -491,6 +494,18 @@ async fn main_impl() -> Result<()> { let cli = Cli::parse(); + let config_path = iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; + let sources = [Some(config_path.as_path()), cli.cfg.as_deref()]; + let config = Config::load( + // potential config files + &sources, + // env var prefix for this config + ENV_PREFIX, + // map of present command line arguments + // args.make_overrides_map(), + HashMap::::new(), + )?; + #[cfg(feature = "metrics")] let metrics_fut = init_metrics_collection(cli.metrics_addr); @@ -502,12 +517,11 @@ async fn main_impl() -> Result<()> { out, single, } => { - let dm = configure_derp_map(); // TODO: pass what is needed. let opts = get::Options { addr, peer_id: Some(peer), keylog: cli.keylog, - derp_map: Some(dm), + derp_map: config.derp_map(), }; let get = GetInteractive::Hash { hash: *hash.as_hash(), @@ -524,11 +538,10 @@ async fn main_impl() -> Result<()> { } } Commands::GetTicket { out, ticket } => { - let dm = configure_derp_map(); // TODO: pass what is needed. let get = GetInteractive::Ticket { ticket, keylog: cli.keylog, - derp_map: Some(dm), + derp_map: config.derp_map(), }; tokio::select! { biased; @@ -562,7 +575,15 @@ async fn main_impl() -> Result<()> { }; let key = Some(iroh_data_root.join("keypair")); - let provider = provide(db.clone(), addr, key, cli.keylog, rpc_port.into()).await?; + let provider = provide( + db.clone(), + addr, + key, + cli.keylog, + rpc_port.into(), + config.derp_map(), + ) + .await?; let controller = provider.controller(); // task that will add data to the provider, either from a file or from stdin @@ -710,14 +731,15 @@ async fn provide( key: Option, keylog: bool, rpc_port: Option, + dm: Option, ) -> Result { let keypair = get_keypair(key).await?; - let dm = configure_derp_map(); // TODO: pass what is needed. - let builder = provider::Provider::builder(db) - .keylog(keylog) - .derp_map(dm) - .bind_addr(addr); + let mut builder = provider::Provider::builder(db).keylog(keylog); + if let Some(dm) = dm { + builder = builder.derp_map(dm); + } + let builder = builder.bind_addr(addr); let provider = if let Some(rpc_port) = rpc_port { let rpc_endpoint = make_rpc_endpoint(&keypair, rpc_port)?; diff --git a/src/main_util.rs b/src/main_util.rs index 7aa3c756e0..b77e5c0210 100644 --- a/src/main_util.rs +++ b/src/main_util.rs @@ -41,7 +41,7 @@ pub fn iroh_config_root() -> Result { /// Path that leads to a file in the iroh config directory. #[allow(dead_code)] -pub fn iroh_config_path(file_name: &Path) -> Result { +pub fn iroh_config_path(file_name: impl AsRef) -> Result { let path = iroh_config_root()?.join(file_name); Ok(path) } @@ -215,16 +215,6 @@ pub fn create_quinn_client( Ok(endpoint) } -pub fn configure_derp_map() -> DerpMap { - // Use google stun server for now - let stun_port = 3478; - let host_name = "derp.iroh.computer".into(); - let derp_port = 3340; - let derp_ipv4 = UseIpv4::Some("35.175.99.113".parse().unwrap()); - let derp_ipv6 = UseIpv6::None; - DerpMap::default_from_node(host_name, stun_port, derp_port, derp_ipv4, derp_ipv6) -} - pub fn configure_local_derp_map() -> DerpMap { // Use google stun server for now let stun_port = 3478;