From 2110613b7fc15566bc9fb480e61ccd59b5f361b8 Mon Sep 17 00:00:00 2001 From: Matt Briggs Date: Wed, 28 Oct 2020 18:54:03 -0700 Subject: [PATCH] tough: update to v0.10.0 This version of tough changes the interface of the Repository struct and its load function. The usage sites have been updated to use the new interface. updog: The query parameter mechanism needed to be changed to accommodate the fact that Repository now owns a Transport instead of holding a reference. This is done by holding an Arc pointer to the query params both inside and outside of the custom Transport. pubsys: A custom Transport is no-longer necessary (its purpose was to support both http and file transport, which DefaultTransport now does). Additionally, the new TransportError gives pubsys the information it needs to know whether a repo already exists or not. update_metadata and migrator: no significant changes. --- sources/Cargo.lock | 12 +- sources/api/migration/migrator/Cargo.toml | 4 +- sources/api/migration/migrator/src/error.rs | 3 - sources/api/migration/migrator/src/main.rs | 69 ++-- sources/api/migration/migrator/src/test.rs | 12 +- sources/updater/update_metadata/Cargo.toml | 2 +- sources/updater/update_metadata/src/lib.rs | 2 +- sources/updater/updog/Cargo.toml | 3 +- sources/updater/updog/src/error.rs | 99 +----- sources/updater/updog/src/main.rs | 77 ++--- sources/updater/updog/src/transport.rs | 102 +++--- tools/Cargo.lock | 308 +++--------------- tools/pubsys/Cargo.toml | 6 +- tools/pubsys/src/repo.rs | 68 ++-- .../pubsys/src/repo/check_expirations/mod.rs | 40 +-- tools/pubsys/src/repo/refresh_repo/mod.rs | 37 +-- tools/pubsys/src/repo/transport.rs | 63 ---- tools/pubsys/src/repo/validate_repo/mod.rs | 43 +-- 18 files changed, 280 insertions(+), 670 deletions(-) delete mode 100644 tools/pubsys/src/repo/transport.rs diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 6728d700352..098171cfc42 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -883,6 +883,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dyn-clone" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" + [[package]] name = "early-boot-config" version = "0.1.0" @@ -3178,11 +3184,12 @@ dependencies = [ [[package]] name = "tough" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b8d86994e9da2233fc30c54223bc448a15bdb782f8060c66107fc6b88619ba" +checksum = "7dc3534fa46badec98ac633028f47a3cea590e9c9a63d85bd15a0436f8b6eb94" dependencies = [ "chrono", + "dyn-clone", "globset", "hex", "log", @@ -3194,6 +3201,7 @@ dependencies = [ "serde_json", "serde_plain", "snafu", + "tempfile", "untrusted", "url", "walkdir", diff --git a/sources/api/migration/migrator/Cargo.toml b/sources/api/migration/migrator/Cargo.toml index 1544ba2a6d5..72ae3ded631 100644 --- a/sources/api/migration/migrator/Cargo.toml +++ b/sources/api/migration/migrator/Cargo.toml @@ -20,8 +20,7 @@ regex = "1.1" semver = "0.11" simplelog = "0.9" snafu = "0.6" -tempfile = "3.1.0" -tough = "0.8" +tough = "0.10" update_metadata = { path = "../../../updater/update_metadata" } url = "2.1.1" @@ -31,6 +30,7 @@ cargo-readme = "3.1" [dev-dependencies] chrono = "0.4.11" storewolf = { path = "../../storewolf" } +tempfile = "3.1.0" [[bin]] name = "migrator" diff --git a/sources/api/migration/migrator/src/error.rs b/sources/api/migration/migrator/src/error.rs index b58c83595bd..d4c63b28d5c 100644 --- a/sources/api/migration/migrator/src/error.rs +++ b/sources/api/migration/migrator/src/error.rs @@ -13,9 +13,6 @@ pub(crate) enum Error { #[snafu(display("Internal error: {}", msg))] Internal { msg: String }, - #[snafu(display("Unable to create tempdir for tough datastore: '{}'", source))] - CreateToughTempDir { source: std::io::Error }, - #[snafu(display("Data store path '{}' contains invalid UTF-8", path.display()))] DataStorePathNotUTF8 { path: PathBuf }, diff --git a/sources/api/migration/migrator/src/main.rs b/sources/api/migration/migrator/src/main.rs index d4b914092cf..bf53542f634 100644 --- a/sources/api/migration/migrator/src/main.rs +++ b/sources/api/migration/migrator/src/main.rs @@ -38,9 +38,9 @@ use std::os::unix::fs::symlink; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::process; -use tempfile::TempDir; -use tough::{ExpirationEnforcement, Limits}; +use tough::{ExpirationEnforcement, FilesystemTransport, RepositoryLoader}; use update_metadata::load_manifest; +use url::Url; mod args; mod direction; @@ -113,49 +113,48 @@ pub(crate) fn run(args: &Args) -> Result<()> { process::exit(0); }); - // Prepare to load the locally cached TUF repository to obtain the manifest. Part of using a - // `TempDir` is disabling timestamp checking, because we want an instance to still come up and - // run migrations regardless of the how the system time relates to what we have cached (for - // example if someone runs an update, then shuts down the instance for several weeks, beyond the - // expiration of at least the cached timestamp.json before booting it back up again). We also - // use a `TempDir` because see no value in keeping a datastore around. The latest known - // versions of the repository metadata will always be the versions of repository metadata we - // have cached on the disk. More info at `ExpirationEnforcement::Unsafe` below. - let tough_datastore = TempDir::new().context(error::CreateToughTempDir)?; - let metadata_url = url::Url::from_directory_path(&args.metadata_directory).map_err(|_| { + // create URLs from the metadata and targets directory paths + let metadata_base_url = Url::from_directory_path(&args.metadata_directory).map_err(|_| { error::Error::DirectoryUrl { path: args.metadata_directory.clone(), } })?; - let migrations_url = + let targets_base_url = url::Url::from_directory_path(&args.migration_directory).map_err(|_| { error::Error::DirectoryUrl { path: args.migration_directory.clone(), } })?; + + // open a reader to the root.json file + let root_file = File::open(&args.root_path).with_context(|| error::OpenRoot { + path: args.root_path.clone(), + })?; + + // We will load the locally cached TUF repository to obtain the manifest. The Repository is + // loaded using a `TempDir` for its internal Datastore (this is the default). Part of using a + // `TempDir` is disabling timestamp checking, because we want an instance to still come up and + // run migrations regardless of the how the system time relates to what we have cached (for + // example if someone runs an update, then shuts down the instance for several weeks, beyond the + // expiration of at least the cached timestamp.json before booting it back up again). We also + // use a `TempDir` because see no value in keeping a datastore around. The latest known + // versions of the repository metadata will always be the versions of repository metadata we + // have cached on the disk. More info at `ExpirationEnforcement::Unsafe` below. + // Failure to load the TUF repo at the expected location is a serious issue because updog should // always create a TUF repo that contains at least the manifest, even if there are no migrations. - let repo = tough::Repository::load( - &tough::FilesystemTransport, - tough::Settings { - root: File::open(&args.root_path).context(error::OpenRoot { - path: args.root_path.clone(), - })?, - datastore: tough_datastore.path(), - metadata_base_url: metadata_url.as_str(), - targets_base_url: migrations_url.as_str(), - limits: Limits::default(), - // The threats TUF mitigates are more than the threats we are attempting to mitigate - // here by caching signatures for migrations locally and using them after a reboot but - // prior to Internet connectivity. We are caching the TUF repo and use it while offline - // after a reboot to mitigate binaries being added or modified in the migrations - // directory; the TUF repo is simply a code signing method we already have in place, - // even if it's not one that initially makes sense for this use case. So, we don't care - // if the targets expired between updog downloading them and now. - expiration_enforcement: ExpirationEnforcement::Unsafe, - }, - ) - .context(error::RepoLoad)?; + let repo = RepositoryLoader::new(root_file, metadata_base_url, targets_base_url) + .transport(FilesystemTransport) + // The threats TUF mitigates are more than the threats we are attempting to mitigate + // here by caching signatures for migrations locally and using them after a reboot but + // prior to Internet connectivity. We are caching the TUF repo and use it while offline + // after a reboot to mitigate binaries being added or modified in the migrations + // directory; the TUF repo is simply a code signing method we already have in place, + // even if it's not one that initially makes sense for this use case. So, we don't care + // if the targets expired between updog downloading them and now. + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .load() + .context(error::RepoLoad)?; let manifest = load_manifest(&repo).context(error::LoadManifest)?; let migrations = update_metadata::find_migrations(¤t_version, &args.migrate_to_version, &manifest) @@ -222,7 +221,7 @@ where /// The given data store is used as a starting point; each migration is given the output of the /// previous migration, and the final output becomes the new data store. fn run_migrations( - repository: &tough::Repository<'_, tough::FilesystemTransport>, + repository: &tough::Repository, direction: Direction, migrations: &[S], source_datastore: P, diff --git a/sources/api/migration/migrator/src/test.rs b/sources/api/migration/migrator/src/test.rs index b8d34be4a4e..b385b79dff9 100644 --- a/sources/api/migration/migrator/src/test.rs +++ b/sources/api/migration/migrator/src/test.rs @@ -138,7 +138,9 @@ fn create_test_repo() -> TestRepo { let one = std::num::NonZeroU64::new(1).unwrap(); editor .targets_version(one) + .unwrap() .targets_expires(long_ago) + .unwrap() .snapshot_version(one) .snapshot_expires(long_ago) .timestamp_version(one) @@ -154,10 +156,12 @@ fn create_test_repo() -> TestRepo { }) .for_each(|dir_entry_result| { let dir_entry = dir_entry_result.unwrap(); - editor.add_target( - dir_entry.file_name().to_str().unwrap().into(), - tough::schema::Target::from_path(dir_entry.path()).unwrap(), - ); + editor + .add_target( + dir_entry.file_name().to_str().unwrap().into(), + tough::schema::Target::from_path(dir_entry.path()).unwrap(), + ) + .unwrap(); }); let signed_repo = editor .sign(&[Box::new(tough::key_source::LocalKeySource { path: pem() })]) diff --git a/sources/updater/update_metadata/Cargo.toml b/sources/updater/update_metadata/Cargo.toml index f3e0b29fb13..e02f6a677c0 100644 --- a/sources/updater/update_metadata/Cargo.toml +++ b/sources/updater/update_metadata/Cargo.toml @@ -18,7 +18,7 @@ serde_json = "1.0.40" serde_plain = "0.3.0" snafu = "0.6.0" toml = "0.5" -tough = "0.8" +tough = "0.10" [lib] name = "update_metadata" diff --git a/sources/updater/update_metadata/src/lib.rs b/sources/updater/update_metadata/src/lib.rs index a3a4dea1c21..467bb560492 100644 --- a/sources/updater/update_metadata/src/lib.rs +++ b/sources/updater/update_metadata/src/lib.rs @@ -444,7 +444,7 @@ fn find_migrations_forward( Ok(targets) } -pub fn load_manifest(repository: &tough::Repository) -> Result { +pub fn load_manifest(repository: &tough::Repository) -> Result { let target = "manifest.json"; serde_json::from_reader( repository diff --git a/sources/updater/updog/Cargo.toml b/sources/updater/updog/Cargo.toml index edd4c4a4587..a9ca3a6e867 100644 --- a/sources/updater/updog/Cargo.toml +++ b/sources/updater/updog/Cargo.toml @@ -22,9 +22,8 @@ serde_plain = "0.3.0" signpost = { path = "../signpost" } simplelog = "0.9" snafu = "0.6.0" -tempfile = "3.1.0" toml = "0.5.1" -tough = { version = "0.8", features = ["http"] } +tough = { version = "0.10", features = ["http"] } update_metadata = { path = "../update_metadata" } structopt = "0.3" url = "2.1.0" diff --git a/sources/updater/updog/src/error.rs b/sources/updater/updog/src/error.rs index 2155f23a4b0..494031d18a2 100644 --- a/sources/updater/updog/src/error.rs +++ b/sources/updater/updog/src/error.rs @@ -1,6 +1,5 @@ #![allow(clippy::default_trait_access)] -use semver::Version; use snafu::{Backtrace, Snafu}; use std::path::PathBuf; use update_metadata::error::Error as update_metadata_error; @@ -40,13 +39,6 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Failed to serialize config file {}: {}", path.display(), source))] - ConfigSerialize { - path: PathBuf, - source: toml::ser::Error, - backtrace: Backtrace, - }, - #[snafu(display("Failed to create metadata cache directory '{}': {}", path, source))] CreateMetadataCache { path: &'static str, @@ -54,12 +46,6 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Failed to create a tempdir for tough datastore: {}", source))] - CreateTempDir { - source: std::io::Error, - backtrace: Backtrace, - }, - #[snafu(display("Failed to create directory: {:?}", path))] DirCreate { backtrace: Backtrace, @@ -73,27 +59,6 @@ pub(crate) enum Error { #[snafu(display("Could not mark inactive partition for boot: {}", source))] InactivePartitionUpgrade { source: signpost::Error }, - #[snafu(display("Failed to attach image to loop device"))] - LoopAttachFailed { - backtrace: Backtrace, - source: std::io::Error, - }, - - #[snafu(display("Failed to open loop device control"))] - LoopControlFailed { - backtrace: Backtrace, - source: std::io::Error, - }, - - #[snafu(display("Failed to find free loop device"))] - LoopFindFailed { - backtrace: Backtrace, - source: std::io::Error, - }, - - #[snafu(display("Could not determine loop device path"))] - LoopNameFailed { backtrace: Backtrace }, - #[snafu(display("Failed to decode LZ4-compressed target {}: {}", target, source))] Lz4Decode { target: String, @@ -101,12 +66,6 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Failed to parse updates manifest: {}", source))] - ManifestParse { - source: serde_json::Error, - backtrace: Backtrace, - }, - #[snafu(display("Metadata error: {}", source))] Metadata { source: tough::error::Error, @@ -120,31 +79,6 @@ pub(crate) enum Error { name: String, }, - #[snafu(display("Migration not found in image: {:?}", name))] - MigrationNotLocal { backtrace: Backtrace, name: PathBuf }, - - #[snafu(display("Migration ({},{}) not present in manifest", from, to))] - MigrationNotPresent { - backtrace: Backtrace, - from: Version, - to: Version, - }, - - #[snafu(display("Missing version in metadata: {}", version))] - MissingVersion { - backtrace: Backtrace, - version: String, - }, - - #[snafu(display("Temporary image mount failed"))] - MountFailed { - backtrace: Backtrace, - source: std::io::Error, - }, - - #[snafu(display("No update available"))] - NoUpdate { backtrace: Backtrace }, - #[snafu(display("Failed to open partition {}: {}", path.display(), source))] OpenPartition { path: PathBuf, @@ -191,12 +125,6 @@ pub(crate) enum Error { #[snafu(display("Unable to get OS version: {}", source))] ReleaseVersion { source: bottlerocket_release::Error }, - #[snafu(display("Failed setting permissions of '{}': {}", path.display(), source))] - SetPermissions { - path: PathBuf, - source: std::io::Error, - }, - #[snafu(display("Target not found: {}", target))] TargetNotFound { target: String, @@ -209,36 +137,15 @@ pub(crate) enum Error { source: std::io::Error, }, - #[snafu(display("2Borrow2Fast"))] - TransportBorrow { - backtrace: Backtrace, - source: std::cell::BorrowMutError, - }, - #[snafu(display("No update available"))] UpdateNotAvailable { backtrace: Backtrace }, - #[snafu(display("Update {} exists but wave in the future", version))] - UpdateNotReady { - backtrace: Backtrace, - version: Version, - }, - #[snafu(display("Failed to serialize update information: {}", source))] UpdateSerialize { source: serde_json::Error, backtrace: Backtrace, }, - #[snafu(display("Update in the incorrect state"))] - UpdateState { backtrace: Backtrace }, - - #[snafu(display("Target partition is unrecognized: {}", partition))] - UnknownPartition { - partition: String, - backtrace: Backtrace, - }, - #[snafu(display("--wave-file required to add waves to update"))] WaveFileArg { backtrace: Backtrace }, @@ -261,6 +168,12 @@ pub(crate) enum Error { #[snafu(display("Failed to store manifest and migrations: {}", source))] RepoCacheMigrations { source: tough::error::Error }, + + #[snafu(display("Unable to parse '{}' as a URL: {}", url, source))] + UrlParse { + source: url::ParseError, + url: String, + }, } impl std::convert::From for Error { diff --git a/sources/updater/updog/src/main.rs b/sources/updater/updog/src/main.rs index e6713aec502..a57ff08f509 100644 --- a/sources/updater/updog/src/main.rs +++ b/sources/updater/updog/src/main.rs @@ -5,7 +5,7 @@ mod error; mod transport; use crate::error::Result; -use crate::transport::{HttpQueryRepo, HttpQueryTransport}; +use crate::transport::{HttpQueryTransport, QueryParams}; use bottlerocket_release::BottlerocketRelease; use chrono::Utc; use model::modeled_types::FriendlyVersion; @@ -22,9 +22,9 @@ use std::path::Path; use std::process; use std::str::FromStr; use std::thread; -use tempfile::TempDir; -use tough::{ExpirationEnforcement, Limits, Repository, Settings}; +use tough::{Repository, RepositoryLoader}; use update_metadata::{find_migrations, load_manifest, Manifest, Update}; +use url::Url; #[cfg(target_arch = "x86_64")] const TARGET_ARCH: &str = "x86_64"; @@ -112,27 +112,23 @@ fn load_config() -> Result { Ok(config) } -fn load_repository<'a>( - transport: &'a HttpQueryTransport, - config: &'a Config, - tough_datastore: &'a Path, -) -> Result> { +fn load_repository(transport: HttpQueryTransport, config: &Config) -> Result { fs::create_dir_all(METADATA_PATH).context(error::CreateMetadataCache { path: METADATA_PATH, })?; - Repository::load( - transport, - Settings { - root: File::open(TRUSTED_ROOT_PATH).context(error::OpenRoot { - path: TRUSTED_ROOT_PATH, - })?, - datastore: tough_datastore, - metadata_base_url: &config.metadata_base_url, - targets_base_url: &config.targets_base_url, - limits: Limits::default(), - expiration_enforcement: ExpirationEnforcement::Safe, - }, + RepositoryLoader::new( + File::open(TRUSTED_ROOT_PATH).context(error::OpenRoot { + path: TRUSTED_ROOT_PATH, + })?, + Url::parse(&config.metadata_base_url).context(error::UrlParse { + url: &config.metadata_base_url, + })?, + Url::parse(&config.targets_base_url).context(error::UrlParse { + url: &config.targets_base_url, + })?, ) + .transport(transport) + .load() .context(error::Metadata) } @@ -211,7 +207,7 @@ fn update_required<'a>( } fn write_target_to_disk>( - repository: &HttpQueryRepo<'_>, + repository: &Repository, target: &str, disk_path: P, ) -> Result<()> { @@ -236,8 +232,8 @@ fn write_target_to_disk>( /// Store required migrations for an update in persistent storage. All intermediate migrations /// between the current version and the target version must be retrieved. fn retrieve_migrations( - repository: &HttpQueryRepo<'_>, - transport: &HttpQueryTransport, + repository: &Repository, + query_params: &mut QueryParams, manifest: &Manifest, update: &Update, current_version: &Version, @@ -263,15 +259,11 @@ fn retrieve_migrations( .cache(METADATA_PATH, MIGRATION_PATH, Some(&targets), true) .context(error::RepoCacheMigrations)?; // Set a query parameter listing the required migrations - transport - .queries_get_mut() - .context(error::TransportBorrow)? - .push(("migrations".to_owned(), targets.join(","))); - + query_params.add("migrations", targets.join(",")); Ok(()) } -fn update_image(update: &Update, repository: &HttpQueryRepo<'_>) -> Result<()> { +fn update_image(update: &Update, repository: &Repository) -> Result<()> { let mut gpt_state = State::load().context(error::PartitionTableRead)?; gpt_state.clear_inactive(); // Write out the clearing of the inactive partition immediately, because we're about to @@ -308,16 +300,12 @@ fn revert_update_flags() -> Result<()> { } fn set_common_query_params( - transport: &HttpQueryTransport, + query_params: &mut QueryParams, current_version: &Version, config: &Config, ) -> Result<()> { - let mut transport_borrow = transport - .queries_get_mut() - .context(error::TransportBorrow)?; - - transport_borrow.push((String::from("version"), current_version.to_string())); - transport_borrow.push((String::from("seed"), config.seed.to_string())); + query_params.add("version", current_version.to_string()); + query_params.add("seed", config.seed.to_string()); Ok(()) } @@ -485,9 +473,11 @@ fn main_inner() -> Result<()> { let current_release = BottlerocketRelease::new().context(error::ReleaseVersion)?; let variant = arguments.variant.unwrap_or(current_release.variant_id); let transport = HttpQueryTransport::new(); - set_common_query_params(&transport, ¤t_release.version_id, &config)?; - let tough_datastore = TempDir::new().context(error::CreateTempDir)?; - let repository = load_repository(&transport, &config, tough_datastore.path())?; + // get a shared pointer to the transport's query_params so we can add metrics information to + // the transport's HTTP calls. + let mut query_params = transport.query_params(); + set_common_query_params(&mut query_params, ¤t_release.version_id, &config)?; + let repository = load_repository(transport, &config)?; let manifest = load_manifest(&repository)?; let ignore_waves = arguments.ignore_waves || config.ignore_waves; match command { @@ -526,15 +516,10 @@ fn main_inner() -> Result<()> { arguments.force_version, )? { eprintln!("Starting update to {}", u.version); - - transport - .queries_get_mut() - .context(error::TransportBorrow)? - .push((String::from("target"), u.version.to_string())); - + query_params.add("target", u.version.to_string()); retrieve_migrations( &repository, - &transport, + &mut query_params, &manifest, u, ¤t_release.version_id, diff --git a/sources/updater/updog/src/transport.rs b/sources/updater/updog/src/transport.rs index 203322b8f4f..24d57c6f1f5 100644 --- a/sources/updater/updog/src/transport.rs +++ b/sources/updater/updog/src/transport.rs @@ -1,61 +1,77 @@ -use std::cell::{BorrowMutError, RefCell}; -use tough::{HttpTransport, Repository, Transport}; +use std::sync::{Arc, RwLock}; + +use log::error; +use tough::{HttpTransport, Transport, TransportError}; use url::Url; -#[derive(Debug)] +/// A shared pointer to a list of query params that the transport will add to HTTP calls. +#[derive(Debug, Clone, Default)] +pub(crate) struct QueryParams(Arc>>); + +/// A `tough` `Transport` that allows us to add query parameters to HTTP calls. +#[derive(Debug, Clone)] #[allow(clippy::module_name_repetitions)] -pub struct HttpQueryTransport { +pub(crate) struct HttpQueryTransport { pub inner: HttpTransport, - parameters: RefCell>, + parameters: QueryParams, } -impl HttpQueryTransport { - pub fn new() -> Self { - Self { - inner: HttpTransport::new(), - parameters: RefCell::new(vec![]), - } +impl QueryParams { + pub(crate) fn add_params_to_url(&self, mut url: Url) -> Url { + let mut params = match self.0.write() { + Err(e) => { + // a thread died while holding a lock to the params. unlikely to occur. + error!("unable to add query params to HTTP call: {}", e); + return url; + } + Ok(lock_result) => lock_result, + }; + params.sort_by(|(a, _), (b, _)| a.cmp(b)); + url.query_pairs_mut().extend_pairs(params.iter()); + url } - /// Try to borrow a mutable reference to parameters; returns an error if - /// a borrow is already active - pub fn queries_get_mut( - &self, - ) -> Result>, BorrowMutError> { - self.parameters.try_borrow_mut() + pub(crate) fn add(&self, key: S1, val: S2) -> () + where + S1: Into, + S2: Into, + { + let mut params = match self.0.write() { + Err(e) => { + // a thread died while holding a lock to the params. unlikely to occur. + error!( + "unable to add query param '{}={}': {}", + key.into(), + val.into(), + e + ); + return; + } + Ok(lock_result) => lock_result, + }; + params.push((key.into(), val.into())) } +} - /// Set the query string appended to tough requests, sorting the queries - /// by key name first - fn set_query_string(&self, mut url: Url) -> Url { - if let Ok(mut queries) = self.parameters.try_borrow_mut() { - queries.sort_by(|(a,_), (b,_)| a.cmp(b)); - - for (key, val) in queries.iter() { - url.query_pairs_mut().append_pair(&key, &val); - } - } else { - // We can't sort the actual data at the moment, but we can sort - // what we append to the URL. - let mut queries = self.parameters.borrow().clone(); - queries.sort_by(|(a,_), (b,_)| a.cmp(b)); - - for (key, val) in queries { - url.query_pairs_mut().append_pair(&key, &val); - } +impl HttpQueryTransport { + pub fn new() -> Self { + Self { + inner: HttpTransport::default(), + parameters: QueryParams::default(), } + } - url + /// Obtain a shared pointer to the query params for this transport. + pub fn query_params(&self) -> QueryParams { + QueryParams(Arc::clone(&self.parameters.0)) } } -pub type HttpQueryRepo<'a> = Repository<'a, HttpQueryTransport>; - impl Transport for HttpQueryTransport { - type Stream = ::Stream; - type Error = ::Error; - - fn fetch(&self, url: Url) -> Result { - self.inner.fetch(self.set_query_string(url)) + fn fetch( + &self, + url: Url, + ) -> std::result::Result, TransportError> { + self.inner.fetch(self.parameters.add_params_to_url(url)) } } diff --git a/tools/Cargo.lock b/tools/Cargo.lock index 56467b64c23..c5abc4408d6 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -157,34 +157,13 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.3", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -207,7 +186,7 @@ dependencies = [ "reqwest", "serde", "serde_plain", - "sha2 0.9.2", + "sha2", "snafu", "toml", "url", @@ -220,18 +199,6 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - [[package]] name = "bytes" version = "0.5.6" @@ -311,12 +278,12 @@ dependencies = [ "bytes", "futures", "indicatif", - "rusoto_core 0.45.0", - "rusoto_credential 0.45.0", + "rusoto_core", + "rusoto_credential", "rusoto_ebs", "rusoto_ec2", - "rusoto_signature 0.45.0", - "sha2 0.9.2", + "rusoto_signature", + "sha2", "snafu", "tempfile", "tokio", @@ -391,24 +358,14 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" -dependencies = [ - "generic-array 0.12.3", - "subtle 1.0.0", -] - [[package]] name = "crypto-mac" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "generic-array 0.14.4", - "subtle 2.4.0", + "generic-array", + "subtle", ] [[package]] @@ -420,22 +377,13 @@ dependencies = [ "sct", ] -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.3", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.4", + "generic-array", ] [[package]] @@ -489,6 +437,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "dyn-clone" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -504,12 +458,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fnv" version = "1.0.7" @@ -637,15 +585,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.4" @@ -747,24 +686,14 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" -[[package]] -name = "hmac" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" -dependencies = [ - "crypto-mac 0.7.0", - "digest 0.8.1", -] - [[package]] name = "hmac" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" dependencies = [ - "crypto-mac 0.8.0", - "digest 0.9.0", + "crypto-mac", + "digest", ] [[package]] @@ -1135,12 +1064,6 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" @@ -1322,12 +1245,12 @@ dependencies = [ "parse-datetime", "pubsys-config", "reqwest", - "rusoto_core 0.45.0", - "rusoto_credential 0.45.0", + "rusoto_core", + "rusoto_credential", "rusoto_ebs", "rusoto_ec2", - "rusoto_signature 0.45.0", - "rusoto_ssm 0.45.0", + "rusoto_signature", + "rusoto_ssm", "rusoto_sts", "semver 0.11.0", "serde", @@ -1339,7 +1262,7 @@ dependencies = [ "tinytemplate", "tokio", "toml", - "tough 0.9.0", + "tough", "tough-kms", "tough-ssm", "update_metadata", @@ -1366,7 +1289,7 @@ dependencies = [ "log", "pubsys-config", "reqwest", - "sha2 0.9.2", + "sha2", "shell-words", "simplelog 0.9.0", "snafu", @@ -1561,35 +1484,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "rusoto_core" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841ca8f73e7498ba39146ab43acea906bbbb807d92ec0b7ea4b6293d2621f80d" -dependencies = [ - "async-trait", - "base64 0.12.3", - "bytes", - "futures", - "hmac 0.7.1", - "http", - "hyper", - "hyper-rustls 0.20.0", - "lazy_static", - "log", - "md5", - "percent-encoding", - "pin-project 0.4.27", - "rusoto_credential 0.44.0", - "rusoto_signature 0.44.0", - "rustc_version", - "serde", - "serde_json", - "sha2 0.8.2", - "tokio", - "xml-rs", -] - [[package]] name = "rusoto_core" version = "0.45.0" @@ -1609,8 +1503,8 @@ dependencies = [ "md5", "percent-encoding", "pin-project 0.4.27", - "rusoto_credential 0.45.0", - "rusoto_signature 0.45.0", + "rusoto_credential", + "rusoto_signature", "rustc_version", "serde", "serde_json", @@ -1618,26 +1512,6 @@ dependencies = [ "xml-rs", ] -[[package]] -name = "rusoto_credential" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60669ddc1bdbb83ce225593649d36b4c5f6bf9db47cc1ab3e81281abffc853f4" -dependencies = [ - "async-trait", - "chrono", - "dirs", - "futures", - "hyper", - "pin-project 0.4.27", - "regex", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - [[package]] name = "rusoto_credential" version = "0.45.0" @@ -1667,7 +1541,7 @@ dependencies = [ "async-trait", "bytes", "futures", - "rusoto_core 0.45.0", + "rusoto_core", "serde", "serde_derive", "serde_json", @@ -1682,7 +1556,7 @@ dependencies = [ "async-trait", "bytes", "futures", - "rusoto_core 0.45.0", + "rusoto_core", "serde_urlencoded 0.6.1", "xml-rs", ] @@ -1696,36 +1570,11 @@ dependencies = [ "async-trait", "bytes", "futures", - "rusoto_core 0.45.0", + "rusoto_core", "serde", "serde_json", ] -[[package]] -name = "rusoto_signature" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eddff187ac18c5a91d9ccda9353f30cf531620dce437c4db661dfe2e23b2029" -dependencies = [ - "base64 0.12.3", - "bytes", - "futures", - "hex", - "hmac 0.7.1", - "http", - "hyper", - "log", - "md5", - "percent-encoding", - "pin-project 0.4.27", - "rusoto_credential 0.44.0", - "rustc_version", - "serde", - "sha2 0.8.2", - "time 0.2.23", - "tokio", -] - [[package]] name = "rusoto_signature" version = "0.45.0" @@ -1736,35 +1585,21 @@ dependencies = [ "bytes", "futures", "hex", - "hmac 0.8.1", + "hmac", "http", "hyper", "log", "md5", "percent-encoding", "pin-project 0.4.27", - "rusoto_credential 0.45.0", + "rusoto_credential", "rustc_version", "serde", - "sha2 0.9.2", + "sha2", "time 0.2.23", "tokio", ] -[[package]] -name = "rusoto_ssm" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9224ad97be05dae1a0f6745252f3fa1430d6bea97c93f59e99edaeb7d70f5d" -dependencies = [ - "async-trait", - "bytes", - "futures", - "rusoto_core 0.44.0", - "serde", - "serde_json", -] - [[package]] name = "rusoto_ssm" version = "0.45.0" @@ -1774,7 +1609,7 @@ dependencies = [ "async-trait", "bytes", "futures", - "rusoto_core 0.45.0", + "rusoto_core", "serde", "serde_json", ] @@ -1789,7 +1624,7 @@ dependencies = [ "bytes", "chrono", "futures", - "rusoto_core 0.45.0", + "rusoto_core", "serde_urlencoded 0.6.1", "tempfile", "xml-rs", @@ -2022,29 +1857,17 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha2" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" dependencies = [ - "block-buffer 0.9.0", + "block-buffer", "cfg-if 1.0.0", "cpuid-bool", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest", + "opaque-debug", ] [[package]] @@ -2233,12 +2056,6 @@ dependencies = [ "syn", ] -[[package]] -name = "subtle" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" - [[package]] name = "subtle" version = "2.4.0" @@ -2465,33 +2282,12 @@ dependencies = [ [[package]] name = "tough" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b8d86994e9da2233fc30c54223bc448a15bdb782f8060c66107fc6b88619ba" -dependencies = [ - "chrono", - "globset", - "hex", - "log", - "olpc-cjson", - "pem", - "ring", - "serde", - "serde_json", - "serde_plain", - "snafu", - "untrusted", - "url", - "walkdir", -] - -[[package]] -name = "tough" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc05d902ccf136ba55d5e2c7222ddc1623f657e6add3f030e93c4dc5341bbdb7" +checksum = "7dc3534fa46badec98ac633028f47a3cea590e9c9a63d85bd15a0436f8b6eb94" dependencies = [ "chrono", + "dyn-clone", "globset", "hex", "log", @@ -2503,6 +2299,7 @@ dependencies = [ "serde_json", "serde_plain", "snafu", + "tempfile", "untrusted", "url", "walkdir", @@ -2510,35 +2307,34 @@ dependencies = [ [[package]] name = "tough-kms" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb9491d4cc35dafcb72c69254c27da97bad4d5796305ffe9974ed1f470ffc39" +checksum = "1aeb5ad37ac31ba5b10f4f53a7c1073a1a28a88dd4537c9abff8718148b95f37" dependencies = [ - "bytes", "pem", "ring", - "rusoto_core 0.45.0", - "rusoto_credential 0.45.0", + "rusoto_core", + "rusoto_credential", "rusoto_kms", "snafu", "tokio", - "tough 0.9.0", + "tough", ] [[package]] name = "tough-ssm" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b7e51b42318a756ab7cbcf036f2adb23a8462d762519d5a3e50b886430ed23" +checksum = "42cb7ac27150db2f2321adb2ac05a9b2e1f8d9ad739262c2779f298083e96eea" dependencies = [ - "rusoto_core 0.44.0", - "rusoto_credential 0.44.0", - "rusoto_ssm 0.44.0", + "rusoto_core", + "rusoto_credential", + "rusoto_ssm", "serde", "serde_json", "snafu", "tokio", - "tough 0.9.0", + "tough", ] [[package]] @@ -2660,7 +2456,7 @@ dependencies = [ "serde_plain", "snafu", "toml", - "tough 0.8.0", + "tough", ] [[package]] diff --git a/tools/pubsys/Cargo.toml b/tools/pubsys/Cargo.toml index 8c147bfec31..94aeb5f52ac 100644 --- a/tools/pubsys/Cargo.toml +++ b/tools/pubsys/Cargo.toml @@ -35,9 +35,9 @@ structopt = { version = "0.3", default-features = false } tinytemplate = "1.1" tokio = { version = "0.2.21", features = ["time"] } toml = "0.5" -tough = { version = "0.9", features = ["http"] } -tough-kms = "0.1.1" -tough-ssm = "0.4" +tough = { version = "0.10", features = ["http"] } +tough-kms = "0.2" +tough-ssm = "0.5" update_metadata = { path = "../../sources/updater/update_metadata/" } url = { version = "2.1.0", features = ["serde"] } tempfile = "3.1" diff --git a/tools/pubsys/src/repo.rs b/tools/pubsys/src/repo.rs index 159dde96706..385b6ac1622 100644 --- a/tools/pubsys/src/repo.rs +++ b/tools/pubsys/src/repo.rs @@ -2,7 +2,6 @@ pub(crate) mod check_expirations; pub(crate) mod refresh_repo; -mod transport; pub(crate) mod validate_repo; use crate::{friendly_version, Args}; @@ -18,17 +17,16 @@ use std::fs::{self, File}; use std::num::NonZeroU64; use std::path::{Path, PathBuf}; use structopt::StructOpt; -use tempfile::{tempdir, NamedTempFile}; +use tempfile::NamedTempFile; use tough::{ editor::signed::PathExists, editor::RepositoryEditor, key_source::{KeySource, LocalKeySource}, schema::Target, - ExpirationEnforcement, Limits, Repository, Settings, + RepositoryLoader, TransportErrorKind, }; use tough_kms::{KmsKeySource, KmsSigningAlgorithm}; use tough_ssm::SsmKeySource; -use transport::RepoTransport; use update_metadata::{Images, Manifest, Release, UpdateWaves}; use url::Url; @@ -187,8 +185,8 @@ fn update_manifest(repo_args: &RepoArgs, manifest: &mut Manifest) -> Result<()> /// Set expirations of all non-root role metadata based on a given `RepoExpirationPolicy` and an /// expiration start time -fn set_expirations<'a>( - editor: &mut RepositoryEditor<'a, RepoTransport>, +fn set_expirations( + editor: &mut RepositoryEditor, expiration_policy: &RepoExpirationPolicy, expiration_start_time: DateTime, ) -> Result<()> { @@ -211,7 +209,7 @@ fn set_expirations<'a>( } /// Set versions of all role metadata; the version will be the UNIX timestamp of the current time. -fn set_versions<'a>(editor: &mut RepositoryEditor<'a, RepoTransport>) -> Result<()> { +fn set_versions(editor: &mut RepositoryEditor) -> Result<()> { let seconds = Utc::now().timestamp(); let unsigned_seconds = seconds.try_into().expect("System clock before 1970??"); let version = NonZeroU64::new(unsigned_seconds).expect("System clock exactly 1970??"); @@ -228,7 +226,7 @@ fn set_versions<'a>(editor: &mut RepositoryEditor<'a, RepoTransport>) -> Result< /// Adds targets, expirations, and version to the RepositoryEditor fn update_editor<'a, P>( repo_args: &'a RepoArgs, - editor: &mut RepositoryEditor<'a, RepoTransport>, + editor: &mut RepositoryEditor, targets: impl Iterator, manifest_path: P, ) -> Result<()> @@ -328,30 +326,25 @@ fn repo_urls<'a>( /// that the repo does not exist. fn load_editor_and_manifest<'a, P>( root_role_path: P, - transport: &'a RepoTransport, - datastore: &'a Path, metadata_url: &'a Url, targets_url: &'a Url, -) -> Result, Manifest)>> +) -> Result> where P: AsRef, { let root_role_path = root_role_path.as_ref(); - // Create a temporary directory where the TUF client can store metadata - let settings = Settings { - root: File::open(root_role_path).context(error::File { + // Try to load the repo... + let repo_load_result = RepositoryLoader::new( + File::open(root_role_path).context(error::File { path: root_role_path, })?, - datastore, - metadata_base_url: metadata_url.as_str(), - targets_base_url: targets_url.as_str(), - limits: Limits::default(), - expiration_enforcement: ExpirationEnforcement::Safe, - }; + metadata_url.clone(), + targets_url.clone(), + ) + .load(); - // Try to load the repo... - match Repository::load(transport, settings) { + match repo_load_result { // If we load it successfully, build an editor and manifest from it. Ok(repo) => { let reader = repo @@ -374,17 +367,26 @@ where // If we fail to load, but we only failed because the repo doesn't exist yet, then start // fresh by signalling that there is no known repo. Otherwise, fail hard. Err(e) => { - if transport.repo_not_found.get() { + if is_file_not_found_error(&e) { Ok(None) } else { Err(e).with_context(|| error::RepoLoad { metadata_base_url: metadata_url.clone(), - })? + }) } } } } +/// Inspects the `tough` error to see if it is a `Transport` error, and if so, is it `FileNotFound`. +fn is_file_not_found_error(e: &tough::error::Error) -> bool { + if let tough::error::Error::Transport { source, .. } = e { + matches!(source.kind(), TransportErrorKind::FileNotFound) + } else { + false + } +} + /// Gets the corresponding `KeySource` according to the signing key config from Infra.toml fn get_signing_key_source(signing_key_config: &SigningKeyConfig) -> Box { match signing_key_config { @@ -443,24 +445,19 @@ pub(crate) fn run(args: &Args, repo_args: &RepoArgs) -> Result<()> { }) { repo_config } else { - info!("Didn't find repo '{}' in Infra.toml, using default configuration", repo_args.repo); + info!( + "Didn't find repo '{}' in Infra.toml, using default configuration", + repo_args.repo + ); &default_repo_config }; // Build a repo editor and manifest, from an existing repo if available, otherwise fresh let maybe_urls = repo_urls(&repo_config, &repo_args.variant, &repo_args.arch)?; - let workdir = tempdir().context(error::TempDir)?; - let transport = RepoTransport::default(); let (mut editor, mut manifest) = if let Some((metadata_url, targets_url)) = maybe_urls.as_ref() { info!("Found metadata and target URLs, loading existing repository"); - match load_editor_and_manifest( - &repo_args.root_role_path, - &transport, - workdir.path(), - &metadata_url, - &targets_url, - )? { + match load_editor_and_manifest(&repo_args.root_role_path, &metadata_url, &targets_url)? { Some((editor, manifest)) => (editor, manifest), None => { info!( @@ -729,9 +726,6 @@ mod error { source: update_metadata::error::Error, }, - #[snafu(display("Failed to create tempdir: {}", source))] - TempDir { source: io::Error }, - #[snafu(display("Failed to create temporary file: {}", source))] TempFile { source: io::Error }, diff --git a/tools/pubsys/src/repo/check_expirations/mod.rs b/tools/pubsys/src/repo/check_expirations/mod.rs index cdd9e9a9b30..dd726054737 100644 --- a/tools/pubsys/src/repo/check_expirations/mod.rs +++ b/tools/pubsys/src/repo/check_expirations/mod.rs @@ -1,7 +1,6 @@ //! The check_expirations module owns the 'check-repo-expirations' subcommand and provide methods for //! checking the metadata expirations of a given TUF repository. -use super::RepoTransport; use crate::repo::{error as repo_error, repo_urls}; use crate::Args; use chrono::{DateTime, Utc}; @@ -13,8 +12,7 @@ use std::collections::HashMap; use std::fs::File; use std::path::PathBuf; use structopt::StructOpt; -use tempfile::tempdir; -use tough::{ExpirationEnforcement, Limits, Repository, Settings}; +use tough::{ExpirationEnforcement, Repository, RepositoryLoader}; use url::Url; /// Checks for metadata expirations for a set of TUF repositories @@ -42,13 +40,10 @@ pub(crate) struct CheckExpirationsArgs { } /// Checks for upcoming role expirations, gathering them in a map of role to expiration datetime. -fn find_upcoming_metadata_expiration( - repo: &Repository<'_, T>, +fn find_upcoming_metadata_expiration( + repo: &Repository, end_date: DateTime, -) -> HashMap> -where - T: tough::Transport, -{ +) -> HashMap> { let mut expirations = HashMap::new(); info!( "Looking for metadata expirations happening from now to {}", @@ -80,28 +75,23 @@ where } fn check_expirations( - transport: &RepoTransport, root_role_path: &PathBuf, metadata_url: &Url, targets_url: &Url, expiration_limit: DateTime, ) -> Result<()> { - // Create a temporary directory where the TUF client can store metadata - let workdir = tempdir().context(repo_error::TempDir)?; - let settings = Settings { - root: File::open(root_role_path).context(repo_error::File { + // Load the repository + let repo = RepositoryLoader::new( + File::open(root_role_path).context(repo_error::File { path: root_role_path, })?, - datastore: workdir.path(), - metadata_base_url: metadata_url.as_str(), - targets_base_url: targets_url.as_str(), - limits: Limits::default(), - // We're gonna check the expiration ourselves - expiration_enforcement: ExpirationEnforcement::Unsafe, - }; - - // Load the repository - let repo = Repository::load(transport, settings).context(repo_error::RepoLoad { + metadata_url.clone(), + targets_url.clone(), + ) + // We're gonna check the expiration ourselves + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .load() + .context(repo_error::RepoLoad { metadata_base_url: metadata_url.clone(), })?; info!("Loaded TUF repo:\t{}", metadata_url); @@ -158,7 +148,6 @@ pub(crate) fn run(args: &Args, check_expirations_args: &CheckExpirationsArgs) -> missing: format!("definition for repo {}", &check_expirations_args.repo), })?; - let transport = RepoTransport::default(); let repo_urls = repo_urls( &repo_config, &check_expirations_args.variant, @@ -168,7 +157,6 @@ pub(crate) fn run(args: &Args, check_expirations_args: &CheckExpirationsArgs) -> repo: &check_expirations_args.repo, })?; check_expirations( - &transport, &check_expirations_args.root_role_path, &repo_urls.0, repo_urls.1, diff --git a/tools/pubsys/src/repo/refresh_repo/mod.rs b/tools/pubsys/src/repo/refresh_repo/mod.rs index 80844422f60..a198ae9342f 100644 --- a/tools/pubsys/src/repo/refresh_repo/mod.rs +++ b/tools/pubsys/src/repo/refresh_repo/mod.rs @@ -1,7 +1,6 @@ //! The refresh_repo module owns the 'refresh-repo' subcommand and provide methods for //! refreshing and re-signing the metadata files of a given TUF repository. -use super::RepoTransport; use crate::repo::{ error as repo_error, get_signing_key_source, repo_urls, set_expirations, set_versions, }; @@ -15,10 +14,9 @@ use std::fs; use std::fs::File; use std::path::{Path, PathBuf}; use structopt::StructOpt; -use tempfile::tempdir; use tough::editor::RepositoryEditor; use tough::key_source::KeySource; -use tough::{ExpirationEnforcement, Limits, Repository, Settings}; +use tough::{ExpirationEnforcement, RepositoryLoader}; use url::Url; lazy_static! { @@ -59,7 +57,6 @@ pub(crate) struct RefreshRepoArgs { } fn refresh_repo( - transport: &RepoTransport, root_role_path: &PathBuf, metadata_out_dir: &PathBuf, metadata_url: &Url, @@ -77,25 +74,23 @@ fn refresh_repo( } ); - // Create a temporary directory where the TUF client can store metadata - let workdir = tempdir().context(repo_error::TempDir)?; - let settings = Settings { - root: File::open(root_role_path).context(repo_error::File { - path: root_role_path, - })?, - datastore: workdir.path(), - metadata_base_url: metadata_url.as_str(), - targets_base_url: targets_url.as_str(), - limits: Limits::default(), - expiration_enforcement: if unsafe_refresh { - ExpirationEnforcement::Unsafe - } else { - ExpirationEnforcement::Safe - }, + let expiration_enforcement = if unsafe_refresh { + ExpirationEnforcement::Unsafe + } else { + ExpirationEnforcement::Safe }; // Load the repository and get the repo editor for it - let repo = Repository::load(transport, settings).context(repo_error::RepoLoad { + let repo = RepositoryLoader::new( + File::open(root_role_path).context(repo_error::File { + path: root_role_path, + })?, + metadata_url.clone(), + targets_url.clone(), + ) + .expiration_enforcement(expiration_enforcement) + .load() + .context(repo_error::RepoLoad { metadata_base_url: metadata_url.clone(), })?; let mut repo_editor = @@ -167,7 +162,6 @@ pub(crate) fn run(args: &Args, refresh_repo_args: &RefreshRepoArgs) -> Result<() RepoExpirationPolicy::from_path(&refresh_repo_args.repo_expiration_policy_path) .context(repo_error::Config)?; - let transport = RepoTransport::default(); let repo_urls = repo_urls( &repo_config, &refresh_repo_args.variant, @@ -177,7 +171,6 @@ pub(crate) fn run(args: &Args, refresh_repo_args: &RefreshRepoArgs) -> Result<() repo: &refresh_repo_args.repo, })?; refresh_repo( - &transport, &refresh_repo_args.root_role_path, &refresh_repo_args .outdir diff --git a/tools/pubsys/src/repo/transport.rs b/tools/pubsys/src/repo/transport.rs deleted file mode 100644 index e889dc4850c..00000000000 --- a/tools/pubsys/src/repo/transport.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::error; -use std::cell::Cell; -use std::io::Read; -use tough::{FilesystemTransport, HttpTransport, Transport}; -use url::Url; - -/// RepoTransport delegates to FilesystemTransport or HttpTransport based on the url scheme. If we -/// detect that the repo isn't found we return a special error so we can start a new repo. -#[derive(Debug, Default, Clone)] -pub(crate) struct RepoTransport { - // If we fail to fetch the repo, we need a way of conveying whether it happened because the - // repo doesn't exist or because we failed to fetch/load a repo that does exist. This - // information can be used to determine whether we want to start a new repo from scratch or to - // fail early, for example. - // - // tough uses a trait object to represent the source error inside its Error::Transport variant, - // so we can't check our own, inner error type to determine which of our variants is inside. - // Also, it defines the `fetch` method of `Transport` to take an immutable reference to self, - // so we can't use a struct field naively to communicate back. - // - // So, we use this Cell to safely convey the information outward in our single-threaded usage. - pub(crate) repo_not_found: Cell, -} - -impl Transport for RepoTransport { - type Stream = Box; - type Error = error::Error; - - fn fetch(&self, url: Url) -> std::result::Result { - if url.scheme() == "file" { - match FilesystemTransport.fetch(url.clone()) { - Ok(reader) => Ok(Box::new(reader)), - Err(e) => match e.kind() { - std::io::ErrorKind::NotFound => { - self.repo_not_found.set(true); - error::RepoNotFound { url }.fail() - } - _ => error::RepoFetch { - url, - msg: e.to_string(), - } - .fail(), - }, - } - } else { - let transport = HttpTransport::new(); - match transport.fetch(url.clone()) { - Ok(reader) => Ok(Box::new(reader)), - Err(e) => match e { - tough::error::Error::HttpFetch { .. } => { - self.repo_not_found.set(true); - error::RepoNotFound { url }.fail() - } - _ => error::RepoFetch { - url, - msg: e.to_string(), - } - .fail(), - }, - } - } - } -} diff --git a/tools/pubsys/src/repo/validate_repo/mod.rs b/tools/pubsys/src/repo/validate_repo/mod.rs index b2b0d04b022..9391eab213a 100644 --- a/tools/pubsys/src/repo/validate_repo/mod.rs +++ b/tools/pubsys/src/repo/validate_repo/mod.rs @@ -1,7 +1,6 @@ //! The validate_repo module owns the 'validate-repo' subcommand and provides methods for validating //! a given TUF repository by attempting to load the repository and download its targets. -use super::RepoTransport; use crate::repo::{error as repo_error, repo_urls}; use crate::Args; use futures::future::join_all; @@ -12,8 +11,7 @@ use std::fs::File; use std::io; use std::path::PathBuf; use structopt::StructOpt; -use tempfile::tempdir; -use tough::{ExpirationEnforcement, Limits, Repository, Settings}; +use tough::{Repository, RepositoryLoader}; use url::Url; /// Validates a set of TUF repositories @@ -40,21 +38,12 @@ pub(crate) struct ValidateRepoArgs { validate_targets: bool, } -/// Retrieves listed targets and attempt to download them for validation purposes -async fn retrieve_targets( - repo: &Repository<'_, T>, -) -> Result<(), Error> -where - T: tough::Transport, - ::Stream: std::marker::Send, -{ +/// Retrieves listed targets and attempts to download them for validation purposes +async fn retrieve_targets(repo: &Repository) -> Result<(), Error> { let targets = &repo.targets().signed.targets; let mut tasks = Vec::new(); - for target in targets - .keys() - .cloned() - { + for target in targets.keys().cloned() { let target = target.to_string(); let mut reader = repo .read_target(&target) @@ -81,27 +70,21 @@ where } async fn validate_repo( - transport: &RepoTransport, root_role_path: &PathBuf, metadata_url: Url, targets_url: &Url, validate_targets: bool, ) -> Result<(), Error> { - // Create a temporary directory where the TUF client can store metadata - let workdir = tempdir().context(repo_error::TempDir)?; - let settings = Settings { - root: File::open(root_role_path).context(repo_error::File { + // Load the repository + let repo = RepositoryLoader::new( + File::open(root_role_path).context(repo_error::File { path: root_role_path, })?, - datastore: workdir.path(), - metadata_base_url: metadata_url.as_str(), - targets_base_url: targets_url.as_str(), - limits: Limits::default(), - expiration_enforcement: ExpirationEnforcement::Safe, - }; - - // Load the repository - let repo = Repository::load(transport, settings).context(repo_error::RepoLoad { + metadata_url.clone(), + targets_url.clone(), + ) + .load() + .context(repo_error::RepoLoad { metadata_base_url: metadata_url.clone(), })?; info!("Loaded TUF repo: {}", metadata_url); @@ -133,7 +116,6 @@ pub(crate) async fn run(args: &Args, validate_repo_args: &ValidateRepoArgs) -> R missing: format!("definition for repo {}", &validate_repo_args.repo), })?; - let transport = RepoTransport::default(); let repo_urls = repo_urls( &repo_config, &validate_repo_args.variant, @@ -143,7 +125,6 @@ pub(crate) async fn run(args: &Args, validate_repo_args: &ValidateRepoArgs) -> R repo: &validate_repo_args.repo, })?; validate_repo( - &transport, &validate_repo_args.root_role_path, repo_urls.0, repo_urls.1,