From 3cc68894a3a8917e5e74b58585f33c32be6ed86f Mon Sep 17 00:00:00 2001 From: Jakub Wieczorek Date: Sun, 19 Jul 2020 14:31:38 +0200 Subject: [PATCH 1/2] Add support for key prefixes when using S3 as the storage The corresponding environment variable is `SCCACHE_S3_KEY_PREFIX`. --- README.md | 2 ++ src/cache/cache.rs | 2 +- src/cache/s3.rs | 16 +++++++++------- src/config.rs | 18 ++++++++++++++---- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8c30ba1f4..32bd490d4 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,8 @@ You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 creden If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=:`. If your endpoint requires TLS, set `SCCACHE_S3_USE_SSL=true`. +You can also define a prefix that will be prepended to the keys of all cache objects created and read within the S3 bucket, effectively creating a scope. To do that use the `SCCACHE_S3_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. + ### Redis Set `SCCACHE_REDIS` to a [Redis](https://redis.io/) url in format `redis://[:@][:port][/]` to store the cache in a Redis instance. Redis can be configured as a LRU (least recently used) cache with a fixed maximum cache size. Set `maxmemory` and `maxmemory-policy` according to the [Redis documentation](https://redis.io/topics/lru-cache). The `allkeys-lru` policy which discards the *least recently accessed or modified* key fits well for the sccache use case. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 1b55e4220..f9503ae22 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -377,7 +377,7 @@ pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc { debug!("Trying S3Cache({}, {})", c.bucket, c.endpoint); #[cfg(feature = "s3")] - match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl) { + match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl, &c.key_prefix) { Ok(s) => { trace!("Using S3Cache"); return Arc::new(s); diff --git a/src/cache/s3.rs b/src/cache/s3.rs index 866c9e016..1f4454f37 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -31,11 +31,13 @@ pub struct S3Cache { bucket: Rc, /// Credentials provider. provider: AutoRefreshingProvider, + /// Prefix to be used for bucket keys. + key_prefix: String, } impl S3Cache { /// Create a new `S3Cache` storing data in `bucket`. - pub fn new(bucket: &str, endpoint: &str, use_ssl: bool) -> Result { + pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result { let user_dirs = UserDirs::new().context("Couldn't get user directories")?; let home = user_dirs.home_dir(); @@ -50,17 +52,17 @@ impl S3Cache { AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); - Ok(S3Cache { bucket, provider }) + Ok(S3Cache { bucket, provider, key_prefix: key_prefix.to_owned() }) } -} -fn normalize_key(key: &str) -> String { - format!("{}/{}/{}/{}", &key[0..1], &key[1..2], &key[2..3], &key) + fn normalize_key(&self, key: &str) -> String { + format!("{}{}/{}/{}/{}", &self.key_prefix, &key[0..1], &key[1..2], &key[2..3], &key) + } } impl Storage for S3Cache { fn get(&self, key: &str) -> SFuture { - let key = normalize_key(key); + let key = self.normalize_key(key); let result_cb = |result| match result { Ok(data) => { @@ -89,7 +91,7 @@ impl Storage for S3Cache { } fn put(&self, key: &str, entry: CacheWrite) -> SFuture { - let key = normalize_key(&key); + let key = self.normalize_key(&key); let start = Instant::now(); let data = match entry.finish() { Ok(data) => data, diff --git a/src/config.rs b/src/config.rs index 859d8b149..b3ded9215 100644 --- a/src/config.rs +++ b/src/config.rs @@ -199,6 +199,7 @@ pub struct S3CacheConfig { pub bucket: String, pub endpoint: String, pub use_ssl: bool, + pub key_prefix: String, } #[derive(Debug, PartialEq, Eq)] @@ -455,14 +456,23 @@ fn config_from_env() -> EnvConfig { _ => format!("{}.s3.amazonaws.com", bucket), }, }; - let use_ssl = match env::var("SCCACHE_S3_USE_SSL") { - Ok(ref value) if value != "off" => true, - _ => false, - }; + let use_ssl = env::var("SCCACHE_S3_USE_SSL") + .ok() + .filter(|value| value != "off") + .is_some(); + let key_prefix = env::var("SCCACHE_S3_KEY_PREFIX") + .ok() + .as_ref() + .map(|s| s.trim_end_matches("/")) + .filter(|s| !s.is_empty()) + .map(|s| s.to_owned() + "/") + .unwrap_or_default(); + S3CacheConfig { bucket, endpoint, use_ssl, + key_prefix, } }); From 253f2c394f8b4481f168021df1d93de9a72c970e Mon Sep 17 00:00:00 2001 From: Jakub Wieczorek Date: Mon, 20 Jul 2020 18:13:21 +0200 Subject: [PATCH 2/2] Fix up style --- src/cache/s3.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/cache/s3.rs b/src/cache/s3.rs index 1f4454f37..b08fcee9e 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -52,11 +52,22 @@ impl S3Cache { AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); - Ok(S3Cache { bucket, provider, key_prefix: key_prefix.to_owned() }) + Ok(S3Cache { + bucket, + provider, + key_prefix: key_prefix.to_owned(), + }) } fn normalize_key(&self, key: &str) -> String { - format!("{}{}/{}/{}/{}", &self.key_prefix, &key[0..1], &key[1..2], &key[2..3], &key) + format!( + "{}{}/{}/{}/{}", + &self.key_prefix, + &key[0..1], + &key[1..2], + &key[2..3], + &key + ) } }