Add support for key prefixes when using S3 as the storage

README.md

@@ -196,6 +196,8 @@ You can use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to set the S3 creden
 
 If you need to override the default endpoint you can set `SCCACHE_ENDPOINT`. To connect to a minio storage for example you can set `SCCACHE_ENDPOINT=<ip>:<port>`. If your endpoint requires TLS, set `SCCACHE_S3_USE_SSL=true`.
 
+You can also define a prefix that will be prepended to the keys of all cache objects created and read within the S3 bucket, effectively creating a scope. To do that use the `SCCACHE_S3_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application.
+
 
 ### Redis
 Set `SCCACHE_REDIS` to a [Redis](https://redis.io/) url in format `redis://[:<passwd>@]<hostname>[:port][/<db>]` to store the cache in a Redis instance. Redis can be configured as a LRU (least recently used) cache with a fixed maximum cache size. Set `maxmemory` and `maxmemory-policy` according to the [Redis documentation](https://redis.io/topics/lru-cache). The `allkeys-lru` policy which discards the *least recently accessed or modified* key fits well for the sccache use case.

src/cache/cache.rs

@@ -377,7 +377,7 @@ pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc<dyn Storag
             CacheType::S3(ref c) => {
                 debug!("Trying S3Cache({}, {})", c.bucket, c.endpoint);
                 #[cfg(feature = "s3")]
-                match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl) {
+                match S3Cache::new(&c.bucket, &c.endpoint, c.use_ssl, &c.key_prefix) {
                     Ok(s) => {
                         trace!("Using S3Cache");
                         return Arc::new(s);

src/cache/s3.rs

@@ -31,11 +31,13 @@ pub struct S3Cache {
     bucket: Rc<Bucket>,
     /// Credentials provider.
     provider: AutoRefreshingProvider<ChainProvider>,
+    /// Prefix to be used for bucket keys.
+    key_prefix: String,
 }
 
 impl S3Cache {
     /// Create a new `S3Cache` storing data in `bucket`.
-    pub fn new(bucket: &str, endpoint: &str, use_ssl: bool) -> Result<S3Cache> {
+    pub fn new(bucket: &str, endpoint: &str, use_ssl: bool, key_prefix: &str) -> Result<S3Cache> {
         let user_dirs = UserDirs::new().context("Couldn't get user directories")?;
         let home = user_dirs.home_dir();
 
@@ -50,17 +52,17 @@ impl S3Cache {
             AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers));
         let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No };
         let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?);
-        Ok(S3Cache { bucket, provider })
+        Ok(S3Cache { bucket, provider, key_prefix: key_prefix.to_owned() })
     }
-}
 
-fn normalize_key(key: &str) -> String {
-    format!("{}/{}/{}/{}", &key[0..1], &key[1..2], &key[2..3], &key)
+    fn normalize_key(&self, key: &str) -> String {
+        format!("{}{}/{}/{}/{}", &self.key_prefix, &key[0..1], &key[1..2], &key[2..3], &key)
+    }
 }
 
 impl Storage for S3Cache {
     fn get(&self, key: &str) -> SFuture<Cache> {
-        let key = normalize_key(key);
+        let key = self.normalize_key(key);
 
         let result_cb = |result| match result {
             Ok(data) => {
@@ -89,7 +91,7 @@ impl Storage for S3Cache {
     }
 
     fn put(&self, key: &str, entry: CacheWrite) -> SFuture<Duration> {
-        let key = normalize_key(&key);
+        let key = self.normalize_key(&key);
         let start = Instant::now();
         let data = match entry.finish() {
             Ok(data) => data,

src/config.rs

@@ -199,6 +199,7 @@ pub struct S3CacheConfig {
     pub bucket: String,
     pub endpoint: String,
     pub use_ssl: bool,
+    pub key_prefix: String,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -455,14 +456,23 @@ fn config_from_env() -> EnvConfig {
                 _ => format!("{}.s3.amazonaws.com", bucket),
             },
         };
-        let use_ssl = match env::var("SCCACHE_S3_USE_SSL") {
-            Ok(ref value) if value != "off" => true,
-            _ => false,
-        };
+        let use_ssl = env::var("SCCACHE_S3_USE_SSL")
+            .ok()
+            .filter(|value| value != "off")
+            .is_some();
+        let key_prefix = env::var("SCCACHE_S3_KEY_PREFIX")
+            .ok()
+            .as_ref()
+            .map(|s| s.trim_end_matches("/"))
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_owned() + "/")
+            .unwrap_or_default();
+
         S3CacheConfig {
             bucket,
             endpoint,
             use_ssl,
+            key_prefix,
         }
     });