[object_store] Support MD5 checksum in attributes

object_store/src/attributes.rs

@@ -18,6 +18,7 @@
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::ops::Deref;
+use crate::checksum::ChecksumAlgorithm;
 
 /// Additional object attribute types
 #[non_exhaustive]
@@ -45,6 +46,8 @@ pub enum Attribute {
     ///
     /// See [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control)
     CacheControl,
+    /// Provides a checksum used to verify object data integrity
+    Checksum(ChecksumAlgorithm),
     /// Specifies a user-defined metadata field for the object
     ///
     /// The String is a user-defined key
@@ -188,21 +191,23 @@ impl<'a> Iterator for AttributesIter<'a> {
 
 #[cfg(test)]
 mod tests {
+    use crate::checksum::ChecksumAlgorithm;
     use super::*;
 
     #[test]
     fn test_attributes_basic() {
         let mut attributes = Attributes::from_iter([
+            (Attribute::CacheControl, "control"),
+            (Attribute::Checksum(ChecksumAlgorithm::MD5), "checksumValue"),
             (Attribute::ContentDisposition, "inline"),
             (Attribute::ContentEncoding, "gzip"),
             (Attribute::ContentLanguage, "en-US"),
             (Attribute::ContentType, "test"),
-            (Attribute::CacheControl, "control"),
             (Attribute::Metadata("key1".into()), "value1"),
         ]);
 
         assert!(!attributes.is_empty());
-        assert_eq!(attributes.len(), 6);
+        assert_eq!(attributes.len(), 7);
 
         assert_eq!(
             attributes.get(&Attribute::ContentType),
@@ -215,19 +220,23 @@ mod tests {
             attributes.insert(Attribute::CacheControl, "v1".into()),
             Some(metav)
         );
-        assert_eq!(attributes.len(), 6);
+        assert_eq!(attributes.len(), 7);
 
         assert_eq!(
             attributes.remove(&Attribute::CacheControl).unwrap(),
             "v1".into()
         );
-        assert_eq!(attributes.len(), 5);
+        assert_eq!(attributes.len(), 6);
 
         let metav: AttributeValue = "v2".into();
         attributes.insert(Attribute::CacheControl, metav.clone());
         assert_eq!(attributes.get(&Attribute::CacheControl), Some(&metav));
-        assert_eq!(attributes.len(), 6);
+        assert_eq!(attributes.len(), 7);
 
+        assert_eq!(
+            attributes.get(&Attribute::Checksum(ChecksumAlgorithm::MD5)),
+            Some(&"checksumValue".into())
+        );
         assert_eq!(
             attributes.get(&Attribute::ContentDisposition),
             Some(&"inline".into())

object_store/src/aws/client.rs

@@ -34,10 +34,7 @@ use crate::client::s3::{
 use crate::client::GetOptionsExt;
 use crate::multipart::PartId;
 use crate::path::DELIMITER;
-use crate::{
-    Attribute, Attributes, ClientOptions, GetOptions, ListResult, MultipartId, Path,
-    PutMultipartOpts, PutPayload, PutResult, Result, RetryConfig, TagSet,
-};
+use crate::{Attribute, Attributes, ClientOptions, GetOptions, ListResult, MultipartId, Path, PutMultipartOpts, PutPayload, PutResult, Result, RetryConfig, TagSet};
 use async_trait::async_trait;
 use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
@@ -58,9 +55,11 @@ use ring::digest::Context;
 use serde::{Deserialize, Serialize};
 use snafu::{ResultExt, Snafu};
 use std::sync::Arc;
+use crate::checksum::ChecksumAlgorithm;
 
 const VERSION_HEADER: &str = "x-amz-version-id";
 const SHA256_CHECKSUM: &str = "x-amz-checksum-sha256";
+const CHECKSUM_MD5_HEADER: &str = "Content-MD5";
 const USER_DEFINED_METADATA_HEADER_PREFIX: &str = "x-amz-meta-";
 const ALGORITHM: &str = "x-amz-checksum-algorithm";
 
@@ -360,6 +359,11 @@ impl<'a> Request<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(CACHE_CONTROL, v.as_ref()),
+                Attribute::Checksum(algorithm) => {
+                    match algorithm {
+                        ChecksumAlgorithm::MD5 => builder.header(CHECKSUM_MD5_HEADER, v.as_ref()),
+                    }
+                }
                 Attribute::ContentDisposition => builder.header(CONTENT_DISPOSITION, v.as_ref()),
                 Attribute::ContentEncoding => builder.header(CONTENT_ENCODING, v.as_ref()),
                 Attribute::ContentLanguage => builder.header(CONTENT_LANGUAGE, v.as_ref()),

object_store/src/azure/client.rs

@@ -47,6 +47,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 use url::Url;
+use crate::checksum::ChecksumAlgorithm;
 
 const VERSION_HEADER: &str = "x-ms-version-id";
 const USER_DEFINED_METADATA_HEADER_PREFIX: &str = "x-ms-meta-";
@@ -58,6 +59,7 @@ static MS_CONTENT_ENCODING: HeaderName = HeaderName::from_static("x-ms-blob-cont
 static MS_CONTENT_LANGUAGE: HeaderName = HeaderName::from_static("x-ms-blob-content-language");
 
 static TAGS_HEADER: HeaderName = HeaderName::from_static("x-ms-tags");
+static CHECKSUM_MD5_HEADER: HeaderName = HeaderName::from_static("Content-MD5");
 
 /// A specialized `Error` for object store-related errors
 #[derive(Debug, Snafu)]
@@ -228,6 +230,9 @@ impl<'a> PutRequest<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(&MS_CACHE_CONTROL, v.as_ref()),
+                Attribute::Checksum(algorithm) => match algorithm {
+                    ChecksumAlgorithm::MD5 => builder.header(&CHECKSUM_MD5_HEADER, v.as_ref()),
+                }
                 Attribute::ContentDisposition => {
                     builder.header(&MS_CONTENT_DISPOSITION, v.as_ref())
                 }

object_store/src/checksum.rs

@@ -0,0 +1,7 @@
+#[allow(non_camel_case_types)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+/// Enum representing checksum algorithms that may be used to verify object integrity.
+pub enum ChecksumAlgorithm {
+    /// MD5 algorithm.
+    MD5,
+}

object_store/src/gcp/client.rs

@@ -46,10 +46,12 @@ use reqwest::{Client, Method, RequestBuilder, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt, Snafu};
 use std::sync::Arc;
+use crate::checksum::ChecksumAlgorithm;
 
 const VERSION_HEADER: &str = "x-goog-generation";
 const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
 const USER_DEFINED_METADATA_HEADER_PREFIX: &str = "x-goog-meta-";
+const CHECKSUM_MD5_HEADER: &str = "Content-MD5";
 
 static VERSION_MATCH: HeaderName = HeaderName::from_static("x-goog-if-generation-match");
 
@@ -196,6 +198,9 @@ impl<'a> Request<'a> {
         for (k, v) in &attributes {
             builder = match k {
                 Attribute::CacheControl => builder.header(CACHE_CONTROL, v.as_ref()),
+                Attribute::Checksum(algorithm) => match algorithm{
+                    ChecksumAlgorithm::MD5 => builder.header(CHECKSUM_MD5_HEADER, v.as_ref()),
+                }
                 Attribute::ContentDisposition => builder.header(CONTENT_DISPOSITION, v.as_ref()),
                 Attribute::ContentEncoding => builder.header(CONTENT_ENCODING, v.as_ref()),
                 Attribute::ContentLanguage => builder.header(CONTENT_LANGUAGE, v.as_ref()),

object_store/src/lib.rs

@@ -545,11 +545,13 @@ mod upload;
 mod util;
 
 mod attributes;
+mod checksum;
 
 #[cfg(any(feature = "integration", test))]
 pub mod integration;
 
 pub use attributes::*;
+pub use checksum::*;
 
 pub use parse::{parse_url, parse_url_opts, ObjectStoreScheme};
 pub use payload::*;