feat(services/s3): Added crc64nvme for s3

core/Cargo.toml

@@ -192,6 +192,7 @@ services-s3 = [
     "reqsign?/services-aws",
     "reqsign?/reqwest_request",
     "dep:crc32c",
+    "dep:crc64fast-nvme",
 ]
 services-seafile = []
 services-sftp = ["dep:openssh", "dep:openssh-sftp-client", "dep:bb8"]
@@ -352,6 +353,7 @@ compio = { version = "0.12.0", optional = true, features = [
 ] }
 # for services-s3
 crc32c = { version = "0.6.6", optional = true }
+crc64fast-nvme = {version = "1.1.1", optional = true}
 # for services-nebula-graph
 rust-nebula = { version = "^0.0.2", optional = true, features = ["graph"] }
 snowflaked = { version = "1", optional = true, features = ["sync"] }

core/src/services/s3/backend.rs

@@ -750,6 +750,7 @@ impl Builder for S3Builder {
 
         let checksum_algorithm = match self.config.checksum_algorithm.as_deref() {
             Some("crc32c") => Some(ChecksumAlgorithm::Crc32c),
+            Some("crc64nvme") => Some(ChecksumAlgorithm::Crc64nvme),
             None => None,
             v => {
                 return Err(Error::new(

core/src/services/s3/config.rs

@@ -179,6 +179,7 @@ pub struct S3Config {
     ///
     /// Available options:
     /// - "crc32c"
+    /// - "crc64nvme"
     pub checksum_algorithm: Option<String>,
     /// Disable write with if match so that opendal will not send write request with if match headers.
     ///

core/src/services/s3/core.rs

@@ -267,6 +267,11 @@ impl S3Core {
                     .for_each(|b| crc = crc32c::crc32c_append(crc, &b));
                 Some(BASE64_STANDARD.encode(crc.to_be_bytes()))
             }
+            Some(ChecksumAlgorithm::Crc64nvme) => {
+                let mut c = crc64fast_nvme::Digest::new();
+                body.clone().for_each(|b| c.write(&b));
+                Some(BASE64_STANDARD.encode(c.sum64().to_be_bytes()))
+            }
         }
     }
     pub fn insert_checksum_header(
@@ -938,6 +943,8 @@ pub struct CompleteMultipartUploadRequestPart {
     pub etag: String,
     #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "Option::is_none")]
     pub checksum_crc32c: Option<String>,
+    #[serde(rename = "ChecksumCRC64NVME", skip_serializing_if = "Option::is_none")]
+    pub checksum_crc64nvme: Option<String>,
 }
 
 /// Request of DeleteObjects.
@@ -1046,11 +1053,13 @@ pub struct ListObjectVersionsOutputDeleteMarker {
 
 pub enum ChecksumAlgorithm {
     Crc32c,
+    Crc64nvme,
 }
 impl ChecksumAlgorithm {
     pub fn to_header_name(&self) -> HeaderName {
         match self {
             Self::Crc32c => HeaderName::from_static("x-amz-checksum-crc32c"),
+            Self::Crc64nvme => HeaderName::from_static("x-amz-checksum-crc64nvme"),
         }
     }
 }
@@ -1061,6 +1070,7 @@ impl Display for ChecksumAlgorithm {
             "{}",
             match self {
                 Self::Crc32c => "CRC32C",
+                Self::Crc64nvme => "CRC64NVME",
             }
         )
     }

core/src/services/s3/writer.rs

@@ -147,6 +147,13 @@ impl oio::MultipartWrite for S3Writer {
                         part_number: p.part_number,
                         etag: p.etag.clone(),
                         checksum_crc32c: p.checksum.clone(),
+                        ..Default::default()
+                    },
+                    ChecksumAlgorithm::Crc64nvme => CompleteMultipartUploadRequestPart {
+                        part_number: p.part_number,
+                        etag: p.etag.clone(),
+                        checksum_crc64nvme: p.checksum.clone(),
+                        ..Default::default()
                     },
                 },
             })