Merge pull request #142 from pingcap/cherry-pick-sm4

Cherry pick sm4
pingcap · Sep 8, 2022 · d89b9d3 · d89b9d3
2 parents e7ded43 + 7394632
commit d89b9d3
Show file tree

Hide file tree

Showing 21 changed files with 96 additions and 58 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/components/encryption/src/config.rs b/components/encryption/src/config.rs
@@ -111,6 +111,7 @@ mod encryption_method_serde {
     const AES128_CTR: &str = "aes128-ctr";
     const AES192_CTR: &str = "aes192-ctr";
     const AES256_CTR: &str = "aes256-ctr";
+    const SM4_CTR: &str = "sm4-ctr";
 
     #[allow(clippy::trivially_copy_pass_by_ref)]
     pub fn serialize<S>(method: &EncryptionMethod, serializer: S) -> Result<S::Ok, S::Error>
@@ -123,6 +124,7 @@ mod encryption_method_serde {
             EncryptionMethod::Aes128Ctr => serializer.serialize_str(AES128_CTR),
             EncryptionMethod::Aes192Ctr => serializer.serialize_str(AES192_CTR),
             EncryptionMethod::Aes256Ctr => serializer.serialize_str(AES256_CTR),
+            EncryptionMethod::Sm4Ctr => serializer.serialize_str(SM4_CTR),
         }
     }
 
@@ -149,6 +151,7 @@ mod encryption_method_serde {
                     AES128_CTR => Ok(EncryptionMethod::Aes128Ctr),
                     AES192_CTR => Ok(EncryptionMethod::Aes192Ctr),
                     AES256_CTR => Ok(EncryptionMethod::Aes256Ctr),
+                    SM4_CTR => Ok(EncryptionMethod::Sm4Ctr),
                     _ => Err(E::invalid_value(Unexpected::Str(value), &self)),
                 }
             }

diff --git a/components/encryption/src/crypter.rs b/components/encryption/src/crypter.rs
@@ -16,6 +16,7 @@ pub fn encryption_method_to_db_encryption_method(method: EncryptionMethod) -> DB
         EncryptionMethod::Aes128Ctr => DBEncryptionMethod::Aes128Ctr,
         EncryptionMethod::Aes192Ctr => DBEncryptionMethod::Aes192Ctr,
         EncryptionMethod::Aes256Ctr => DBEncryptionMethod::Aes256Ctr,
+        EncryptionMethod::Sm4Ctr => DBEncryptionMethod::Sm4Ctr,
         EncryptionMethod::Unknown => DBEncryptionMethod::Unknown,
     }
 }
@@ -26,6 +27,7 @@ pub fn encryption_method_from_db_encryption_method(method: DBEncryptionMethod) -
         DBEncryptionMethod::Aes128Ctr => EncryptionMethod::Aes128Ctr,
         DBEncryptionMethod::Aes192Ctr => EncryptionMethod::Aes192Ctr,
         DBEncryptionMethod::Aes256Ctr => EncryptionMethod::Aes256Ctr,
+        DBEncryptionMethod::Sm4Ctr => EncryptionMethod::Sm4Ctr,
         DBEncryptionMethod::Unknown => EncryptionMethod::Unknown,
     }
 }
@@ -40,6 +42,7 @@ pub fn get_method_key_length(method: EncryptionMethod) -> usize {
         EncryptionMethod::Aes128Ctr => 16,
         EncryptionMethod::Aes192Ctr => 24,
         EncryptionMethod::Aes256Ctr => 32,
+        EncryptionMethod::Sm4Ctr => 16,
         unknown => panic!("bad EncryptionMethod {:?}", unknown),
     }
 }

diff --git a/components/encryption/src/io.rs b/components/encryption/src/io.rs
@@ -377,6 +377,7 @@ pub fn create_aes_ctr_crypter(
         EncryptionMethod::Aes128Ctr => OCipher::aes_128_ctr(),
         EncryptionMethod::Aes192Ctr => OCipher::aes_192_ctr(),
         EncryptionMethod::Aes256Ctr => OCipher::aes_256_ctr(),
+        EncryptionMethod::Sm4Ctr => OCipher::sm4_ctr(),
     };
     let crypter = OCrypter::new(cipher, mode, key, Some(iv.as_slice()))?;
     Ok((cipher, crypter))
@@ -525,6 +526,7 @@ mod tests {
             EncryptionMethod::Aes128Ctr,
             EncryptionMethod::Aes192Ctr,
             EncryptionMethod::Aes256Ctr,
+            EncryptionMethod::Sm4Ctr,
         ];
         let ivs = [
             Iv::new_ctr(),
@@ -593,6 +595,7 @@ mod tests {
             EncryptionMethod::Aes128Ctr,
             EncryptionMethod::Aes192Ctr,
             EncryptionMethod::Aes256Ctr,
+            EncryptionMethod::Sm4Ctr,
         ];
         let mut plaintext = vec![0; 10240];
         OsRng.fill_bytes(&mut plaintext);
@@ -628,6 +631,7 @@ mod tests {
             EncryptionMethod::Aes128Ctr,
             EncryptionMethod::Aes192Ctr,
             EncryptionMethod::Aes256Ctr,
+            EncryptionMethod::Sm4Ctr,
         ];
         let mut plaintext = vec![0; 10240];
         OsRng.fill_bytes(&mut plaintext);
@@ -700,6 +704,7 @@ mod tests {
             EncryptionMethod::Aes128Ctr,
             EncryptionMethod::Aes192Ctr,
             EncryptionMethod::Aes256Ctr,
+            EncryptionMethod::Sm4Ctr,
         ];
         let iv = Iv::new_ctr();
         let mut plain_text = vec![0; 10240];

diff --git a/components/engine_rocks/src/encryption.rs b/components/engine_rocks/src/encryption.rs
@@ -64,6 +64,7 @@ fn convert_encryption_method(input: EncryptionMethod) -> DBEncryptionMethod {
         EncryptionMethod::Aes128Ctr => DBEncryptionMethod::Aes128Ctr,
         EncryptionMethod::Aes192Ctr => DBEncryptionMethod::Aes192Ctr,
         EncryptionMethod::Aes256Ctr => DBEncryptionMethod::Aes256Ctr,
+        EncryptionMethod::Sm4Ctr => DBEncryptionMethod::Sm4Ctr,
         EncryptionMethod::Unknown => DBEncryptionMethod::Unknown,
     }
 }
diff --git a/components/engine_rocks/src/options.rs b/components/engine_rocks/src/options.rs
@@ -16,7 +16,7 @@ impl RocksReadOptions {
 impl From<engine_traits::ReadOptions> for RocksReadOptions {
     fn from(opts: engine_traits::ReadOptions) -> Self {
         let mut r = RawReadOptions::default();
-        r.fill_cache(opts.fill_cache());
+        r.set_fill_cache(opts.fill_cache());
         RocksReadOptions(r)
     }
 }
@@ -59,7 +59,7 @@ impl From<engine_traits::IterOptions> for RocksReadOptions {
 
 fn build_read_opts(iter_opts: engine_traits::IterOptions) -> RawReadOptions {
     let mut opts = RawReadOptions::new();
-    opts.fill_cache(iter_opts.fill_cache());
+    opts.set_fill_cache(iter_opts.fill_cache());
     opts.set_max_skippable_internal_keys(iter_opts.max_skippable_internal_keys());
     if iter_opts.key_only() {
         opts.set_titan_key_only(true);

diff --git a/components/engine_rocks/src/sst.rs b/components/engine_rocks/src/sst.rs
@@ -222,9 +222,15 @@ impl SstWriterBuilder<RocksEngine> for RocksSstWriterBuilder {
         };
         // TODO: 0 is a valid value for compression_level
         if self.compression_level != 0 {
-            // other three fields are default value.
-            // see: https://github.com/facebook/rocksdb/blob/8cb278d11a43773a3ac22e523f4d183b06d37d88/include/rocksdb/advanced_options.h#L146-L153
-            io_options.set_compression_options(-14, self.compression_level, 0, 0, 0);
+            // other 4 fields are default value.
+            io_options.set_compression_options(
+                -14,
+                self.compression_level,
+                0, // strategy
+                0, // max_dict_bytes
+                0, // zstd_max_train_bytes
+                1, // parallel_threads
+            );
         }
         io_options.compression(compress_type);
         // in rocksdb 5.5.1, SstFileWriter will try to use bottommost_compression and

diff --git a/components/engine_traits/src/encryption.rs b/components/engine_traits/src/encryption.rs
@@ -53,4 +53,5 @@ pub enum EncryptionMethod {
     Aes128Ctr = 2,
     Aes192Ctr = 3,
     Aes256Ctr = 4,
+    Sm4Ctr = 5,
 }
diff --git a/components/raftstore/src/engine_store_ffi/interfaces.rs b/components/raftstore/src/engine_store_ffi/interfaces.rs
@@ -33,6 +33,7 @@ pub mod root {
             Aes128Ctr = 2,
             Aes192Ctr = 3,
             Aes256Ctr = 4,
+            SM4Ctr = 5,
         }
         #[repr(C)]
         #[derive(Debug)]
@@ -443,7 +444,7 @@ pub mod root {
                 ),
             >,
         }
-        pub const RAFT_STORE_PROXY_VERSION: u64 = 794398293737678384;
+        pub const RAFT_STORE_PROXY_VERSION: u64 = 14699247891578305166;
         pub const RAFT_STORE_PROXY_MAGIC_NUMBER: u32 = 324508639;
     }
 }
diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs
@@ -2492,6 +2492,9 @@ where
             ExtraMessageType::MsgHibernateResponse => {
                 self.on_hibernate_response(msg.get_from_peer());
             }
+            ExtraMessageType::MsgRejectRaftLogCausedByMemoryUsage => {
+                unimplemented!()
+            }
         }
     }
 

diff --git a/components/tikv_alloc/Cargo.toml b/components/tikv_alloc/Cargo.toml
@@ -35,15 +35,15 @@ optional = true
 features = ["bundled"]
 
 [dependencies.tikv-jemalloc-ctl]
-version = "0.4.0"
+version = "0.5.0"
 optional = true
 
 [dependencies.tikv-jemalloc-sys]
-version = "0.4.0"
+version = "0.5.0"
 optional = true
 features = ["stats"]
 
 [dependencies.tikv-jemallocator]
-version = "0.4.0"
+version = "0.5.0"
 optional = true
 features = ["unprefixed_malloc_on_supported_platforms", "stats"]
diff --git a/engine_tiflash/src/encryption.rs b/engine_tiflash/src/encryption.rs
@@ -64,6 +64,7 @@ fn convert_encryption_method(input: EncryptionMethod) -> DBEncryptionMethod {
         EncryptionMethod::Aes128Ctr => DBEncryptionMethod::Aes128Ctr,
         EncryptionMethod::Aes192Ctr => DBEncryptionMethod::Aes192Ctr,
         EncryptionMethod::Aes256Ctr => DBEncryptionMethod::Aes256Ctr,
+        EncryptionMethod::Sm4Ctr => DBEncryptionMethod::Sm4Ctr,
         EncryptionMethod::Unknown => DBEncryptionMethod::Unknown,
     }
 }
diff --git a/engine_tiflash/src/options.rs b/engine_tiflash/src/options.rs
@@ -16,7 +16,7 @@ impl RocksReadOptions {
 impl From<engine_traits::ReadOptions> for RocksReadOptions {
     fn from(opts: engine_traits::ReadOptions) -> Self {
         let mut r = RawReadOptions::default();
-        r.fill_cache(opts.fill_cache());
+        r.set_fill_cache(opts.fill_cache());
         RocksReadOptions(r)
     }
 }
@@ -59,7 +59,7 @@ impl From<engine_traits::IterOptions> for RocksReadOptions {
 
 fn build_read_opts(iter_opts: engine_traits::IterOptions) -> RawReadOptions {
     let mut opts = RawReadOptions::new();
-    opts.fill_cache(iter_opts.fill_cache());
+    opts.set_fill_cache(iter_opts.fill_cache());
     opts.set_max_skippable_internal_keys(iter_opts.max_skippable_internal_keys());
     if iter_opts.key_only() {
         opts.set_titan_key_only(true);

diff --git a/engine_tiflash/src/sst.rs b/engine_tiflash/src/sst.rs
@@ -222,9 +222,15 @@ impl SstWriterBuilder<RocksEngine> for RocksSstWriterBuilder {
         };
         // TODO: 0 is a valid value for compression_level
         if self.compression_level != 0 {
-            // other three fields are default value.
-            // see: https://github.com/facebook/rocksdb/blob/8cb278d11a43773a3ac22e523f4d183b06d37d88/include/rocksdb/advanced_options.h#L146-L153
-            io_options.set_compression_options(-14, self.compression_level, 0, 0, 0);
+            // other 4 fields are default value.
+            io_options.set_compression_options(
+                -14,
+                self.compression_level,
+                0, // strategy
+                0, // max_dict_bytes
+                0, // zstd_max_train_bytes
+                1, // parallel_threads
+            );
         }
         io_options.compression(compress_type);
         // in rocksdb 5.5.1, SstFileWriter will try to use bottommost_compression and

diff --git a/etc/config-template.toml b/etc/config-template.toml
@@ -1020,8 +1020,9 @@
 ## Configurations for encryption at rest. Experimental.
 [security.encryption]
 ## Encryption method to use for data files.
-## Possible values are "plaintext", "aes128-ctr", "aes192-ctr" and "aes256-ctr". Value other than
-## "plaintext" means encryption is enabled, in which case master key must be specified.
+## Possible values are "plaintext", "aes128-ctr", "aes192-ctr", "aes256-ctr" and "sm4-ctr".
+## Value other than "plaintext" means encryption is enabled, in which case
+## master key must be specified.
 # data-encryption-method = "plaintext"
 
 ## Specifies how often TiKV rotates data encryption key.

diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version
@@ -1,3 +1,3 @@
 #pragma once
 #include <cstdint>
-namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 794398293737678384ull; }
+namespace DB { constexpr uint64_t RAFT_STORE_PROXY_VERSION = 14699247891578305166ull; }
diff --git a/raftstore-proxy/ffi/src/RaftStoreProxyFFI/EncryptionFFI.h b/raftstore-proxy/ffi/src/RaftStoreProxyFFI/EncryptionFFI.h
@@ -15,6 +15,7 @@ enum class EncryptionMethod : uint8_t {
   Aes128Ctr,
   Aes192Ctr,
   Aes256Ctr,
+  SM4Ctr,
 };
 struct FileEncryptionInfoRaw {
   FileEncryptionRes res;