feat: add disktable to tablet and nameserver (#1483)

* special handling for storage mode * fix gc_interval * set flags_hdd/ssd_root_path default to "" * add GetCount for DiskTable * fix create root path Co-authored-by: Zhang Hao <[email protected]> Co-authored-by: litongxin <[email protected]>
4paradigm · Apr 20, 2022 · 2e750a1 · 2e750a1
1 parent 7a392df
commit 2e750a1
Show file tree

Hide file tree

Showing 26 changed files with 1,799 additions and 358 deletions.
diff --git a/src/client/tablet_client.cc b/src/client/tablet_client.cc
@@ -268,8 +268,6 @@ bool TabletClient::Put(uint32_t tid, uint32_t pid, uint64_t time, const std::str
     return false;
 }
 
-
-
 bool TabletClient::Put(uint32_t tid, uint32_t pid, const char* pk, uint64_t time, const char* value, uint32_t size,
                        uint32_t format_version) {
     ::openmldb::api::PutRequest request;
@@ -523,12 +521,13 @@ bool TabletClient::DeleteOPTask(const std::vector<uint64_t>& op_id_vec) {
     return true;
 }
 
-bool TabletClient::GetTermPair(uint32_t tid, uint32_t pid, uint64_t& term, uint64_t& offset, bool& has_table,
-                               bool& is_leader) {
+bool TabletClient::GetTermPair(uint32_t tid, uint32_t pid, ::openmldb::common::StorageMode storage_mode, uint64_t& term,
+                               uint64_t& offset, bool& has_table, bool& is_leader) {
     ::openmldb::api::GetTermPairRequest request;
     ::openmldb::api::GetTermPairResponse response;
     request.set_tid(tid);
     request.set_pid(pid);
+    request.set_storage_mode(storage_mode);
     bool ret = client_.SendRequest(&::openmldb::api::TabletServer_Stub::GetTermPair, &request, &response,
                                    FLAGS_request_timeout_ms, FLAGS_request_max_retry);
     if (!ret || response.code() != 0) {
@@ -543,11 +542,13 @@ bool TabletClient::GetTermPair(uint32_t tid, uint32_t pid, uint64_t& term, uint6
     return true;
 }
 
-bool TabletClient::GetManifest(uint32_t tid, uint32_t pid, ::openmldb::api::Manifest& manifest) {
+bool TabletClient::GetManifest(uint32_t tid, uint32_t pid, ::openmldb::common::StorageMode storage_mode,
+                               ::openmldb::api::Manifest& manifest) {
     ::openmldb::api::GetManifestRequest request;
     ::openmldb::api::GetManifestResponse response;
     request.set_tid(tid);
     request.set_pid(pid);
+    request.set_storage_mode(storage_mode);
     bool ret = client_.SendRequest(&::openmldb::api::TabletServer_Stub::GetManifest, &request, &response,
                                    FLAGS_request_timeout_ms, FLAGS_request_max_retry);
     if (!ret || response.code() != 0) {
@@ -974,10 +975,11 @@ bool TabletClient::DisConnectZK() {
     return true;
 }
 
-bool TabletClient::DeleteBinlog(uint32_t tid, uint32_t pid) {
+bool TabletClient::DeleteBinlog(uint32_t tid, uint32_t pid, openmldb::common::StorageMode storage_mode) {
     ::openmldb::api::GeneralRequest request;
     request.set_tid(tid);
     request.set_pid(pid);
+    request.set_storage_mode(storage_mode);
     ::openmldb::api::GeneralResponse response;
     bool ok = client_.SendRequest(&::openmldb::api::TabletServer_Stub::DeleteBinlog, &request, &response,
                                   FLAGS_request_timeout_ms, 1);

diff --git a/src/client/tablet_client.h b/src/client/tablet_client.h
@@ -178,18 +178,19 @@ class TabletClient : public Client {
     bool UpdateTTL(uint32_t tid, uint32_t pid, const ::openmldb::type::TTLType& type, uint64_t abs_ttl,
                    uint64_t lat_ttl, const std::string& index_name);
 
-    bool DeleteBinlog(uint32_t tid, uint32_t pid);
+    bool DeleteBinlog(uint32_t tid, uint32_t pid, ::openmldb::common::StorageMode storage_mode);
 
     bool GetTaskStatus(::openmldb::api::TaskStatusResponse& response);  // NOLINT
 
     bool DeleteOPTask(const std::vector<uint64_t>& op_id_vec);
 
     bool GetTermPair(uint32_t tid, uint32_t pid,
+                     ::openmldb::common::StorageMode storage_mode,  // NOLINT
                      uint64_t& term,                     // NOLINT
                      uint64_t& offset, bool& has_table,  // NOLINT
                      bool& is_leader);                   // NOLINT
 
-    bool GetManifest(uint32_t tid, uint32_t pid,
+    bool GetManifest(uint32_t tid, uint32_t pid, ::openmldb::common::StorageMode storage_mode,
                      ::openmldb::api::Manifest& manifest);  // NOLINT
 
     bool GetTableStatus(::openmldb::api::GetTableStatusResponse& response);  // NOLINT

diff --git a/src/flags.cc b/src/flags.cc
@@ -46,6 +46,7 @@ DEFINE_uint32(replica_num, 3,
               "config the default replica_num. if set 3, there is one leader and two followers");
 DEFINE_uint32(system_table_replica_num, 1, "config the default replica_num of system table.");
 DEFINE_int32(gc_interval, 120, "the gc interval of tablet every two hour");
+DEFINE_int32(disk_gc_interval, 120, "the rocksdb gc interval of tablet");
 DEFINE_int32(gc_pool_size, 2, "the size of tablet gc thread pool");
 DEFINE_int32(gc_safe_offset, 1, "the safe offset of tablet gc in minute");
 DEFINE_uint64(gc_on_table_recover_count, 10000000, "make a gc on recover count");
@@ -119,6 +120,8 @@ DEFINE_int32(snapshot_pool_size, 1, "the size of tablet thread pool for making s
 DEFINE_uint32(load_index_max_wait_time, 120 * 60 * 1000, "config the max wait time of load index");
 
 DEFINE_string(recycle_bin_root_path, "/tmp/recycle", "specify the root path of recycle bin");
+DEFINE_string(recycle_bin_ssd_root_path, "", "specify the root path of recycle bin in ssd");
+DEFINE_string(recycle_bin_hdd_root_path, "", "specify the root path of recycle bin in hdd");
 DEFINE_bool(recycle_bin_enabled, true, "enable the recycle bin storage");
 DEFINE_uint32(recycle_ttl, 0, "ttl of recycle in minute");