diff --git a/docs/en/reference/sql/deployment_manage/DEPLOY_STATEMENT.md b/docs/en/reference/sql/deployment_manage/DEPLOY_STATEMENT.md index 39a6688f7b9..d8b5f31f153 100644 --- a/docs/en/reference/sql/deployment_manage/DEPLOY_STATEMENT.md +++ b/docs/en/reference/sql/deployment_manage/DEPLOY_STATEMENT.md @@ -164,6 +164,15 @@ DEPLOY demo OPTIONS (SKIP_INDEX_CHECK="TRUE") SELECT * FROM t1 LAST JOIN t2 ORDER BY t2.col3 ON t1.col1 = t2.col1; ``` +### Synchronization/Asynchronization Settings +When executing deploy, you can set the synchronous/asynchronous mode through the `SYNC` option. The default value of `SYNC` is `true`, that is, the synchronous mode. If the relevant tables involved in the deploy statement have data and needs to add one or more indexs, executing deploy will initiate a job to execute a series of tasks such as loading data. In this case a job id will be returned if the `SYNC` option is set to `false`. You can get the job execution status by `SHOW JOBS FROM NAMESERVER LIKE '{job_id}'` + +**Example** +```sql +deploy demo options(SYNC="false") SELECT t1.col1, t2.col2, sum(col4) OVER w1 as w1_col4_sum FROM t1 LAST JOIN t2 ORDER BY t2.col3 ON t1.col2 = t2.col2 + WINDOW w1 AS (PARTITION BY t1.col2 ORDER BY t1.col3 ROWS BETWEEN 2 PRECEDING AND CURRENT ROW); +``` + ## Relevant SQL [USE DATABASE](../ddl/USE_DATABASE_STATEMENT.md) diff --git a/docs/zh/openmldb_sql/deployment_manage/DEPLOY_STATEMENT.md b/docs/zh/openmldb_sql/deployment_manage/DEPLOY_STATEMENT.md index 2798e5813dc..fe873b92e6c 100644 --- a/docs/zh/openmldb_sql/deployment_manage/DEPLOY_STATEMENT.md +++ b/docs/zh/openmldb_sql/deployment_manage/DEPLOY_STATEMENT.md @@ -99,6 +99,7 @@ DeployOption DeployOptionItem ::= 'LONG_WINDOWS' '=' LongWindowDefinitions | 'SKIP_INDEX_CHECK' '=' string_literal + | 'SYNC' '=' string_literal ``` #### 长窗口优化 @@ -160,6 +161,15 @@ DEPLOY demo OPTIONS (SKIP_INDEX_CHECK="TRUE") SELECT * FROM t1 LAST JOIN t2 ORDER BY t2.col3 ON t1.col1 = t2.col1; ``` +### 设置同步/异步 +执行deploy的时候可以通过`SYNC`选项来设置同步/异步模式, 默认为`true`即同步模式。如果deploy语句中涉及的相关表有数据，并且需要添加索引的情况下，执行deploy会发起数据加载等任务，如果`SYNC`选项设置为`false`就会返回一个任务id。可以通过`SHOW JOBS FROM NAMESERVER LIKE '{job_id}'`来查看任务执行状态。 + +**Example** +```sql +deploy demo options(SYNC="false") SELECT t1.col1, t2.col2, sum(col4) OVER w1 as w1_col4_sum FROM t1 LAST JOIN t2 ORDER BY t2.col3 ON t1.col2 = t2.col2 + WINDOW w1 AS (PARTITION BY t1.col2 ORDER BY t1.col3 ROWS BETWEEN 2 PRECEDING AND CURRENT ROW); +``` + ## 相关SQL [USE DATABASE](../ddl/USE_DATABASE_STATEMENT.md) diff --git a/src/client/ns_client.cc b/src/client/ns_client.cc index c4b6fb70e78..781272dab70 100644 --- a/src/client/ns_client.cc +++ b/src/client/ns_client.cc @@ -187,8 +187,19 @@ bool NsClient::MakeSnapshot(const std::string& name, const std::string& db, uint return false; } -bool NsClient::ShowOPStatus(::openmldb::nameserver::ShowOPStatusResponse& response, const std::string& name, - uint32_t pid, std::string& msg) { +base::Status NsClient::ShowOPStatus(uint64_t op_id, nameserver::ShowOPStatusResponse* response) { + ::openmldb::nameserver::ShowOPStatusRequest request; + request.set_op_id(op_id); + bool ok = client_.SendRequest(&::openmldb::nameserver::NameServer_Stub::ShowOPStatus, &request, response, + FLAGS_request_timeout_ms, 1); + if (ok && response->code() == 0) { + return {}; + } + return {base::ReturnCode::kError, response->msg()}; +} + +base::Status NsClient::ShowOPStatus(const std::string& name, uint32_t pid, + ::openmldb::nameserver::ShowOPStatusResponse* response) { ::openmldb::nameserver::ShowOPStatusRequest request; if (const std::string& db = GetDb(); !db.empty()) { request.set_db(db); @@ -199,13 +210,12 @@ bool NsClient::ShowOPStatus(::openmldb::nameserver::ShowOPStatusResponse& respon if (pid != INVALID_PID) { request.set_pid(pid); } - bool ok = client_.SendRequest(&::openmldb::nameserver::NameServer_Stub::ShowOPStatus, &request, &response, + bool ok = client_.SendRequest(&::openmldb::nameserver::NameServer_Stub::ShowOPStatus, &request, response, FLAGS_request_timeout_ms, 1); - msg = response.msg(); - if (ok && response.code() == 0) { - return true; + if (ok && response->code() == 0) { + return {}; } - return false; + return {base::ReturnCode::kError, response->msg()}; } bool NsClient::CancelOP(uint64_t op_id, std::string& msg) { @@ -917,7 +927,7 @@ bool NsClient::AddIndex(const std::string& db_name, } base::Status NsClient::AddMultiIndex(const std::string& db, const std::string& table_name, - const std::vector<::openmldb::common::ColumnKey>& column_keys) { + const std::vector<::openmldb::common::ColumnKey>& column_keys, bool skip_load_data) { ::openmldb::nameserver::AddIndexRequest request; ::openmldb::nameserver::GeneralResponse response; if (column_keys.empty()) { @@ -929,6 +939,7 @@ base::Status NsClient::AddMultiIndex(const std::string& db, const std::string& t } request.set_name(table_name); request.set_db(db); + request.set_skip_load_data(skip_load_data); bool ok = client_.SendRequest(&::openmldb::nameserver::NameServer_Stub::AddIndex, &request, &response, FLAGS_request_timeout_ms, 1); if (ok && response.code() == 0) { @@ -1065,5 +1076,31 @@ base::Status NsClient::ShowFunction(const std::string& name, return {}; } +base::Status NsClient::DeploySQL(const ::openmldb::api::ProcedureInfo& sp_info, + const std::map>& new_index_map, + uint64_t* op_id) { + if (new_index_map.empty()) { + return {base::ReturnCode::kError, "no index to add"}; + } + nameserver::DeploySQLRequest request; + request.mutable_sp_info()->CopyFrom(sp_info); + for (const auto& kv : new_index_map) { + auto index = request.add_index(); + index->set_name(kv.first); + index->set_db(sp_info.db_name()); + for (const auto& column_key : kv.second) { + index->add_column_key()->CopyFrom(column_key); + } + } + nameserver::DeploySQLResponse response; + bool ok = client_.SendRequest(&::openmldb::nameserver::NameServer_Stub::DeploySQL, &request, &response, + FLAGS_request_timeout_ms, 1); + if (!ok || response.code() != 0) { + return {base::ReturnCode::kError, response.msg()}; + } + *op_id = response.op_id(); + return {}; +} + } // namespace client } // namespace openmldb diff --git a/src/client/ns_client.h b/src/client/ns_client.h index 198c377c427..fcbe2dc3494 100644 --- a/src/client/ns_client.h +++ b/src/client/ns_client.h @@ -94,8 +94,10 @@ class NsClient : public Client { bool MakeSnapshot(const std::string& name, const std::string& db, uint32_t pid, uint64_t end_offset, std::string& msg); // NOLINT - bool ShowOPStatus(::openmldb::nameserver::ShowOPStatusResponse& response, // NOLINT - const std::string& name, uint32_t pid, std::string& msg); // NOLINT + base::Status ShowOPStatus(const std::string& name, uint32_t pid, + ::openmldb::nameserver::ShowOPStatusResponse* response); + + base::Status ShowOPStatus(uint64_t op_id, ::openmldb::nameserver::ShowOPStatusResponse* response); bool CancelOP(uint64_t op_id, std::string& msg); // NOLINT @@ -208,8 +210,7 @@ class NsClient : public Client { bool RemoveReplicaCluster(const std::string& alias, std::string& msg); // NOLINT - bool SwitchMode(const ::openmldb::nameserver::ServerMode& mode, - std::string& msg); // NOLINT + bool SwitchMode(const ::openmldb::nameserver::ServerMode& mode, std::string& msg); // NOLINT bool AddIndex(const std::string& table_name, const ::openmldb::common::ColumnKey& column_key, std::vector* cols, @@ -222,7 +223,7 @@ class NsClient : public Client { std::string& msg); // NOLINT base::Status AddMultiIndex(const std::string& db, const std::string& table_name, - const std::vector<::openmldb::common::ColumnKey>& column_keys); + const std::vector<::openmldb::common::ColumnKey>& column_keys, bool skip_load_data); bool DeleteIndex(const std::string& table_name, const std::string& idx_name, std::string& msg); // NOLINT @@ -246,6 +247,10 @@ class NsClient : public Client { base::Status UpdateOfflineTableInfo(const nameserver::TableInfo& table_info); + base::Status DeploySQL(const ::openmldb::api::ProcedureInfo& sp_info, + const std::map>& new_index_map, + uint64_t* op_id); + private: ::openmldb::RpcClient<::openmldb::nameserver::NameServer_Stub> client_; std::string db_; diff --git a/src/client/tablet_client.cc b/src/client/tablet_client.cc index 22237f8c413..13af0c63b9c 100644 --- a/src/client/tablet_client.cc +++ b/src/client/tablet_client.cc @@ -958,10 +958,10 @@ bool TabletClient::DeleteIndex(uint32_t tid, uint32_t pid, const std::string& id bool TabletClient::AddIndex(uint32_t tid, uint32_t pid, const ::openmldb::common::ColumnKey& column_key, std::shared_ptr task_info) { - return AddMultiIndex(tid, pid, {column_key}, task_info).OK(); + return AddMultiIndex(tid, pid, {column_key}, task_info); } -base::Status TabletClient::AddMultiIndex(uint32_t tid, uint32_t pid, +bool TabletClient::AddMultiIndex(uint32_t tid, uint32_t pid, const std::vector<::openmldb::common::ColumnKey>& column_keys, std::shared_ptr task_info) { ::openmldb::api::AddIndexRequest request; @@ -972,7 +972,7 @@ base::Status TabletClient::AddMultiIndex(uint32_t tid, uint32_t pid, if (task_info) { task_info->set_status(::openmldb::api::TaskStatus::kFailed); } - return {base::ReturnCode::kError, "no column key"}; + return false; } else if (column_keys.size() == 1) { request.mutable_column_key()->CopyFrom(column_keys[0]); } else { @@ -986,33 +986,11 @@ base::Status TabletClient::AddMultiIndex(uint32_t tid, uint32_t pid, if (task_info) { task_info->set_status(::openmldb::api::TaskStatus::kFailed); } - return {base::ReturnCode::kError, response.msg()}; + return false; } if (task_info) { task_info->set_status(::openmldb::api::TaskStatus::kDone); } - return {}; -} - -bool TabletClient::DumpIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx, - std::shared_ptr task_info) { - ::openmldb::api::DumpIndexDataRequest request; - ::openmldb::api::GeneralResponse response; - request.set_tid(tid); - request.set_pid(pid); - request.set_partition_num(partition_num); - request.set_idx(idx); - ::openmldb::common::ColumnKey* cur_column_key = request.mutable_column_key(); - cur_column_key->CopyFrom(column_key); - if (task_info) { - request.mutable_task_info()->CopyFrom(*task_info); - } - bool ok = client_.SendRequest(&openmldb::api::TabletServer_Stub::DumpIndexData, &request, &response, - FLAGS_request_timeout_ms, 1); - if (!ok || response.code() != 0) { - return false; - } return true; } @@ -1057,16 +1035,25 @@ bool TabletClient::LoadIndexData(uint32_t tid, uint32_t pid, uint32_t partition_ } bool TabletClient::ExtractIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx, + const std::vector<::openmldb::common::ColumnKey>& column_key, + uint64_t offset, bool dump_data, std::shared_ptr task_info) { + if (column_key.empty()) { + if (task_info) { + task_info->set_status(::openmldb::api::TaskStatus::kFailed); + } + return false; + } ::openmldb::api::ExtractIndexDataRequest request; ::openmldb::api::GeneralResponse response; request.set_tid(tid); request.set_pid(pid); request.set_partition_num(partition_num); - request.set_idx(idx); - ::openmldb::common::ColumnKey* cur_column_key = request.mutable_column_key(); - cur_column_key->CopyFrom(column_key); + request.set_offset(offset); + request.set_dump_data(dump_data); + for (const auto& cur_column_key : column_key) { + request.add_column_key()->CopyFrom(cur_column_key); + } if (task_info) { request.mutable_task_info()->CopyFrom(*task_info); } @@ -1078,25 +1065,6 @@ bool TabletClient::ExtractIndexData(uint32_t tid, uint32_t pid, uint32_t partiti return true; } -bool TabletClient::ExtractMultiIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const std::vector<::openmldb::common::ColumnKey>& column_key_vec) { - ::openmldb::api::ExtractMultiIndexDataRequest request; - ::openmldb::api::GeneralResponse response; - request.set_tid(tid); - request.set_pid(pid); - request.set_partition_num(partition_num); - for (const auto& column_key : column_key_vec) { - auto cur_column_key = request.add_column_key(); - cur_column_key->CopyFrom(column_key); - } - bool ok = client_.SendRequest(&openmldb::api::TabletServer_Stub::ExtractMultiIndexData, &request, &response, - FLAGS_request_timeout_ms, 1); - if (!ok || response.code() != 0) { - return false; - } - return true; -} - bool TabletClient::CancelOP(const uint64_t op_id) { ::openmldb::api::CancelOPRequest request; ::openmldb::api::GeneralResponse response; @@ -1140,15 +1108,14 @@ bool TabletClient::UpdateRealEndpointMap(const std::map task_info); - base::Status AddMultiIndex(uint32_t tid, uint32_t pid, + bool AddMultiIndex(uint32_t tid, uint32_t pid, const std::vector<::openmldb::common::ColumnKey>& column_keys, std::shared_ptr task_info); - bool DumpIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx, - std::shared_ptr task_info); - bool GetCatalog(uint64_t* version); bool SendIndexData(uint32_t tid, uint32_t pid, const std::map& pid_endpoint_map, @@ -216,18 +212,15 @@ class TabletClient : public Client { bool LoadIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, std::shared_ptr task_info); bool ExtractIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx, + const std::vector<::openmldb::common::ColumnKey>& column_key, + uint64_t offset, bool dump_data, std::shared_ptr task_info); - bool ExtractMultiIndexData(uint32_t tid, uint32_t pid, uint32_t partition_num, - const std::vector<::openmldb::common::ColumnKey>& column_key_vec); - bool CancelOP(const uint64_t op_id); bool UpdateRealEndpointMap(const std::map& map); - bool CreateProcedure(const openmldb::api::CreateProcedureRequest& sp_request, - std::string& msg); // NOLINT + base::Status CreateProcedure(const openmldb::api::CreateProcedureRequest& sp_request); bool CallProcedure(const std::string& db, const std::string& sp_name, const std::string& row, brpc::Controller* cntl, openmldb::api::QueryResponse* response, bool is_debug, diff --git a/src/cmd/openmldb.cc b/src/cmd/openmldb.cc index 58d6c3785bf..6878d00a945 100644 --- a/src/cmd/openmldb.cc +++ b/src/cmd/openmldb.cc @@ -2717,7 +2717,6 @@ void HandleNSShowOPStatus(const std::vector& parts, ::openmldb::cli ::baidu::common::TPrinter tp(row.size(), FLAGS_max_col_display_length); tp.AddRow(row); ::openmldb::nameserver::ShowOPStatusResponse response; - std::string msg; std::string name; uint32_t pid = ::openmldb::client::INVALID_PID; if (parts.size() > 1) { @@ -2731,9 +2730,9 @@ void HandleNSShowOPStatus(const std::vector& parts, ::openmldb::cli return; } } - bool ok = client->ShowOPStatus(response, name, pid, msg); - if (!ok) { - std::cout << "Fail to show tablets. error msg: " << msg << std::endl; + auto status = client->ShowOPStatus(name, pid, &response); + if (!status.OK()) { + std::cout << "Fail to show tablets. error msg: " << status.GetMsg() << std::endl; return; } for (int idx = 0; idx < response.op_status_size(); idx++) { diff --git a/src/cmd/sql_cmd_test.cc b/src/cmd/sql_cmd_test.cc index db5465ca8f6..e69e1d2a901 100644 --- a/src/cmd/sql_cmd_test.cc +++ b/src/cmd/sql_cmd_test.cc @@ -839,6 +839,43 @@ TEST_P(DBSDKTest, DeploySkipIndexCheck) { ASSERT_TRUE(cs->GetNsClient()->DropDatabase("test2", msg)); } +TEST_P(DBSDKTest, DeployWithData) { + auto cli = GetParam(); + cs = cli->cs; + sr = cli->sr; + std::string ddl1 = + "create table if not exists t1 (col1 string, col2 string, col3 bigint, col4 int, " + "index(key=col1, ts=col3, TTL_TYPE=absolute)) options (partitionnum=2, replicanum=1);"; + std::string ddl2 = + "create table if not exists t2 (col1 string, col2 string, col3 bigint, " + "index(key=col1, ts=col3, TTL_TYPE=absolute)) options (partitionnum=2, replicanum=1);"; + ProcessSQLs(sr, { + "set @@execute_mode = 'online';", + "create database test2;", + "use test2;", + ddl1, + ddl2, + }); + hybridse::sdk::Status status; + for (int i = 0; i < 100; i++) { + std::string key1 = absl::StrCat("col1", i); + std::string key2 = absl::StrCat("col2", i); + sr->ExecuteSQL(absl::StrCat("insert into t1 values ('", key1, "', '", key2, "', 1635247427000, 5);"), &status); + sr->ExecuteSQL(absl::StrCat("insert into t2 values ('", key1, "', '", key2, "', 1635247427000);"), &status); + } + sleep(2); + std::string deploy_sql = "deploy demo SELECT t1.col1, t2.col2, sum(col4) OVER w1 as w1_col4_sum FROM t1 " + "LAST JOIN t2 ORDER BY t2.col3 ON t1.col2 = t2.col2 " + "WINDOW w1 AS (PARTITION BY t1.col2 ORDER BY t1.col3 ROWS BETWEEN 2 PRECEDING AND CURRENT ROW);"; + sr->ExecuteSQL(deploy_sql, &status); + ASSERT_TRUE(status.IsOK()); + std::string msg; + ASSERT_TRUE(cs->GetNsClient()->DropProcedure("test2", "demo", msg)); + ASSERT_TRUE(cs->GetNsClient()->DropTable("test2", "t1", msg)); + ASSERT_TRUE(cs->GetNsClient()->DropTable("test2", "t2", msg)); + ASSERT_TRUE(cs->GetNsClient()->DropDatabase("test2", msg)); +} + TEST_P(DBSDKTest, Delete) { auto cli = GetParam(); sr = cli->sr; diff --git a/src/codec/codec.cc b/src/codec/codec.cc index 6937650b26f..8d5e24bc8c8 100644 --- a/src/codec/codec.cc +++ b/src/codec/codec.cc @@ -751,14 +751,14 @@ int32_t RowView::GetInteger(const int8_t* row, uint32_t idx, ::openmldb::type::D } case ::openmldb::type::kInt: { int32_t tmp_val = 0; - GetValue(row, idx, type, &tmp_val); + ret = GetValue(row, idx, type, &tmp_val); if (ret == 0) *val = tmp_val; break; } case ::openmldb::type::kTimestamp: case ::openmldb::type::kBigInt: { int64_t tmp_val = 0; - GetValue(row, idx, type, &tmp_val); + ret = GetValue(row, idx, type, &tmp_val); if (ret == 0) *val = tmp_val; break; } diff --git a/src/codec/row_codec.cc b/src/codec/row_codec.cc new file mode 100644 index 00000000000..64641d4f14c --- /dev/null +++ b/src/codec/row_codec.cc @@ -0,0 +1,387 @@ +/* + * Copyright 2021 4Paradigm + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "base/glog_wrapper.h" +#include "boost/algorithm/string.hpp" +#include "boost/container/deque.hpp" +#include "codec/schema_codec.h" +#include "codec/row_codec.h" +#include "storage/segment.h" + +namespace openmldb { +namespace codec { + +using ::openmldb::storage::DataBlock; + +int32_t RowCodec::CalStrLength(const std::map& str_map, const Schema& schema) { + int32_t str_len = 0; + for (int i = 0; i < schema.size(); i++) { + const ::openmldb::common::ColumnDesc& col = schema.Get(i); + if (col.data_type() == ::openmldb::type::kVarchar || col.data_type() == ::openmldb::type::kString) { + auto iter = str_map.find(col.name()); + if (iter == str_map.end()) { + return -1; + } + if (!col.not_null() && (iter->second == "null" || iter->second == NONETOKEN)) { + continue; + } else if (iter->second == "null" || iter->second == NONETOKEN) { + return -1; + } + str_len += iter->second.length(); + } + } + return str_len; +} + +int32_t RowCodec::CalStrLength(const std::vector& input_value, const Schema& schema) { + if (input_value.size() != (uint64_t)schema.size()) { + return -1; + } + int32_t str_len = 0; + for (int i = 0; i < schema.size(); i++) { + const ::openmldb::common::ColumnDesc& col = schema.Get(i); + if (col.data_type() == ::openmldb::type::kVarchar || col.data_type() == ::openmldb::type::kString) { + if (!col.not_null() && (input_value[i] == "null" || input_value[i] == NONETOKEN)) { + continue; + } else if (input_value[i] == "null" || input_value[i] == NONETOKEN) { + return -1; + } + str_len += input_value[i].length(); + } + } + return str_len; +} + +::openmldb::base::Status RowCodec::EncodeRow(const std::vector input_value, const Schema& schema, + uint32_t version, std::string& row) { + if (input_value.empty() || input_value.size() != (uint64_t)schema.size()) { + return ::openmldb::base::Status(-1, "input error"); + } + int32_t str_len = CalStrLength(input_value, schema); + if (str_len < 0) { + return ::openmldb::base::Status(-1, "cal str len failed"); + } + ::openmldb::codec::RowBuilder builder(schema); + uint32_t size = builder.CalTotalLength(str_len); + builder.SetSchemaVersion(version); + row.resize(size); + builder.SetBuffer(reinterpret_cast(&(row[0])), size); + for (int i = 0; i < schema.size(); i++) { + const ::openmldb::common::ColumnDesc& col = schema.Get(i); + if (!col.not_null() && (input_value[i] == "null" || input_value[i] == NONETOKEN)) { + if (!builder.AppendNULL()) { + return ::openmldb::base::Status(-1, + absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); + } + continue; + } else if (input_value[i] == "null" || input_value[i] == NONETOKEN) { + return ::openmldb::base::Status(-1, col.name() + " should not be null"); + } + if (!builder.AppendValue(input_value[i])) { + return ::openmldb::base::Status(-1, + absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); + } + } + return {}; +} + +::openmldb::base::Status RowCodec::EncodeRow(const std::map& str_map, + const Schema& schema, int32_t version, + std::string& row) { + if (str_map.empty() || str_map.size() != (uint64_t)schema.size()) { + return ::openmldb::base::Status(-1, "input error"); + } + int32_t str_len = CalStrLength(str_map, schema); + if (str_len < 0) { + return ::openmldb::base::Status(-1, "cal str len error"); + } + ::openmldb::codec::RowBuilder builder(schema); + builder.SetSchemaVersion(version); + uint32_t size = builder.CalTotalLength(str_len); + row.resize(size); + builder.SetBuffer(reinterpret_cast(&(row[0])), size); + for (int i = 0; i < schema.size(); i++) { + const ::openmldb::common::ColumnDesc& col = schema.Get(i); + auto iter = str_map.find(col.name()); + if (iter == str_map.end()) { + return ::openmldb::base::Status(-1, col.name() + " not in str_map"); + } + if (!col.not_null() && (iter->second == "null" || iter->second == NONETOKEN)) { + if (!builder.AppendNULL()) { + return ::openmldb::base::Status(-1, + absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); + } + continue; + } else if (iter->second == "null" || iter->second == NONETOKEN) { + return ::openmldb::base::Status(-1, col.name() + " should not be null"); + } + if (!builder.AppendValue(iter->second)) { + return ::openmldb::base::Status(-1, + absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); + } + } + return {}; +} + +bool RowCodec::DecodeRow(const Schema& schema, const ::openmldb::base::Slice& value, + std::vector& value_vec) { + openmldb::codec::RowView rv(schema, reinterpret_cast(const_cast(value.data())), value.size()); + return DecodeRow(schema, rv, false, 0, schema.size(), &value_vec); +} + +bool RowCodec::DecodeRow(const Schema& schema, const int8_t* data, int32_t size, bool replace_empty_str, int start, + int len, std::vector& values) { + openmldb::codec::RowView rv(schema, data, size); + return DecodeRow(schema, rv, replace_empty_str, start, len, &values); +} + +bool RowCodec::DecodeRow(const Schema& schema, openmldb::codec::RowView& rv, std::vector& value_vec) { + return DecodeRow(schema, rv, false, 0, schema.size(), &value_vec); +} + +bool RowCodec::DecodeRow(const Schema& schema, openmldb::codec::RowView& rv, + bool replace_empty_str, int start, int length, std::vector* value_vec) { + int end = start + length; + if (length <= 0) { + return false; + } + for (int32_t i = 0; i < end && i < schema.size(); i++) { + if (rv.IsNULL(i)) { + value_vec->emplace_back(NONETOKEN); + continue; + } + std::string col; + rv.GetStrValue(i, &col); + if (replace_empty_str && col.empty()) { + col = EMPTY_STRING; + } + value_vec->emplace_back(std::move(col)); + } + return true; +} + +bool RowCodec::DecodeRow(const openmldb::codec::RowView& rv, const int8_t* data, + const std::vector& cols, bool replace_null, std::vector* value_vec) { + for (auto col_idx : cols) { + std::string col; + int ret = rv.GetStrValue(data, col_idx, &col); + if (ret < 0) { + return false; + } + if (replace_null) { + if (ret == 1) { + value_vec->emplace_back(NONETOKEN); + } else if (col.empty()) { + value_vec->emplace_back(EMPTY_STRING); + } else { + value_vec->emplace_back(std::move(col)); + } + } else { + value_vec->emplace_back(std::move(col)); + } + } + return true; +} + +bool DecodeRows(const std::string& data, uint32_t count, const Schema& schema, + std::vector>* row_vec) { + openmldb::codec::RowView rv(schema); + uint32_t offset = 0; + for (uint32_t i = 0; i < count; i++) { + std::vector row; + const char* ch = data.c_str(); + ch += offset; + uint32_t value_size = 0; + memcpy(static_cast(&value_size), ch, 4); + ch += 4; + bool ok = rv.Reset(reinterpret_cast(const_cast(ch)), value_size); + if (!ok) { + return false; + } + offset += 4 + value_size; + if (!openmldb::codec::RowCodec::DecodeRow(schema, rv, row)) { + return false; + } + for (uint64_t i = 0; i < row.size(); i++) { + if (row[i] == openmldb::codec::NONETOKEN) { + row[i] = "null"; + } + } + row_vec->push_back(std::move(row)); + } + return true; +} + +void Encode(uint64_t time, const char* data, const size_t size, char* buffer, uint32_t offset) { + buffer += offset; + uint32_t total_size = 8 + size; + memcpy(buffer, static_cast(&total_size), 4); + memrev32ifbe(buffer); + buffer += 4; + memcpy(buffer, static_cast(&time), 8); + memrev64ifbe(buffer); + buffer += 8; + memcpy(buffer, static_cast(data), size); +} + +void Encode(uint64_t time, const DataBlock* data, char* buffer, uint32_t offset) { + return Encode(time, data->data, data->size, buffer, offset); +} + +void Encode(const char* data, const size_t size, char* buffer, uint32_t offset) { + buffer += offset; + memcpy(buffer, static_cast(&size), 4); + memrev32ifbe(buffer); + buffer += 4; + memcpy(buffer, static_cast(data), size); +} + +void Encode(const DataBlock* data, char* buffer, uint32_t offset) { + return Encode(data->data, data->size, buffer, offset); +} + +int32_t EncodeRows(const std::vector<::openmldb::base::Slice>& rows, uint32_t total_block_size, + std::string* body) { + if (body == NULL) { + PDLOG(WARNING, "invalid output body"); + return -1; + } + + uint32_t total_size = rows.size() * 4 + total_block_size; + if (rows.size() > 0) { + body->resize(total_size); + } + uint32_t offset = 0; + char* rbuffer = reinterpret_cast(&((*body)[0])); + for (auto lit = rows.begin(); lit != rows.end(); ++lit) { + ::openmldb::codec::Encode(lit->data(), lit->size(), rbuffer, offset); + offset += (4 + lit->size()); + } + return total_size; +} + +int32_t EncodeRows(const boost::container::deque>& rows, + uint32_t total_block_size, std::string* pairs) { + if (pairs == NULL) { + PDLOG(WARNING, "invalid output pairs"); + return -1; + } + + uint32_t total_size = rows.size() * (8 + 4) + total_block_size; + if (rows.size() > 0) { + pairs->resize(total_size); + } + + char* rbuffer = reinterpret_cast(&((*pairs)[0])); + uint32_t offset = 0; + for (auto lit = rows.begin(); lit != rows.end(); ++lit) { + ::openmldb::codec::Encode(lit->first, lit->second.data(), lit->second.size(), rbuffer, offset); + offset += (4 + 8 + lit->second.size()); + } + return total_size; +} + +void EncodeFull(const std::string& pk, uint64_t time, const char* data, const size_t size, char* buffer, + uint32_t offset) { + buffer += offset; + uint32_t pk_size = pk.length(); + uint32_t total_size = 8 + pk_size + size; + DEBUGLOG("encode total size %u pk size %u", total_size, pk_size); + memcpy(buffer, static_cast(&total_size), 4); + memrev32ifbe(buffer); + buffer += 4; + memcpy(buffer, static_cast(&pk_size), 4); + memrev32ifbe(buffer); + buffer += 4; + memcpy(buffer, static_cast(&time), 8); + memrev64ifbe(buffer); + buffer += 8; + memcpy(buffer, static_cast(pk.c_str()), pk_size); + buffer += pk_size; + memcpy(buffer, static_cast(data), size); +} + +void EncodeFull(const std::string& pk, uint64_t time, const DataBlock* data, char* buffer, + uint32_t offset) { + return EncodeFull(pk, time, data->data, data->size, buffer, offset); +} + +void Decode(const std::string* str, std::vector>& pairs) { // NOLINT + const char* buffer = str->c_str(); + uint32_t total_size = str->length(); + DEBUGLOG("total size %d %s", total_size, ::openmldb::base::DebugString(*str).c_str()); + while (total_size > 0) { + uint32_t size = 0; + memcpy(static_cast(&size), buffer, 4); + memrev32ifbe(static_cast(&size)); + DEBUGLOG("decode size %d", size); + buffer += 4; + uint64_t time = 0; + memcpy(static_cast(&time), buffer, 8); + memrev64ifbe(static_cast(&time)); + buffer += 8; + assert(size >= 8); + std::string* data = new std::string(size - 8, '0'); + memcpy(reinterpret_cast(&((*data)[0])), buffer, size - 8); + buffer += (size - 8); + pairs.push_back(std::make_pair(time, data)); + total_size -= (size + 4); + } +} + +void DecodeFull(const std::string* str, + std::map>>& value_map) { // NOLINT + const char* buffer = str->c_str(); + uint32_t total_size = str->length(); + DEBUGLOG("total size %u %s", total_size, ::openmldb::base::DebugString(*str).c_str()); + while (total_size > 0) { + uint32_t size = 0; + memcpy(static_cast(&size), buffer, 4); + memrev32ifbe(static_cast(&size)); + DEBUGLOG("decode size %u", size); + buffer += 4; + uint32_t pk_size = 0; + memcpy(static_cast(&pk_size), buffer, 4); + buffer += 4; + memrev32ifbe(static_cast(&pk_size)); + DEBUGLOG("decode size %u", pk_size); + assert(size > pk_size + 8); + uint64_t time = 0; + memcpy(static_cast(&time), buffer, 8); + memrev64ifbe(static_cast(&time)); + buffer += 8; + std::string pk(buffer, pk_size); + buffer += pk_size; + uint32_t value_size = size - 8 - pk_size; + std::string* data = new std::string(value_size, '0'); + memcpy(reinterpret_cast(&((*data)[0])), buffer, value_size); + buffer += value_size; + if (value_map.find(pk) == value_map.end()) { + value_map.insert(std::make_pair(pk, std::vector>())); + } + value_map[pk].push_back(std::make_pair(time, data)); + total_size -= (size + 8); + } +} + +} // namespace codec +} // namespace openmldb diff --git a/src/codec/row_codec.h b/src/codec/row_codec.h index 1fdb978af43..5f4f01b9690 100644 --- a/src/codec/row_codec.h +++ b/src/codec/row_codec.h @@ -22,351 +22,76 @@ #include #include -#include "base/glog_wrapper.h" -#include "boost/algorithm/string.hpp" +#include "base/status.h" #include "boost/container/deque.hpp" -#include "codec/schema_codec.h" +#include "codec/codec.h" #include "storage/segment.h" namespace openmldb { namespace codec { using ::openmldb::storage::DataBlock; +using Schema = google::protobuf::RepeatedPtrField; class RowCodec { public: - static int32_t CalStrLength(const std::map& str_map, const Schema& schema) { - int32_t str_len = 0; - for (int i = 0; i < schema.size(); i++) { - const ::openmldb::common::ColumnDesc& col = schema.Get(i); - if (col.data_type() == ::openmldb::type::kVarchar || col.data_type() == ::openmldb::type::kString) { - auto iter = str_map.find(col.name()); - if (iter == str_map.end()) { - return -1; - } - if (!col.not_null() && (iter->second == "null" || iter->second == NONETOKEN)) { - continue; - } else if (iter->second == "null" || iter->second == NONETOKEN) { - return -1; - } - str_len += iter->second.length(); - } - } - return str_len; - } + static int32_t CalStrLength(const std::map& str_map, const Schema& schema); - static int32_t CalStrLength(const std::vector& input_value, const Schema& schema) { - if (input_value.size() != (uint64_t)schema.size()) { - return -1; - } - int32_t str_len = 0; - for (int i = 0; i < schema.size(); i++) { - const ::openmldb::common::ColumnDesc& col = schema.Get(i); - if (col.data_type() == ::openmldb::type::kVarchar || col.data_type() == ::openmldb::type::kString) { - if (!col.not_null() && (input_value[i] == "null" || input_value[i] == NONETOKEN)) { - continue; - } else if (input_value[i] == "null" || input_value[i] == NONETOKEN) { - return -1; - } - str_len += input_value[i].length(); - } - } - return str_len; - } + static int32_t CalStrLength(const std::vector& input_value, const Schema& schema); static ::openmldb::base::Status EncodeRow(const std::vector input_value, const Schema& schema, uint32_t version, - std::string& row) { // NOLINT - if (input_value.empty() || input_value.size() != (uint64_t)schema.size()) { - return ::openmldb::base::Status(-1, "input error"); - } - int32_t str_len = CalStrLength(input_value, schema); - if (str_len < 0) { - return ::openmldb::base::Status(-1, "cal str len failed"); - } - ::openmldb::codec::RowBuilder builder(schema); - uint32_t size = builder.CalTotalLength(str_len); - builder.SetSchemaVersion(version); - row.resize(size); - builder.SetBuffer(reinterpret_cast(&(row[0])), size); - for (int i = 0; i < schema.size(); i++) { - const ::openmldb::common::ColumnDesc& col = schema.Get(i); - if (!col.not_null() && (input_value[i] == "null" || input_value[i] == NONETOKEN)) { - if (!builder.AppendNULL()) { - return ::openmldb::base::Status(-1, - absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); - } - continue; - } else if (input_value[i] == "null" || input_value[i] == NONETOKEN) { - return ::openmldb::base::Status(-1, col.name() + " should not be null"); - } - if (!builder.AppendValue(input_value[i])) { - return ::openmldb::base::Status(-1, - absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); - } - } - return {}; - } + std::string& row); // NOLINT static ::openmldb::base::Status EncodeRow(const std::map& str_map, const Schema& schema, int32_t version, - std::string& row) { // NOLINT - if (str_map.empty() || str_map.size() != (uint64_t)schema.size()) { - return ::openmldb::base::Status(-1, "input error"); - } - int32_t str_len = CalStrLength(str_map, schema); - if (str_len < 0) { - return ::openmldb::base::Status(-1, "cal str len error"); - } - ::openmldb::codec::RowBuilder builder(schema); - builder.SetSchemaVersion(version); - uint32_t size = builder.CalTotalLength(str_len); - row.resize(size); - builder.SetBuffer(reinterpret_cast(&(row[0])), size); - for (int i = 0; i < schema.size(); i++) { - const ::openmldb::common::ColumnDesc& col = schema.Get(i); - auto iter = str_map.find(col.name()); - if (iter == str_map.end()) { - return ::openmldb::base::Status(-1, col.name() + " not in str_map"); - } - if (!col.not_null() && (iter->second == "null" || iter->second == NONETOKEN)) { - if (!builder.AppendNULL()) { - return ::openmldb::base::Status(-1, - absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); - } - continue; - } else if (iter->second == "null" || iter->second == NONETOKEN) { - return ::openmldb::base::Status(-1, col.name() + " should not be null"); - } - if (!builder.AppendValue(iter->second)) { - return ::openmldb::base::Status(-1, - absl::StrCat("append ", ::openmldb::type::DataType_Name(col.data_type()), " error")); - } - } - return {}; - } + std::string& row); // NOLINT - - static bool DecodeRow(const Schema& schema, // NOLINT - const ::openmldb::base::Slice& value, - std::vector& value_vec) { // NOLINT - openmldb::codec::RowView rv(schema, reinterpret_cast(const_cast(value.data())), value.size()); - return DecodeRow(schema, rv, false, 0, schema.size(), &value_vec); - } + static bool DecodeRow(const Schema& schema, const ::openmldb::base::Slice& value, + std::vector& value_vec); // NOLINT static bool DecodeRow(const Schema& schema, const int8_t* data, int32_t size, bool replace_empty_str, int start, - int len, std::vector& values) { // NOLINT - openmldb::codec::RowView rv(schema, data, size); - return DecodeRow(schema, rv, replace_empty_str, start, len, &values); - } + int len, std::vector& values); // NOLINT - static bool DecodeRow(const Schema& schema, // NOLINT + static bool DecodeRow(const Schema& schema, openmldb::codec::RowView& rv, // NOLINT - std::vector& value_vec) { // NOLINT - return DecodeRow(schema, rv, false, 0, schema.size(), &value_vec); - } + std::vector& value_vec); // NOLINT static bool DecodeRow(const Schema& schema, openmldb::codec::RowView& rv, // NOLINT - bool replace_empty_str, int start, int length, std::vector* value_vec) { - int end = start + length; - if (length <= 0) { - return false; - } - for (int32_t i = 0; i < end && i < schema.size(); i++) { - if (rv.IsNULL(i)) { - value_vec->emplace_back(NONETOKEN); - continue; - } - std::string col; - rv.GetStrValue(i, &col); - if (replace_empty_str && col.empty()) { - col = EMPTY_STRING; - } - value_vec->emplace_back(std::move(col)); - } - return true; - } -}; -__attribute__((unused)) static bool DecodeRows(const std::string& data, uint32_t count, const Schema& schema, - std::vector>* row_vec) { - openmldb::codec::RowView rv(schema); - uint32_t offset = 0; - for (uint32_t i = 0; i < count; i++) { - std::vector row; - const char* ch = data.c_str(); - ch += offset; - uint32_t value_size = 0; - memcpy(static_cast(&value_size), ch, 4); - ch += 4; - bool ok = rv.Reset(reinterpret_cast(const_cast(ch)), value_size); - if (!ok) { - return false; - } - offset += 4 + value_size; - if (!openmldb::codec::RowCodec::DecodeRow(schema, rv, row)) { - return false; - } - for (uint64_t i = 0; i < row.size(); i++) { - if (row[i] == openmldb::codec::NONETOKEN) { - row[i] = "null"; - } - } - row_vec->push_back(std::move(row)); - } - return true; -} + bool replace_empty_str, int start, int length, std::vector* value_vec); -static inline void Encode(uint64_t time, const char* data, const size_t size, char* buffer, uint32_t offset) { - buffer += offset; - uint32_t total_size = 8 + size; - memcpy(buffer, static_cast(&total_size), 4); - memrev32ifbe(buffer); - buffer += 4; - memcpy(buffer, static_cast(&time), 8); - memrev64ifbe(buffer); - buffer += 8; - memcpy(buffer, static_cast(data), size); -} - -static inline void Encode(uint64_t time, const DataBlock* data, char* buffer, uint32_t offset) { - return Encode(time, data->data, data->size, buffer, offset); -} + static bool DecodeRow(const openmldb::codec::RowView& rv, const int8_t* data, + const std::vector& cols, bool replace_null, std::vector* value_vec); +}; -static inline void Encode(const char* data, const size_t size, char* buffer, uint32_t offset) { - buffer += offset; - memcpy(buffer, static_cast(&size), 4); - memrev32ifbe(buffer); - buffer += 4; - memcpy(buffer, static_cast(data), size); -} +bool DecodeRows(const std::string& data, uint32_t count, const Schema& schema, + std::vector>* row_vec); -static inline void Encode(const DataBlock* data, char* buffer, uint32_t offset) { - return Encode(data->data, data->size, buffer, offset); -} -static inline int32_t EncodeRows(const std::vector<::openmldb::base::Slice>& rows, uint32_t total_block_size, - std::string* body) { - if (body == NULL) { - PDLOG(WARNING, "invalid output body"); - return -1; - } +void Encode(uint64_t time, const char* data, const size_t size, char* buffer, uint32_t offset); - uint32_t total_size = rows.size() * 4 + total_block_size; - if (rows.size() > 0) { - body->resize(total_size); - } - uint32_t offset = 0; - char* rbuffer = reinterpret_cast(&((*body)[0])); - for (auto lit = rows.begin(); lit != rows.end(); ++lit) { - ::openmldb::codec::Encode(lit->data(), lit->size(), rbuffer, offset); - offset += (4 + lit->size()); - } - return total_size; -} +void Encode(uint64_t time, const DataBlock* data, char* buffer, uint32_t offset); -static inline int32_t EncodeRows(const boost::container::deque>& rows, - uint32_t total_block_size, std::string* pairs) { - if (pairs == NULL) { - PDLOG(WARNING, "invalid output pairs"); - return -1; - } +void Encode(const char* data, const size_t size, char* buffer, uint32_t offset); - uint32_t total_size = rows.size() * (8 + 4) + total_block_size; - if (rows.size() > 0) { - pairs->resize(total_size); - } +void Encode(const DataBlock* data, char* buffer, uint32_t offset); - char* rbuffer = reinterpret_cast(&((*pairs)[0])); - uint32_t offset = 0; - for (auto lit = rows.begin(); lit != rows.end(); ++lit) { - ::openmldb::codec::Encode(lit->first, lit->second.data(), lit->second.size(), rbuffer, offset); - offset += (4 + 8 + lit->second.size()); - } - return total_size; -} +int32_t EncodeRows(const std::vector<::openmldb::base::Slice>& rows, uint32_t total_block_size, + std::string* body); +int32_t EncodeRows(const boost::container::deque>& rows, + uint32_t total_block_size, std::string* pairs); // encode pk, ts and value -static inline void EncodeFull(const std::string& pk, uint64_t time, const char* data, const size_t size, char* buffer, - uint32_t offset) { - buffer += offset; - uint32_t pk_size = pk.length(); - uint32_t total_size = 8 + pk_size + size; - DEBUGLOG("encode total size %u pk size %u", total_size, pk_size); - memcpy(buffer, static_cast(&total_size), 4); - memrev32ifbe(buffer); - buffer += 4; - memcpy(buffer, static_cast(&pk_size), 4); - memrev32ifbe(buffer); - buffer += 4; - memcpy(buffer, static_cast(&time), 8); - memrev64ifbe(buffer); - buffer += 8; - memcpy(buffer, static_cast(pk.c_str()), pk_size); - buffer += pk_size; - memcpy(buffer, static_cast(data), size); -} -static inline void EncodeFull(const std::string& pk, uint64_t time, const DataBlock* data, char* buffer, - uint32_t offset) { - return EncodeFull(pk, time, data->data, data->size, buffer, offset); -} +void EncodeFull(const std::string& pk, uint64_t time, const char* data, const size_t size, char* buffer, + uint32_t offset); + +void EncodeFull(const std::string& pk, uint64_t time, const DataBlock* data, char* buffer, + uint32_t offset); -static inline void Decode(const std::string* str, std::vector>& pairs) { // NOLINT - const char* buffer = str->c_str(); - uint32_t total_size = str->length(); - DEBUGLOG("total size %d %s", total_size, ::openmldb::base::DebugString(*str).c_str()); - while (total_size > 0) { - uint32_t size = 0; - memcpy(static_cast(&size), buffer, 4); - memrev32ifbe(static_cast(&size)); - DEBUGLOG("decode size %d", size); - buffer += 4; - uint64_t time = 0; - memcpy(static_cast(&time), buffer, 8); - memrev64ifbe(static_cast(&time)); - buffer += 8; - assert(size >= 8); - std::string* data = new std::string(size - 8, '0'); - memcpy(reinterpret_cast(&((*data)[0])), buffer, size - 8); - buffer += (size - 8); - pairs.push_back(std::make_pair(time, data)); - total_size -= (size + 4); - } -} +void Decode(const std::string* str, std::vector>& pairs); // NOLINT -static inline void DecodeFull(const std::string* str, - std::map>>& value_map) { // NOLINT - const char* buffer = str->c_str(); - uint32_t total_size = str->length(); - DEBUGLOG("total size %u %s", total_size, ::openmldb::base::DebugString(*str).c_str()); - while (total_size > 0) { - uint32_t size = 0; - memcpy(static_cast(&size), buffer, 4); - memrev32ifbe(static_cast(&size)); - DEBUGLOG("decode size %u", size); - buffer += 4; - uint32_t pk_size = 0; - memcpy(static_cast(&pk_size), buffer, 4); - buffer += 4; - memrev32ifbe(static_cast(&pk_size)); - DEBUGLOG("decode size %u", pk_size); - assert(size > pk_size + 8); - uint64_t time = 0; - memcpy(static_cast(&time), buffer, 8); - memrev64ifbe(static_cast(&time)); - buffer += 8; - std::string pk(buffer, pk_size); - buffer += pk_size; - uint32_t value_size = size - 8 - pk_size; - std::string* data = new std::string(value_size, '0'); - memcpy(reinterpret_cast(&((*data)[0])), buffer, value_size); - buffer += value_size; - if (value_map.find(pk) == value_map.end()) { - value_map.insert(std::make_pair(pk, std::vector>())); - } - value_map[pk].push_back(std::make_pair(time, data)); - total_size -= (size + 8); - } -} +void DecodeFull(const std::string* str, + std::map>>& value_map); // NOLINT } // namespace codec } // namespace openmldb diff --git a/src/flags.cc b/src/flags.cc index 045cd3f87e1..bed34c0150d 100644 --- a/src/flags.cc +++ b/src/flags.cc @@ -178,3 +178,4 @@ DEFINE_uint32(keep_log_file_num, 5, "Maximal info log files to be kept"); DEFINE_int32(sync_job_timeout, 30 * 60 * 1000, "sync job timeout, unit is milliseconds, should <= server.channel_keep_alive_time in TaskManager"); +DEFINE_int32(deploy_job_max_wait_time_ms, 30 * 60 * 1000, "the max wait time of waiting deploy job"); diff --git a/src/log/log_writer.h b/src/log/log_writer.h index 15c5f9270ca..dd361a5e2f7 100644 --- a/src/log/log_writer.h +++ b/src/log/log_writer.h @@ -24,6 +24,7 @@ #include #include +#include #include "base/slice.h" #include "log/status.h" @@ -110,6 +111,15 @@ struct WriteHandle { } }; +std::shared_ptr inline CreateWriteHandle(const std::string& compress_type, + const std::string& fname, const std::string& path) { + FILE* fd = fopen(path.c_str(), "ab+"); + if (fd == nullptr) { + return {}; + } + return std::make_shared(compress_type, fname, fd); +} + } // namespace log } // namespace openmldb diff --git a/src/nameserver/name_server_impl.cc b/src/nameserver/name_server_impl.cc index d7f2a451a0c..33cd429f316 100644 --- a/src/nameserver/name_server_impl.cc +++ b/src/nameserver/name_server_impl.cc @@ -75,14 +75,15 @@ DECLARE_bool(use_name); DECLARE_bool(enable_distsql); DECLARE_uint32(sync_deploy_stats_timeout); -using ::openmldb::api::OPType::kAddIndexOP; -using ::openmldb::base::ReturnCode; namespace openmldb { namespace nameserver { +using ::openmldb::base::ReturnCode; + const std::string OFFLINE_LEADER_ENDPOINT = "OFFLINE_LEADER_ENDPOINT"; // NOLINT constexpr uint8_t MAX_ADD_TABLE_FIELD_COUNT = 63; +constexpr uint32_t SEQ_TASK_CHECK_INTERVAL = 100; // 100ms void NameServerImpl::CheckSyncExistTable(const std::string& alias, const std::vector<::openmldb::nameserver::TableInfo>& tables_remote, @@ -958,7 +959,7 @@ bool NameServerImpl::RecoverOPTask() { } break; case ::openmldb::api::OPType::kAddIndexOP: - if (CreateAddIndexOPTask(op_data) < 0) { + if (!CreateAddIndexOPTask(op_data).OK()) { PDLOG(WARNING, "recover op[%s] failed. op_id[%lu]", op_type_str.c_str(), op_id); continue; } @@ -1029,9 +1030,8 @@ int NameServerImpl::CreateMakeSnapshotOPTask(std::shared_ptr op_data) { if (request.has_offset() && request.offset() > 0) { end_offset = request.offset(); } - auto task_meta = std::make_shared(op_data->op_info_.op_id(), + auto task = CreateTask(op_data->op_info_.op_id(), ::openmldb::api::OPType::kMakeSnapshotOP, endpoint, tid, pid, end_offset); - auto task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create makesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; @@ -1403,7 +1403,6 @@ bool NameServerImpl::Init(const std::string& zk_cluster, const std::string& zk_p std::string zk_op_path = zk_path + "/op"; zk_path_.op_index_node_ = zk_op_path + "/op_index"; zk_path_.op_data_path_ = zk_op_path + "/op_data"; - zk_path_.op_sync_path_ = zk_op_path + "/op_sync"; zk_path_.offline_endpoint_lock_node_ = zk_path + "/offline_endpoint_lock"; std::string zk_config_path = zk_path + "/config"; zk_path_.zone_data_path_ = zk_path + "/cluster"; @@ -1529,9 +1528,7 @@ int NameServerImpl::UpdateTaskStatus(bool is_recover_op) { continue; } if (task->task_info_->has_endpoint() && task->task_info_->endpoint() == iter->first) { - PDLOG(WARNING, - "tablet is offline. update task status " - "from[kDoing] to[kFailed]. " + PDLOG(WARNING, "tablet is offline. update task status from[kDoing] to[kFailed]. " "op_id[%lu], task_type[%s] endpoint[%s]", op_data->op_info_.op_id(), ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str(), @@ -1556,10 +1553,7 @@ int NameServerImpl::UpdateTaskStatus(bool is_recover_op) { } std::string endpoint = iter->first; for (const auto& op_list : task_vec_) { - std::string endpoint_role = "tablet"; - if (UpdateTask(op_list, endpoint, endpoint_role, is_recover_op, response) < 0) { - continue; - } + UpdateTask(op_list, endpoint, is_recover_op, response); } } } @@ -1586,8 +1580,8 @@ int NameServerImpl::UpdateTaskStatusRemote(bool is_recover_op) { PDLOG(INFO, "cluster[%s] is not Healthy", iter->first.c_str()); continue; } - client_map.insert(std::make_pair( - iter->first, std::atomic_load_explicit(&iter->second->client_, std::memory_order_relaxed))); + client_map.emplace(iter->first, + std::atomic_load_explicit(&iter->second->client_, std::memory_order_relaxed)); } } uint64_t last_task_rpc_version = task_rpc_version_.load(std::memory_order_acquire); @@ -1600,17 +1594,14 @@ int NameServerImpl::UpdateTaskStatusRemote(bool is_recover_op) { DEBUGLOG("task_rpc_version mismatch"); break; } - std::string endpoint = iter->first; + std::string endpoint = iter->second->GetEndpoint(); uint32_t index = 0; for (const auto& op_list : task_vec_) { index++; if (index <= FLAGS_name_server_task_max_concurrency) { continue; } - std::string endpoint_role = "replica cluster"; - if (UpdateTask(op_list, endpoint, endpoint_role, is_recover_op, response) < 0) { - continue; - } + UpdateTask(op_list, endpoint, is_recover_op, response); } } else { if (response.has_msg()) { @@ -1622,8 +1613,7 @@ int NameServerImpl::UpdateTaskStatusRemote(bool is_recover_op) { } int NameServerImpl::UpdateTask(const std::list>& op_list, const std::string& endpoint, - const std::string& msg, bool is_recover_op, - ::openmldb::api::TaskStatusResponse& response) { + bool is_recover_op, const ::openmldb::api::TaskStatusResponse& response) { if (op_list.empty()) { return -1; } @@ -1631,83 +1621,11 @@ int NameServerImpl::UpdateTask(const std::list>& op_list if (op_data->task_list_.empty()) { return -1; } - // update task status std::shared_ptr task = op_data->task_list_.front(); if (task->task_info_->status() != ::openmldb::api::kDoing) { return -1; } - bool has_op_task = false; - for (int idx = 0; idx < response.task_size(); idx++) { - if (op_data->op_info_.op_id() == response.task(idx).op_id() && - task->task_info_->task_type() == response.task(idx).task_type()) { - has_op_task = true; - if (response.task(idx).status() != ::openmldb::api::kInited) { - if (!task->sub_task_.empty()) { - for (auto& sub_task : task->sub_task_) { - if (sub_task->task_info_->has_endpoint() && sub_task->task_info_->endpoint() == endpoint && - sub_task->task_info_->status() != response.task(idx).status()) { - PDLOG(INFO, - "update sub task status from[%s] to[%s]. " - "op_id[%lu], task_type[%s]", - ::openmldb::api::TaskStatus_Name(sub_task->task_info_->status()).c_str(), - ::openmldb::api::TaskStatus_Name(response.task(idx).status()).c_str(), - response.task(idx).op_id(), - ::openmldb::api::TaskType_Name(sub_task->task_info_->task_type()).c_str()); - sub_task->task_info_->set_status(response.task(idx).status()); - if (response.task(idx).status() == ::openmldb::api::kFailed) { - task->task_info_->set_status(::openmldb::api::kFailed); - PDLOG(INFO, - "update task status from[%s] " - "to[kFailed]. op_id[%lu], task_type[%s]", - ::openmldb::api::TaskStatus_Name(task->task_info_->status()).c_str(), - response.task(idx).op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str()); - } - break; - } - } - } else if (task->task_info_->status() != response.task(idx).status()) { - PDLOG(INFO, - "update task status from[%s] to[%s]. op_id[%lu], " - "task_type[%s]", - ::openmldb::api::TaskStatus_Name(task->task_info_->status()).c_str(), - ::openmldb::api::TaskStatus_Name(response.task(idx).status()).c_str(), - response.task(idx).op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str()); - task->task_info_->set_status(response.task(idx).status()); - } - } - break; - } - } - if (!has_op_task && (is_recover_op || task->task_info_->is_rpc_send())) { - if (!task->sub_task_.empty()) { - for (auto& sub_task : task->sub_task_) { - if (sub_task->task_info_->has_endpoint() && sub_task->task_info_->endpoint() == endpoint) { - if (sub_task->task_info_->status() == ::openmldb::api::kDoing || - sub_task->task_info_->status() == ::openmldb::api::kInited) { - PDLOG(WARNING, - "not found op in [%s]. update sub task status " - "from[kDoing] to[kFailed]. " - "op_id[%lu], task_type[%s] endpoint[%s]", - msg.c_str(), op_data->op_info_.op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str(), endpoint.c_str()); - sub_task->task_info_->set_status(::openmldb::api::kFailed); - task->task_info_->set_status(::openmldb::api::kFailed); - } - break; - } - } - } else if (task->task_info_->has_endpoint() && task->task_info_->endpoint() == endpoint) { - PDLOG(WARNING, - "not found op in [%s]. update task status from[kDoing] " - "to[kFailed]. " - "op_id[%lu], task_type[%s] endpoint[%s]", - msg.c_str(), op_data->op_info_.op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str(), endpoint.c_str()); - task->task_info_->set_status(::openmldb::api::kFailed); - } - } + task->UpdateTaskStatus(response.task(), endpoint, is_recover_op); return 1; } @@ -1722,52 +1640,23 @@ int NameServerImpl::UpdateZKTaskStatus() { continue; } std::shared_ptr task = op_data->task_list_.front(); - if (!task->sub_task_.empty()) { - bool has_done = true; - bool has_failed = false; - for (const auto& cur_task : task->sub_task_) { - if (cur_task->task_info_->status() == ::openmldb::api::kFailed) { - has_failed = true; - break; - } else if (cur_task->task_info_->status() != ::openmldb::api::kDone) { - has_done = false; - break; - } - } - if (has_failed) { - PDLOG(INFO, - "update task status from[%s] to[kFailed]. op_id[%lu], " - "task_type[%s]", - ::openmldb::api::TaskStatus_Name(task->task_info_->status()).c_str(), op_data->op_info_.op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str()); - task->task_info_->set_status(::openmldb::api::kFailed); - } else if (has_done) { - PDLOG(INFO, - "update task status from[%s] to[kDone]. op_id[%lu], " - "task_type[%s]", - ::openmldb::api::TaskStatus_Name(task->task_info_->status()).c_str(), op_data->op_info_.op_id(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str()); - task->task_info_->set_status(::openmldb::api::kDone); - } - } - if (task->task_info_->status() == ::openmldb::api::kDone) { + task->UpdateStatusFromSubTask(); + if (task->GetStatus() == ::openmldb::api::kDone) { uint32_t cur_task_index = op_data->op_info_.task_index(); op_data->op_info_.set_task_index(cur_task_index + 1); std::string value; op_data->op_info_.SerializeToString(&value); - std::string node = zk_path_.op_data_path_ + "/" + std::to_string(op_data->op_info_.op_id()); + std::string node = absl::StrCat(zk_path_.op_data_path_, "/", op_data->op_info_.op_id()); if (zk_client_->SetNodeValue(node, value)) { - DEBUGLOG("set zk status value success. node[%s] value[%s]", node.c_str(), value.c_str()); + PDLOG(INFO, "set zk status value success. node[%s] value[%s]", node.c_str(), value.c_str()); op_data->task_list_.pop_front(); continue; } // revert task index op_data->op_info_.set_task_index(cur_task_index); - PDLOG(WARNING, - "set zk status value failed! node[%s] op_id[%lu] op_type[%s] " - "task_index[%u]", - node.c_str(), op_data->op_info_.op_id(), - ::openmldb::api::OPType_Name(op_data->op_info_.op_type()).c_str(), op_data->op_info_.task_index()); + PDLOG(WARNING, "set zk status value failed! node[%s] op_id[%lu] op_type[%s] task_index[%u]", + node.c_str(), op_data->GetOpId(), op_data->GetReadableType().c_str(), + op_data->op_info_.task_index()); } } return 0; @@ -1787,14 +1676,10 @@ void NameServerImpl::UpdateTaskMapStatus(uint64_t remote_op_id, uint64_t op_id, if (status == ::openmldb::api::kFailed || status == ::openmldb::api::kCanceled) { task_info->set_status(status); if (status == ::openmldb::api::kFailed) { - DEBUGLOG( - "update task status from[kDoing] to[kFailed]. " - "op_id[%lu], task_type[%s]", + DEBUGLOG("update task status from[kDoing] to[kFailed]. op_id[%lu], task_type[%s]", task_info->op_id(), ::openmldb::api::TaskType_Name(task_info->task_type()).c_str()); } else { - DEBUGLOG( - "update task status from[kDoing] to[kCanceled]. " - "op_id[%lu], task_type[%s]", + DEBUGLOG("update task status from[kDoing] to[kCanceled]. op_id[%lu], task_type[%s]", task_info->op_id(), ::openmldb::api::TaskType_Name(task_info->task_type()).c_str()); } } @@ -1802,9 +1687,7 @@ void NameServerImpl::UpdateTaskMapStatus(uint64_t remote_op_id, uint64_t op_id, if (status == ::openmldb::api::kDone && task_info->status() != ::openmldb::api::kFailed && task_info->status() != ::openmldb::api::kCanceled) { task_info->set_status(status); - DEBUGLOG( - "update task status from[kDoing] to[kDone]. " - "op_id[%lu], task_type[%s]", + DEBUGLOG("update task status from[kDoing] to[kDone]. op_id[%lu], task_type[%s]", task_info->op_id(), ::openmldb::api::TaskType_Name(task_info->task_type()).c_str()); } } @@ -1921,7 +1804,7 @@ void NameServerImpl::DeleteTask(const std::vector& done_task_vec) { continue; } for (iter = task_vec_[idx].begin(); iter != task_vec_[idx].end(); iter++) { - if ((*iter)->op_info_.op_id() == op_id) { + if ((*iter)->GetOpId() == op_id) { op_data = *iter; index = idx; break; @@ -1935,13 +1818,12 @@ void NameServerImpl::DeleteTask(const std::vector& done_task_vec) { PDLOG(WARNING, "has not found op[%lu] in running op", op_id); continue; } - std::string node = zk_path_.op_data_path_ + "/" + std::to_string(op_id); + std::string node = absl::StrCat(zk_path_.op_data_path_, "/", op_id); if (!op_data->task_list_.empty() && - op_data->task_list_.front()->task_info_->status() == ::openmldb::api::kFailed) { - op_data->op_info_.set_task_status(::openmldb::api::kFailed); + op_data->task_list_.front()->GetStatus() == ::openmldb::api::kFailed) { + op_data->SetTaskStatus(::openmldb::api::kFailed); op_data->op_info_.set_end_time(::baidu::common::timer::now_time()); - PDLOG(WARNING, "set op[%s] status failed. op_id[%lu]", - ::openmldb::api::OPType_Name(op_data->op_info_.op_type()).c_str(), op_id); + PDLOG(WARNING, "set op[%s] status failed. op_id[%lu]", op_data->GetReadableType().c_str(), op_id); std::string value; op_data->op_info_.SerializeToString(&value); if (!zk_client_->SetNodeValue(node, value)) { @@ -1954,8 +1836,8 @@ void NameServerImpl::DeleteTask(const std::vector& done_task_vec) { if (zk_client_->DeleteNode(node)) { PDLOG(INFO, "delete zk op node[%s] success.", node.c_str()); op_data->op_info_.set_end_time(::baidu::common::timer::now_time()); - if (op_data->op_info_.task_status() == ::openmldb::api::kDoing) { - op_data->op_info_.set_task_status(::openmldb::api::kDone); + if (op_data->GetTaskStatus() == ::openmldb::api::kDoing) { + op_data->SetTaskStatus(::openmldb::api::kDone); op_data->task_list_.clear(); } done_op_list_.push_back(op_data); @@ -1992,45 +1874,39 @@ void NameServerImpl::ProcessTask() { continue; } std::shared_ptr op_data = op_list.front(); - if (op_data->task_list_.empty() || op_data->op_info_.task_status() == ::openmldb::api::kFailed || - op_data->op_info_.task_status() == ::openmldb::api::kCanceled) { + if (op_data->task_list_.empty() || op_data->GetTaskStatus() == ::openmldb::api::kFailed || + op_data->GetTaskStatus() == ::openmldb::api::kCanceled) { continue; } - if (op_data->op_info_.task_status() == ::openmldb::api::kInited) { + if (op_data->GetTaskStatus() == ::openmldb::api::kInited) { op_data->op_info_.set_start_time(::baidu::common::timer::now_time()); - op_data->op_info_.set_task_status(::openmldb::api::kDoing); + op_data->SetTaskStatus(::openmldb::api::kDoing); std::string value; op_data->op_info_.SerializeToString(&value); - std::string node = zk_path_.op_data_path_ + "/" + std::to_string(op_data->op_info_.op_id()); + std::string node = absl::StrCat(zk_path_.op_data_path_ , "/", op_data->GetOpId()); if (!zk_client_->SetNodeValue(node, value)) { - PDLOG(WARNING, "set zk op status value failed. node[%s] value[%s]", node.c_str(), - value.c_str()); - op_data->op_info_.set_task_status(::openmldb::api::kInited); + PDLOG(WARNING, "set zk op status value failed. node[%s] value[%s]", + node.c_str(), value.c_str()); + op_data->SetTaskStatus(::openmldb::api::kInited); continue; } } std::shared_ptr task = op_data->task_list_.front(); - if (task->task_info_->status() == ::openmldb::api::kFailed) { + if (task->GetStatus() == ::openmldb::api::kFailed) { PDLOG(WARNING, "task[%s] run failed, terminate op[%s]. op_id[%lu]", - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str(), - ::openmldb::api::OPType_Name(task->task_info_->op_type()).c_str(), task->task_info_->op_id()); + task->GetReadableType().c_str(), task->GetReadableOpType().c_str(), task->GetOpId()); } else if (task->task_info_->status() == ::openmldb::api::kInited) { - DEBUGLOG("run task. opid[%lu] op_type[%s] task_type[%s]", task->task_info_->op_id(), - ::openmldb::api::OPType_Name(task->task_info_->op_type()).c_str(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str()); + DEBUGLOG("run task. opid[%lu] op_type[%s] task_type[%s]", + task->GetOpId(), task->GetReadableOpType().c_str(), task->GetReadableType().c_str()); task_thread_pool_.AddTask(task->fun_); - task->task_info_->set_status(::openmldb::api::kDoing); - } else if (task->task_info_->status() == ::openmldb::api::kDoing) { - if (::baidu::common::timer::now_time() - op_data->op_info_.start_time() > - FLAGS_name_server_op_execute_timeout / 1000) { - PDLOG(INFO, - "The execution time of op is too long. " - "opid[%lu] op_type[%s] cur task_type[%s] " + task->SetStatus(::openmldb::api::kDoing); + } else if (task->GetStatus() == ::openmldb::api::kDoing) { + uint64_t cur_ts = ::baidu::common::timer::now_time(); + if (cur_ts - op_data->op_info_.start_time() > FLAGS_name_server_op_execute_timeout / 1000) { + PDLOG(INFO, "The execution time of op is too long. opid[%lu] op_type[%s] cur task_type[%s] " "start_time[%lu] cur_time[%lu]", - task->task_info_->op_id(), - ::openmldb::api::OPType_Name(task->task_info_->op_type()).c_str(), - ::openmldb::api::TaskType_Name(task->task_info_->task_type()).c_str(), - op_data->op_info_.start_time(), ::baidu::common::timer::now_time()); + task->GetOpId(), task->GetReadableOpType().c_str(), task->GetReadableType().c_str(), + op_data->op_info_.start_time(), cur_ts); cv_.wait_for(lock, std::chrono::milliseconds(FLAGS_name_server_task_wait_time)); } } @@ -2856,6 +2732,9 @@ void NameServerImpl::ShowOPStatus(RpcController* controller, const ShowOPStatusR std::lock_guard lock(mu_); DeleteDoneOP(); for (const auto& op_data : done_op_list_) { + if (request->has_op_id() && op_data->op_info_.op_id() != request->op_id()) { + continue; + } if (request->has_db() && op_data->op_info_.db() != request->db()) { continue; } @@ -2872,6 +2751,9 @@ void NameServerImpl::ShowOPStatus(RpcController* controller, const ShowOPStatusR continue; } for (const auto& op_data : op_list) { + if (request->has_op_id() && op_data->op_info_.op_id() != request->op_id()) { + continue; + } if (request->has_name() && op_data->op_info_.name() != request->name()) { continue; } @@ -4021,16 +3903,16 @@ int NameServerImpl::CreateAddReplicaSimplyRemoteOPTask(std::shared_ptr o } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kAddReplicaSimplyRemoteOP; - auto task = CreateTask(std::make_shared(op_index, op_type, leader_endpoint, tid, pid, - add_replica_data.endpoint(), add_replica_data.remote_tid())); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, + add_replica_data.endpoint(), add_replica_data.remote_tid()); if (!task) { PDLOG(WARNING, "create addreplica task failed. leader cluster tid[%u] replica cluster tid[%u] pid[%u]", tid, add_replica_data.remote_tid(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, add_replica_data.name(), - add_replica_data.db(), pid, add_replica_data.endpoint(), alias, add_replica_data.remote_tid())); + task = CreateTask(op_index, op_type, add_replica_data.name(), + add_replica_data.db(), pid, add_replica_data.endpoint(), alias, add_replica_data.remote_tid()); if (!task) { PDLOG(WARNING, "create addtableinfo task failed. tid[%u] pid[%u]", tid, pid); return -1; @@ -4111,17 +3993,14 @@ int NameServerImpl::CreateAddReplicaRemoteOPTask(std::shared_ptr op_data } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kAddReplicaRemoteOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, + task = CreateTask(op_index, op_type, leader_endpoint, tid, remote_tid, pid, endpoint); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create sendsnapshot task failed. leader cluster tid[%u] replica " @@ -4139,8 +4018,7 @@ int NameServerImpl::CreateAddReplicaRemoteOPTask(std::shared_ptr op_data } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared( - op_index, op_type, leader_endpoint, tid, pid, endpoint, remote_tid)); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint, remote_tid); if (!task) { PDLOG(WARNING, "create addreplica task failed. leader cluster tid[%u] replica cluster tid[%u] pid[%u]", tid, remote_tid, pid); @@ -4148,8 +4026,7 @@ int NameServerImpl::CreateAddReplicaRemoteOPTask(std::shared_ptr op_data } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; @@ -4164,8 +4041,7 @@ int NameServerImpl::CreateAddReplicaRemoteOPTask(std::shared_ptr op_data } } if (!endpoint_vec.empty()) { - task = CreateTask( - std::make_shared(op_index, op_type, name, alias, endpoint_vec, pid)); + task = CreateTask(op_index, op_type, name, alias, endpoint_vec, pid); if (!task) { PDLOG(WARNING, "create addreplicaNS remote task failed. leader cluster tid[%u] replica cluster tid[%u] pid[%u]", @@ -4175,8 +4051,7 @@ int NameServerImpl::CreateAddReplicaRemoteOPTask(std::shared_ptr op_data op_data->task_list_.push_back(task); } - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, alias, remote_tid)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, alias, remote_tid); if (!task) { PDLOG(WARNING, "create addtableinfo task failed. tid[%u] pid[%u]", tid, pid); return -1; @@ -4418,61 +4293,55 @@ int NameServerImpl::CreateAddReplicaOPTask(std::shared_ptr op_data) { } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kAddReplicaOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, + task = CreateTask(op_index, op_type, leader_endpoint, tid, tid, pid, request.endpoint()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create sendsnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, request.endpoint(), + task = CreateTask(op_index, op_type, request.endpoint(), request.name(), tid, pid, seg_cnt, false, table_info->storage_mode()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, request.endpoint())); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, request.endpoint()); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - request.name(), request.db(), pid, request.endpoint())); + task = CreateTask(op_index, op_type, + request.name(), request.db(), pid, request.endpoint()); if (!task) { PDLOG(WARNING, "create addtableinfo task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - request.name(), request.db(), pid, request.endpoint(), FLAGS_check_binlog_sync_progress_delta)); + task = CreateTask(op_index, op_type, + request.name(), request.db(), pid, request.endpoint(), FLAGS_check_binlog_sync_progress_delta); if (!task) { PDLOG(WARNING, "create checkbinlogsyncprogress task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared( - op_index, op_type, request.name(), request.db(), pid, request.endpoint(), false, true)); + task = CreateTask( + op_index, op_type, request.name(), request.db(), pid, request.endpoint(), false, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", request.name().c_str(), pid, request.endpoint().c_str()); @@ -4659,42 +4528,35 @@ int NameServerImpl::CreateMigrateTask(std::shared_ptr op_data) { } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kMigrateOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u] endpoint[%s]", tid, pid, leader_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, - tid, tid, pid, des_endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, tid, pid, des_endpoint); if (!task) { PDLOG(WARNING, "create sendsnapshot task failed. tid[%u] pid[%u] endpoint[%s] des_endpoint[%s]", tid, pid, leader_endpoint.c_str(), des_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u] endpoint[%s] des_endpoint[%s]", tid, pid, leader_endpoint.c_str(), des_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, des_endpoint, + task = CreateTask(op_index, op_type, des_endpoint, name, tid, pid, table_info->seg_cnt(), false, table_info->storage_mode()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u] endpoint[%s]", tid, pid, des_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid, des_endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, des_endpoint); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u] endpoint[%s] " @@ -4703,37 +4565,35 @@ int NameServerImpl::CreateMigrateTask(std::shared_ptr op_data) { return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, name, db, pid, des_endpoint)); + task = CreateTask(op_index, op_type, name, db, pid, des_endpoint); if (!task) { PDLOG(WARNING, "create addtableinfo task failed. tid[%u] pid[%u] endpoint[%s] des_endpoint[%s]", tid, pid, leader_endpoint.c_str(), des_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, des_endpoint, FLAGS_check_binlog_sync_progress_delta)); + task = CreateTask(op_index, op_type, + name, db, pid, des_endpoint, FLAGS_check_binlog_sync_progress_delta); if (!task) { PDLOG(WARNING, "create CheckBinlogSyncProgressTask failed. name[%s] pid[%u]", name.c_str(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, src_endpoint)); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, src_endpoint); if (!task) { PDLOG(WARNING, "create delreplica task failed. tid[%u] pid[%u] leader[%s] follower[%s]", tid, pid, leader_endpoint.c_str(), src_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, src_endpoint, des_endpoint)); + task = CreateTask(op_index, op_type, name, db, pid, src_endpoint, des_endpoint); if (!task) { PDLOG(WARNING, "create update table info task failed. tid[%u] pid[%u] endpoint[%s] des_endpoint[%s]", tid, pid, src_endpoint.c_str(), des_endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, src_endpoint, tid, pid)); + task = CreateTask(op_index, op_type, src_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create droptable task failed. tid[%u] pid[%u] endpoint[%s]", tid, pid, src_endpoint.c_str()); return -1; @@ -4878,7 +4738,7 @@ int NameServerImpl::AddOPTask(const ::openmldb::api::TaskInfo& task_info, ::open } std::shared_ptr<::openmldb::api::TaskInfo> NameServerImpl::FindTask(uint64_t op_id, - ::openmldb::api::TaskType task_type) { + ::openmldb::api::TaskType task_type) { auto iter = task_map_.find(op_id); if (iter == task_map_.end()) { return std::shared_ptr<::openmldb::api::TaskInfo>(); @@ -4941,10 +4801,10 @@ int NameServerImpl::AddOPData(const std::shared_ptr& op_data, uint32_t c op_data->op_info_.set_vec_idx(idx); std::string value; op_data->op_info_.SerializeToString(&value); - std::string node = zk_path_.op_data_path_ + "/" + std::to_string(op_data->op_info_.op_id()); + std::string node = absl::StrCat(zk_path_.op_data_path_, "/", op_data->GetOpId()); if (!zk_client_->CreateNode(node, value)) { - PDLOG(WARNING, "create op node[%s] failed. op_index[%lu] op_type[%s]", node.c_str(), op_data->op_info_.op_id(), - ::openmldb::api::OPType_Name(op_data->op_info_.op_type()).c_str()); + PDLOG(WARNING, "create op node[%s] failed. op_index[%lu] op_type[%s]", + node.c_str(), op_data->GetOpId(), op_data->GetReadableType().c_str()); return -1; } uint64_t parent_id = op_data->op_info_.parent_id(); @@ -4960,8 +4820,7 @@ int NameServerImpl::AddOPData(const std::shared_ptr& op_data, uint32_t c task_vec_[idx].insert(iter, op_data); } else { PDLOG(WARNING, "not found parent_id[%lu] with index[%u]. add op[%lu] failed, op_type[%s]", - parent_id, idx, op_data->op_info_.op_id(), - ::openmldb::api::OPType_Name(op_data->op_info_.op_type()).c_str()); + parent_id, idx, op_data->GetOpId(), op_data->GetReadableType().c_str()); return -1; } } else { @@ -4979,20 +4838,17 @@ void NameServerImpl::DeleteDoneOP() { while (done_op_list_.size() > (uint32_t)FLAGS_max_op_num) { std::shared_ptr op_data = done_op_list_.front(); if (op_data->op_info_.task_status() == ::openmldb::api::TaskStatus::kFailed) { - std::string node = zk_path_.op_data_path_ + "/" + std::to_string(op_data->op_info_.op_id()); + std::string node = absl::StrCat(zk_path_.op_data_path_, "/", op_data->GetOpId()); if (zk_client_->DeleteNode(node)) { PDLOG(INFO, "delete zk op node[%s] success.", node.c_str()); op_data->task_list_.clear(); } else { - PDLOG(WARNING, "delete zk op_node failed. op_id[%lu] node[%s]", op_data->op_info_.op_id(), - node.c_str()); + PDLOG(WARNING, "delete zk op_node failed. op_id[%lu] node[%s]", op_data->GetOpId(), node.c_str()); break; } } - PDLOG(INFO, - "done_op_list size[%u] is greater than the max_op_num[%u], " - "delete op[%lu]", - done_op_list_.size(), (uint32_t)FLAGS_max_op_num, op_data->op_info_.op_id()); + PDLOG(INFO, "done_op_list size[%u] is greater than the max_op_num[%u], delete op[%lu]", + done_op_list_.size(), (uint32_t)FLAGS_max_op_num, op_data->GetOpId()); done_op_list_.pop_front(); } } @@ -5111,9 +4967,7 @@ void NameServerImpl::SchedMakeSnapshot() { continue; } if (part_iter->second < 1) { - PDLOG(WARNING, - "table %s pid %u snapshot offset is %lu, too small, skip " - "makesnapshot", + PDLOG(WARNING, "table %s pid %u snapshot offset is %lu, too small, skip makesnapshot", table.second->name().c_str(), part.pid(), part_iter->second); continue; } @@ -5160,9 +5014,9 @@ void NameServerImpl::UpdateTableStatus() { continue; } for (int pos = 0; pos < tablet_status_response.all_table_status_size(); pos++) { - std::string key = std::to_string(tablet_status_response.all_table_status(pos).tid()) + "_" + - std::to_string(tablet_status_response.all_table_status(pos).pid()) + "_" + kv.first; - pos_response.insert(std::make_pair(key, tablet_status_response.all_table_status(pos))); + std::string key = absl::StrCat(tablet_status_response.all_table_status(pos).tid(), "_", + tablet_status_response.all_table_status(pos).pid(), "_", kv.first); + pos_response.emplace(key, tablet_status_response.all_table_status(pos)); } } if (pos_response.empty()) { @@ -5191,14 +5045,13 @@ void NameServerImpl::UpdateTableStatusFun( } for (int idx = 0; idx < kv.second->table_partition_size(); idx++) { uint32_t pid = kv.second->table_partition(idx).pid(); - ::openmldb::nameserver::TablePartition* table_partition = kv.second->mutable_table_partition(idx); - ::google::protobuf::RepeatedPtrField<::openmldb::nameserver::PartitionMeta>* partition_meta_field = - table_partition->mutable_partition_meta(); + auto table_partition = kv.second->mutable_table_partition(idx); + auto partition_meta_field = table_partition->mutable_partition_meta(); for (int meta_idx = 0; meta_idx < kv.second->table_partition(idx).partition_meta_size(); meta_idx++) { std::string endpoint = kv.second->table_partition(idx).partition_meta(meta_idx).endpoint(); bool tablet_has_partition = false; - ::openmldb::nameserver::PartitionMeta* partition_meta = partition_meta_field->Mutable(meta_idx); - std::string pos_key = std::to_string(tid) + "_" + std::to_string(pid) + "_" + endpoint; + auto partition_meta = partition_meta_field->Mutable(meta_idx); + std::string pos_key = absl::StrCat(tid, "_", pid, "_", endpoint); auto pos_response_iter = pos_response.find(pos_key); if (pos_response_iter != pos_response.end()) { const ::openmldb::api::TableStatus& table_status = pos_response_iter->second; @@ -5279,22 +5132,21 @@ int NameServerImpl::CreateDelReplicaOPTask(std::shared_ptr op_data) { } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kDelReplicaOP; - auto task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint)); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create delreplica task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, name, db, pid, endpoint)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint); if (!task) { PDLOG(WARNING, "create deltableinfo task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, endpoint, tid, pid)); + task = CreateTask(op_index, op_type, endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create droptable task failed. tid[%u] pid[%u] endpoint[%s]", tid, pid, endpoint.c_str()); return -1; @@ -5322,15 +5174,14 @@ int NameServerImpl::CreateDelReplicaRemoteOPTask(std::shared_ptr op_data } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kDelReplicaRemoteOP; - auto task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint)); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create delreplica task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, name, db, pid, endpoint, 1)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, 1); if (!task) { PDLOG(WARNING, "create deltableinfo task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -5350,9 +5201,7 @@ int NameServerImpl::CreateOfflineReplicaOP(const std::string& name, const std::s return -1; } if (CreateOfflineReplicaTask(op_data) < 0) { - PDLOG(WARNING, - "create offline replica task failed. table[%s] pid[%u] " - "endpoint[%s]", + PDLOG(WARNING, "create offline replica task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); return -1; } @@ -5387,16 +5236,15 @@ int NameServerImpl::CreateOfflineReplicaTask(std::shared_ptr op_data) { return -1; } auto op_type = ::openmldb::api::OPType::kOfflineReplicaOP; - auto task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint)); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create delreplica task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared( - op_index, op_type, name, db, pid, endpoint, false, false)); + task = CreateTask( + op_index, op_type, name, db, pid, endpoint, false, false); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -5516,22 +5364,21 @@ int NameServerImpl::CreateChangeLeaderOPTask(std::shared_ptr op_data) { for (int idx = 0; idx < change_leader_data.follower_size(); idx++) { follower_endpoint.push_back(change_leader_data.follower(idx)); } - auto task = CreateTask( - std::make_shared(op_index, op_type, name, db, tid, pid, follower_endpoint)); + auto task = CreateTask(op_index, op_type, name, db, tid, pid, follower_endpoint); if (!task) { PDLOG(WARNING, "create selectleader task failed. table[%s] pid[%u]", name.c_str(), pid); return -1; } PDLOG(INFO, "create SelectLeader task success. name[%s] tid[%u] pid[%u]", name.c_str(), tid, pid); op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type)); + task = CreateTask(op_index, op_type); if (!task) { PDLOG(WARNING, "create changeleader task failed. table[%s] pid[%u]", name.c_str(), pid); return -1; } PDLOG(INFO, "create ChangeLeader task success. name[%s] pid[%u]", name.c_str(), pid); op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type)); + task = CreateTask(op_index, op_type); if (!task) { PDLOG(WARNING, "create updateleaderinfo task failed. table[%s] pid[%u]", name.c_str(), pid); return -1; @@ -5661,8 +5508,7 @@ int NameServerImpl::CreateRecoverTableOPTask(std::shared_ptr op_data) { PDLOG(WARNING, "endpoint is leader. table[%s] pid[%u]", name.c_str(), pid); return -1; } - auto task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint)); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create delreplica task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -5670,8 +5516,8 @@ int NameServerImpl::CreateRecoverTableOPTask(std::shared_ptr op_data) { } op_data->task_list_.push_back(task); } - auto task = CreateTask(std::make_shared( - op_index, op_type, name, db, pid, endpoint, offset_delta, concurrency)); + auto task = CreateTask( + op_index, op_type, name, db, pid, endpoint, offset_delta, concurrency); if (!task) { PDLOG(WARNING, "create RecoverTable task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -5715,9 +5561,7 @@ void NameServerImpl::RecoverEndpointTable(const std::string& name, const std::st std::string leader_endpoint = partition_meta.endpoint(); auto tablet_iter = tablets_.find(leader_endpoint); if (tablet_iter == tablets_.end()) { - PDLOG(WARNING, - "can not find the leader endpoint[%s]'s " - "client. op_id[%lu]", + PDLOG(WARNING, "can not find the leader endpoint[%s]'s client. op_id[%lu]", leader_endpoint.c_str(), task_info->op_id()); task_info->set_status(::openmldb::api::TaskStatus::kFailed); return; @@ -5737,18 +5581,14 @@ void NameServerImpl::RecoverEndpointTable(const std::string& name, const std::st } if (partition_meta.endpoint() == endpoint) { if (partition_meta.is_alive()) { - PDLOG(INFO, - "endpoint[%s] is alive, need not recover. " - "name[%s] pid[%u]", + PDLOG(INFO, "endpoint[%s] is alive, need not recover. name[%s] pid[%u]", endpoint.c_str(), name.c_str(), pid); task_info->set_status(::openmldb::api::TaskStatus::kDone); return; } auto tablet_iter = tablets_.find(endpoint); if (tablet_iter == tablets_.end()) { - PDLOG(WARNING, - "can not find the endpoint[%s]'s client. " - "op_id[%lu]", + PDLOG(WARNING, "can not find the endpoint[%s]'s client. op_id[%lu]", endpoint.c_str(), task_info->op_id()); task_info->set_status(::openmldb::api::TaskStatus::kFailed); return; @@ -5779,9 +5619,7 @@ void NameServerImpl::RecoverEndpointTable(const std::string& name, const std::st uint64_t term = 0; uint64_t offset = 0; if (!tablet_ptr->client_->GetTermPair(tid, pid, storage_mode, term, offset, has_table, is_leader)) { - PDLOG(WARNING, - "GetTermPair failed. name[%s] tid[%u] pid[%u] endpoint[%s] " - "op_id[%lu]", + PDLOG(WARNING, "GetTermPair failed. name[%s] tid[%u] pid[%u] endpoint[%s] op_id[%lu]", name.c_str(), tid, pid, endpoint.c_str(), task_info->op_id()); task_info->set_status(::openmldb::api::TaskStatus::kFailed); return; @@ -5794,17 +5632,13 @@ void NameServerImpl::RecoverEndpointTable(const std::string& name, const std::st CreateReLoadTableOP(name, db, pid, endpoint, task_info->op_id(), concurrency); } task_info->set_status(::openmldb::api::TaskStatus::kDone); - PDLOG(INFO, - "update task status from[kDoing] to[kDone]. op_id[%lu], " - "task_type[%s]", + PDLOG(INFO, "update task status from[kDoing] to[kDone]. op_id[%lu], task_type[%s]", task_info->op_id(), ::openmldb::api::TaskType_Name(task_info->task_type()).c_str()); return; } if (has_table && is_leader) { if (!tablet_ptr->client_->ChangeRole(tid, pid, false, 0)) { - PDLOG(WARNING, - "change role failed. name[%s] tid[%u] pid[%u] endpoint[%s] " - "op_id[%lu]", + PDLOG(WARNING, "change role failed. name[%s] tid[%u] pid[%u] endpoint[%s] op_id[%lu]", name.c_str(), tid, pid, endpoint.c_str(), task_info->op_id()); task_info->set_status(::openmldb::api::TaskStatus::kFailed); return; @@ -5814,9 +5648,7 @@ void NameServerImpl::RecoverEndpointTable(const std::string& name, const std::st } if (!has_table) { if (!tablet_ptr->client_->DeleteBinlog(tid, pid, storage_mode)) { - PDLOG(WARNING, - "delete binlog failed. name[%s] tid[%u] pid[%u] endpoint[%s] " - "op_id[%lu]", + PDLOG(WARNING, "delete binlog failed. name[%s] tid[%u] pid[%u] endpoint[%s] op_id[%lu]", name.c_str(), tid, pid, endpoint.c_str(), task_info->op_id()); task_info->set_status(::openmldb::api::TaskStatus::kFailed); return; @@ -5919,52 +5751,45 @@ int NameServerImpl::CreateReAddReplicaTask(std::shared_ptr op_data) { } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kReAddReplicaOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, tid, pid, endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create sendsnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, endpoint, + task = CreateTask(op_index, op_type, endpoint, name, tid, pid, seg_cnt, false, table_info->storage_mode()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, offset_delta)); + task = CreateTask(op_index, op_type, + name, db, pid, endpoint, offset_delta); if (!task) { PDLOG(WARNING, "create CheckBinlogSyncProgressTask failed. name[%s] pid[%u]", name.c_str(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, endpoint, false, true)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, false, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6039,58 +5864,51 @@ int NameServerImpl::CreateReAddReplicaWithDropTask(std::shared_ptr op_da } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kReAddReplicaWithDropOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, endpoint, tid, pid)); + task = CreateTask(op_index, op_type, endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create droptable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, tid, pid, endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create sendsnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, endpoint, + task = CreateTask(op_index, op_type, endpoint, name, tid, pid, seg_cnt, false, table_info->storage_mode()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, offset_delta)); + task = CreateTask(op_index, op_type, + name, db, pid, endpoint, offset_delta); if (!task) { PDLOG(WARNING, "create CheckBinlogSyncProgressTask failed. name[%s] pid[%u]", name.c_str(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, endpoint, false, true)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, false, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6168,45 +5986,39 @@ int NameServerImpl::CreateReAddReplicaNoSendTask(std::shared_ptr op_data } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kReAddReplicaNoSendOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create pausesnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, endpoint, + task = CreateTask(op_index, op_type, endpoint, name, tid, pid, seg_cnt, false, table_info->storage_mode()); - task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid); - task = CreateTask(task_meta); + task = CreateTask(op_index, op_type, leader_endpoint, tid, pid); if (!task) { PDLOG(WARNING, "create recoversnapshot task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, offset_delta)); + task = CreateTask(op_index, op_type, + name, db, pid, endpoint, offset_delta); if (!task) { PDLOG(WARNING, "create CheckBinlogSyncProgressTask failed. name[%s] pid[%u]", name.c_str(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, endpoint, false, true)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, false, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6299,23 +6111,20 @@ int NameServerImpl::CreateReAddReplicaSimplifyTask(std::shared_ptr op_da } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kReAddReplicaSimplifyOP; - std::shared_ptr task_meta; - task_meta = std::make_shared(op_index, op_type, leader_endpoint, tid, pid, endpoint); - auto task = CreateTask(task_meta); + auto task = CreateTask(op_index, op_type, leader_endpoint, tid, pid, endpoint); if (!task) { PDLOG(WARNING, "create addreplica task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, offset_delta)); + task = CreateTask(op_index, op_type, + name, db, pid, endpoint, offset_delta); if (!task) { PDLOG(WARNING, "create CheckBinlogSyncProgressTask failed. name[%s] pid[%u]", name.c_str(), pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, endpoint, false, true)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, false, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6363,7 +6172,7 @@ int NameServerImpl::DropTableRemoteTask(std::shared_ptr op_data) { } uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kDropTableRemoteOP; - auto task = CreateTask(std::make_shared(op_index, op_type, name, db, alias)); + auto task = CreateTask(op_index, op_type, name, db, alias); if (!task) { PDLOG(WARNING, "create DropTableRemote task failed. table[%s] pid[%u]", name.c_str(), op_data->op_info_.pid()); return -1; @@ -6379,10 +6188,8 @@ int NameServerImpl::CreateTableRemoteOP(const ::openmldb::nameserver::TableInfo& const std::string& alias, uint64_t parent_id, uint32_t concurrency) { CreateTableData create_table_data; create_table_data.set_alias(alias); - ::openmldb::nameserver::TableInfo* table_info_p = create_table_data.mutable_table_info(); - table_info_p->CopyFrom(table_info); - ::openmldb::nameserver::TableInfo* remote_table_info_p = create_table_data.mutable_remote_table_info(); - remote_table_info_p->CopyFrom(remote_table_info); + create_table_data.mutable_table_info()->CopyFrom(table_info); + create_table_data.mutable_remote_table_info()->CopyFrom(remote_table_info); std::string value; create_table_data.SerializeToString(&value); std::string name = table_info.name(); @@ -6390,15 +6197,13 @@ int NameServerImpl::CreateTableRemoteOP(const ::openmldb::nameserver::TableInfo& uint32_t pid = INVALID_PID; std::shared_ptr op_data; if (CreateOPData(::openmldb::api::OPType::kCreateTableRemoteOP, value, op_data, name, db, pid, parent_id) < 0) { - PDLOG(WARNING, - "create CreateTableRemoteOP data error. table[%s] pid[%u] " - "alias[%s]", + PDLOG(WARNING, "create CreateTableRemoteOP data error. table[%s] pid[%u] alias[%s]", name.c_str(), pid, alias.c_str()); return -1; } if (CreateTableRemoteTask(op_data) < 0) { - PDLOG(WARNING, "create CreateTableRemote task failed. table[%s] pid[%u] alias[%s]", table_info.name().c_str(), - pid, alias.c_str()); + PDLOG(WARNING, "create CreateTableRemote task failed. table[%s] pid[%u] alias[%s]", + table_info.name().c_str(), pid, alias.c_str()); return -1; } op_data->op_info_.set_for_replica_cluster(1); @@ -6418,10 +6223,10 @@ int NameServerImpl::CreateTableRemoteTask(std::shared_ptr op_data) { return -1; } std::string alias = create_table_data.alias(); - ::openmldb::nameserver::TableInfo remote_table_info = create_table_data.remote_table_info(); + auto remote_table_info = create_table_data.remote_table_info(); uint64_t op_index = op_data->op_info_.op_id(); auto op_type = ::openmldb::api::OPType::kCreateTableRemoteOP; - auto task = CreateTask(std::make_shared(op_index, op_type, remote_table_info, alias)); + auto task = CreateTask(op_index, op_type, remote_table_info, alias); if (!task) { PDLOG(WARNING, "create CreateTableRemote task failed. table[%s] pid[%u]", remote_table_info.name().c_str(), op_data->op_info_.pid()); @@ -6429,7 +6234,7 @@ int NameServerImpl::CreateTableRemoteTask(std::shared_ptr op_data) { } op_data->task_list_.push_back(task); - ::openmldb::nameserver::TableInfo table_info = create_table_data.table_info(); + auto table_info = create_table_data.table_info(); uint32_t tid = table_info.tid(); uint32_t remote_tid = remote_table_info.tid(); std::string name = table_info.name(); @@ -6448,8 +6253,8 @@ int NameServerImpl::CreateTableRemoteTask(std::shared_ptr op_data) { PDLOG(WARNING, "get leader failed. table[%s] pid[%u]", name.c_str(), pid); return -1; } - task = CreateTask(std::make_shared( - op_index, op_type, leader_endpoint, tid, pid, endpoint, remote_tid, idx)); + task = CreateTask( + op_index, op_type, leader_endpoint, tid, pid, endpoint, remote_tid, idx); if (!task) { PDLOG(WARNING, "create addreplica task failed. leader cluster tid[%u] replica cluster tid[%u] pid[%u]", @@ -6457,8 +6262,8 @@ int NameServerImpl::CreateTableRemoteTask(std::shared_ptr op_data) { return -1; } op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, endpoint, alias, remote_tid)); + task = CreateTask(op_index, op_type, + name, db, pid, endpoint, alias, remote_tid); if (!task) { PDLOG(WARNING, "create addtableinfo task failed. tid[%u] pid[%u]", tid, pid); return -1; @@ -6540,19 +6345,16 @@ int NameServerImpl::CreateReLoadTableTask(std::shared_ptr op_data) { } uint32_t tid = table_info->tid(); uint32_t seg_cnt = table_info->seg_cnt(); - std::shared_ptr task_meta; auto op_type = ::openmldb::api::OPType::kReLoadTableOP; uint64_t op_index = op_data->op_info_.op_id(); - task_meta = std::make_shared(op_index, op_type, endpoint, + auto task = CreateTask(op_index, op_type, endpoint, name, tid, pid, seg_cnt, true, table_info->storage_mode()); - auto task = CreateTask(task_meta); if (!task) { PDLOG(WARNING, "create loadtable task failed. tid[%u] pid[%u]", tid, pid); return -1; } op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, name, db, pid, endpoint, true, true)); + task = CreateTask(op_index, op_type, name, db, pid, endpoint, true, true); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6624,8 +6426,8 @@ int NameServerImpl::CreateUpdatePartitionStatusOPTask(std::shared_ptr op } uint64_t op_index = op_data->op_info_.op_id(); ::openmldb::api::OPType op_type = ::openmldb::api::OPType::kUpdatePartitionStatusOP; - auto task = CreateTask(std::make_shared( - op_index, op_type, name, db, pid, endpoint, is_leader, is_alive)); + auto task = CreateTask( + op_index, op_type, name, db, pid, endpoint, is_leader, is_alive); if (!task) { PDLOG(WARNING, "create update table alive status task failed. table[%s] pid[%u] endpoint[%s]", name.c_str(), pid, endpoint.c_str()); @@ -6694,17 +6496,30 @@ int NameServerImpl::MatchTermOffset(const std::string& name, const std::string& void NameServerImpl::WrapTaskFun(const boost::function& fun, std::shared_ptr<::openmldb::api::TaskInfo> task_info) { + std::string msg = absl::StrCat("op_id ", task_info->op_id(), " type ", + ::openmldb::api::TaskType_Name(task_info->task_type()), " ", Task::GetAdditionalMsg(*task_info)); if (!fun()) { task_info->set_status(::openmldb::api::TaskStatus::kFailed); - PDLOG(WARNING, "task[%s] run failed. op_id[%lu]", - ::openmldb::api::TaskType_Name(task_info->task_type()).c_str(), task_info->op_id()); + PDLOG(WARNING, "task run failed. %s", msg.c_str()); } - PDLOG(INFO, "task[%s] starts running. op_id[%lu]", ::openmldb::api::TaskType_Name(task_info->task_type()).c_str(), - task_info->op_id()); + PDLOG(INFO, "task starts running. %s", msg.c_str()); task_rpc_version_.fetch_add(1, std::memory_order_acq_rel); task_info->set_is_rpc_send(true); } +void NameServerImpl::WrapNormalTaskFun(const boost::function& fun, + std::shared_ptr<::openmldb::api::TaskInfo> task_info) { + std::string msg = absl::StrCat("op_id ", task_info->op_id(), " type ", + ::openmldb::api::TaskType_Name(task_info->task_type()), " ", Task::GetAdditionalMsg(*task_info)); + auto status = fun(); + if (!status.OK()) { + task_info->set_status(::openmldb::api::TaskStatus::kFailed); + PDLOG(WARNING, "task run failed. %s", msg.c_str()); + } + task_info->set_status(::openmldb::api::TaskStatus::kDone); + PDLOG(INFO, "task run success. %s", msg.c_str()); +} + std::shared_ptr NameServerImpl::CreateLoadTableRemoteTask(const std::string& alias, const std::string& name, const std::string& db, const std::string& endpoint, uint32_t pid, uint64_t op_index, @@ -7002,9 +6817,7 @@ void NameServerImpl::DelTableInfo(const std::string& name, const std::string& db } if (!has_found) { task_info->set_status(::openmldb::api::TaskStatus::kFailed); - PDLOG(INFO, - "not found endpoint[%s] in partition_meta. name[%s] pid[%u] " - "op_id[%lu]", + PDLOG(INFO, "not found endpoint[%s] in partition_meta. name[%s] pid[%u] op_id[%lu]", endpoint.c_str(), name.c_str(), pid, task_info->op_id()); return; } @@ -7504,9 +7317,7 @@ void NameServerImpl::UpdateLeaderInfo(std::shared_ptr<::openmldb::api::TaskInfo> leader_endpoint.c_str()); task_info->set_status(::openmldb::api::TaskStatus::kDone); // notify client to update table partition information - PDLOG(INFO, - "update task status from[kDoing] to[kDone]. op_id[%lu], " - "task_type[%s]", + PDLOG(INFO, "update task status from[kDoing] to[kDone]. op_id[%lu], task_type[%s]", task_info->op_id(), ::openmldb::api::TaskType_Name(task_info->task_type()).c_str()); return; } @@ -7601,14 +7412,10 @@ bool NameServerImpl::UpdateTTLOnTablet(const std::string& endpoint, int32_t tid, } bool ok = tablet->client_->UpdateTTL(tid, pid, ttl.ttl_type(), ttl.abs_ttl(), ttl.lat_ttl(), index_name); if (!ok) { - PDLOG(WARNING, - "fail to update ttl with tid %d, pid %d, abs_ttl %lu, lat_ttl " - "%lu, endpoint %s", + PDLOG(WARNING, "fail to update ttl with tid %d, pid %d, abs_ttl %lu, lat_ttl %lu, endpoint %s", tid, pid, ttl.abs_ttl(), ttl.lat_ttl(), endpoint.c_str()); } else { - PDLOG(INFO, - "update ttl with tid %d pid %d abs_ttl %lu, lat_ttl %lu endpoint " - "%s ok", + PDLOG(INFO, "update ttl with tid %d pid %d abs_ttl %lu, lat_ttl %lu endpoint %s ok", tid, pid, ttl.abs_ttl(), ttl.lat_ttl(), endpoint.c_str()); } return ok; @@ -8557,8 +8364,7 @@ base::Status NameServerImpl::AddMultiIndexs(const std::string& db, const std::st } std::vector indexs; for (int idx = 0; idx < column_keys.size(); idx++) { - int32_t index_pos = 0; - if (schema::IndexUtil::CheckExist(column_keys.Get(idx), table_info->column_key(), &index_pos)) { + if (schema::IndexUtil::IsExist(column_keys.Get(idx), table_info->column_key())) { return {ReturnCode::kIndexAlreadyExists, "index has already exist!"}; } indexs.push_back(column_keys.Get(idx)); @@ -8576,11 +8382,10 @@ base::Status NameServerImpl::AddMultiIndexs(const std::string& db, const std::st LOG(WARNING) << "endpoint[" << meta.endpoint() << "] is offline"; return {base::ReturnCode::kError, "endpoint" + meta.endpoint() + ""}; } - auto status = tablet->client_->AddMultiIndex(tid, pid, indexs, nullptr); - if (!status.OK()) { + if (!tablet->client_->AddMultiIndex(tid, pid, indexs, nullptr)) { LOG(WARNING) << "add index failed. tid " << tid << " pid " << pid << " endpoint " << meta.endpoint(); - return status; + return {base::ReturnCode::kError, "add index failed"}; } endpoint_set.insert(meta.endpoint()); } @@ -8624,8 +8429,6 @@ void NameServerImpl::AddIndex(RpcController* controller, const AddIndexRequest* const std::string& name = request->name(); const std::string& db = request->db(); std::shared_ptr table_info; - const std::string& index_name = request->column_key().index_name(); - std::map> tablet_client_map; if (!GetTableInfo(name, db, &table_info)) { base::SetResponseStatus(ReturnCode::kTableIsNotExist, "table does not exist!", response); LOG(WARNING) << "table[" << db << "." << name << "] does not exist!"; @@ -8637,46 +8440,27 @@ void NameServerImpl::AddIndex(RpcController* controller, const AddIndexRequest* LOG(WARNING) << "cannot add index. table " << name; return; } - if (table_info->column_key_size() == 0) { - base::SetResponseStatus(ReturnCode::kHasNotColumnKey, "table has no column key", response); - LOG(WARNING) << "table " << name << " has no column key"; - return; - } + std::vector<::openmldb::common::ColumnKey> column_key_vec; if (request->column_keys_size() > 0) { - auto status = AddMultiIndexs(db, name, table_info, request->column_keys()); - if (status.OK()) { - base::SetResponseOK(response); - } else { - base::SetResponseStatus(status, response); + for (const auto& column_key : request->column_keys()) { + column_key_vec.push_back(column_key); } - return; - } - int32_t index_pos = 0; - if (schema::IndexUtil::CheckExist(request->column_key(), table_info->column_key(), &index_pos)) { - base::SetResponseStatus(ReturnCode::kIndexAlreadyExists, "index has already exist!", response); - LOG(WARNING) << "index" << index_name << " has already exist! table " << name; - return; + } else { + column_key_vec.push_back(request->column_key()); } - if ((uint32_t)table_info->table_partition_size() > FLAGS_name_server_task_max_concurrency) { - base::SetResponseStatus(ReturnCode::kTooManyPartition, - "partition num is greater than name_server_task_max_concurrency", response); - LOG(WARNING) << "parition num[" << table_info->table_partition_size() - << "] is greater than name_server_task_max_concurrency[" << FLAGS_name_server_task_max_concurrency - << "] table " << name; - return; + for (const auto& column_key : column_key_vec) { + if (schema::IndexUtil::IsExist(column_key, table_info->column_key())) { + base::SetResponseStatus(ReturnCode::kIndexAlreadyExists, "index has already exist!", response); + LOG(WARNING) << "index" << column_key.index_name() << " has already exist! table " << name; + return; + } } std::map col_map; for (const auto& column_desc : table_info->column_desc()) { - col_map.insert(std::make_pair(column_desc.name(), column_desc)); + col_map.emplace(column_desc.name(), column_desc); } for (const auto& col : table_info->added_column_desc()) { - col_map.insert(std::make_pair(col.name(), col)); - } - auto status = schema::IndexUtil::CheckNewIndex(request->column_key(), *table_info); - if (!status.OK()) { - base::SetResponseStatus(ReturnCode::kWrongColumnKey, status.msg, response); - LOG(WARNING) << status.msg; - return; + col_map.emplace(col.name(), col); } std::map request_cols; for (const auto& col : request->cols()) { @@ -8686,7 +8470,7 @@ void NameServerImpl::AddIndex(RpcController* controller, const AddIndexRequest* << " it is not allow be index col"; return; } - request_cols.insert(std::make_pair(col.name(), col)); + request_cols.emplace(col.name(), col); } std::set need_create_cols; std::vector add_cols; @@ -8741,287 +8525,214 @@ void NameServerImpl::AddIndex(RpcController* controller, const AddIndexRequest* openmldb::common::VersionPair* pair = table_info->add_schema_versions(); pair->CopyFrom(new_pair); } - if (IsClusterMode()) { + if (IsClusterMode() && !request->skip_load_data()) { std::lock_guard lock(mu_); - for (uint32_t pid = 0; pid < (uint32_t)table_info->table_partition_size(); pid++) { - if (CreateAddIndexOP(name, db, pid, add_cols, request->column_key(), index_pos) < 0) { - LOG(WARNING) << "create AddIndexOP failed, table " << name << " pid " << pid; - break; - } + auto status = CreateAddIndexOP(name, db, column_key_vec); + if (!status.OK()) { + LOG(WARNING) << "create AddIndexOP failed, table " << name << " msg " << status.GetMsg(); + base::SetResponseStatus(ReturnCode::kAddIndexFailed, "add index failed. msg " + status.GetMsg(), response); + return; } } else { - std::shared_ptr tablet_ptr = nullptr; for (const auto& partition : table_info->table_partition()) { uint32_t pid = partition.pid(); for (const auto& meta : partition.partition_meta()) { - tablet_ptr = GetTablet(meta.endpoint()); + auto tablet_ptr = GetTablet(meta.endpoint()); if (!tablet_ptr) { PDLOG(WARNING, "endpoint[%s] can not find client", meta.endpoint().c_str()); base::SetResponseStatus(ReturnCode::kTabletIsNotHealthy, "tablet does not exist", response); return; } - if (!tablet_ptr->client_->AddIndex(table_info->tid(), pid, request->column_key(), nullptr)) { + if (!tablet_ptr->client_->AddMultiIndex(table_info->tid(), pid, column_key_vec, nullptr)) { base::SetResponseStatus(ReturnCode::kAddIndexFailed, "add index failed", response); return; } + if (!request->skip_load_data()) { + auto ret = tablet_ptr->client_->ExtractIndexData(table_info->tid(), pid, + (uint32_t)table_info->table_partition_size(), column_key_vec, 0, false, nullptr); + if (!ret) { + base::SetResponseStatus(ReturnCode::kAddIndexFailed, "extract multi index failed", response); + return; + } + } } } - std::vector<::openmldb::common::ColumnKey> column_keys = {request->column_key()}; - if (!tablet_ptr->client_->ExtractMultiIndexData( - table_info->tid(), 0, (uint32_t)table_info->table_partition_size(), column_keys)) { - base::SetResponseStatus(ReturnCode::kAddIndexFailed, "extract multi index failed", response); - return; - } - AddIndexToTableInfo(name, db, request->column_key(), table_info->column_key_size()); + AddIndexToTableInfo(name, db, column_key_vec, nullptr); } base::SetResponseOK(response); - LOG(INFO) << "add index. table[" << name << "] index[" << index_name << "]"; + LOG(INFO) << "add index. table[" << name << "] index count[" << column_key_vec.size() << "]"; } bool NameServerImpl::AddIndexToTableInfo(const std::string& name, const std::string& db, - const ::openmldb::common::ColumnKey& column_key, uint32_t index_pos) { - std::lock_guard lock(mu_); + const std::vector<::openmldb::common::ColumnKey>& column_key, + std::shared_ptr<::openmldb::api::TaskInfo> task_info) { std::shared_ptr<::openmldb::nameserver::TableInfo> table_info; + std::lock_guard lock(mu_); if (!GetTableInfoUnlock(name, db, &table_info)) { PDLOG(WARNING, "table[%s] does not exist!", name.c_str()); + if (task_info) { + task_info->set_status(::openmldb::api::TaskStatus::kFailed); + } return false; } - if (index_pos < (uint32_t)table_info->column_key_size()) { - ::openmldb::common::ColumnKey* cur_column_key = table_info->mutable_column_key(index_pos); - cur_column_key->CopyFrom(column_key); - } else { - ::openmldb::common::ColumnKey* cur_column_key = table_info->add_column_key(); - cur_column_key->CopyFrom(column_key); + for (const auto& cur_column_key : column_key) { + int index_pos = schema::IndexUtil::GetPosition(cur_column_key, table_info->column_key()); + if (index_pos >= 0) { + table_info->mutable_column_key(index_pos)->CopyFrom(cur_column_key); + } else { + table_info->add_column_key()->CopyFrom(cur_column_key); + } } UpdateZkTableNode(table_info); - PDLOG(INFO, "add index ok. table[%s] index[%s]", name.c_str(), column_key.index_name().c_str()); + PDLOG(INFO, "add index ok. table %s index cnt %d", name.c_str(), column_key.size()); + if (task_info) { + task_info->set_status(::openmldb::api::TaskStatus::kDone); + } return true; } -int NameServerImpl::CreateAddIndexOP(const std::string& name, const std::string& db, uint32_t pid, - const std::vector& new_cols, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx) { +base::Status NameServerImpl::CreateAddIndexOP(const std::string& name, const std::string& db, + const std::vector<::openmldb::common::ColumnKey>& column_key) { std::shared_ptr<::openmldb::nameserver::TableInfo> table_info; if (!GetTableInfoUnlock(name, db, &table_info)) { - PDLOG(WARNING, "table[%s] does not exist!", name.c_str()); - return -1; - } - // zk_op_sync_node only need to create once, so implement that through pid == 0 - if (pid == 0) { - std::string partition_num_value = std::to_string(table_info->table_partition_size()); - std::string table_sync_node = zk_path_.op_sync_path_ + "/" + std::to_string(table_info->tid()); - if (zk_client_->IsExistNode(table_sync_node) == 0) { - if (!zk_client_->SetNodeValue(table_sync_node, partition_num_value)) { - LOG(WARNING) << "set sync value failed. table " << name << "node " << table_sync_node; - return -1; - } - } else if (!zk_client_->CreateNode(table_sync_node, partition_num_value)) { - LOG(WARNING) << "create sync node failed. table " << name << " node " << table_sync_node; - return -1; - } + return {-1, "table does not exist"}; } std::shared_ptr op_data; AddIndexMeta add_index_meta; add_index_meta.set_name(name); - add_index_meta.set_pid(pid); - add_index_meta.set_idx(idx); + add_index_meta.set_pid(0); add_index_meta.set_db(db); - if (!new_cols.empty()) { - add_index_meta.set_skip_data(true); + for (const auto& cur_column_key : column_key) { + add_index_meta.add_column_keys()->CopyFrom(cur_column_key); } - ::openmldb::common::ColumnKey* cur_column_key = add_index_meta.mutable_column_key(); - cur_column_key->CopyFrom(column_key); std::string value; add_index_meta.SerializeToString(&value); - if (CreateOPData(kAddIndexOP, value, op_data, name, db, pid) < 0) { - PDLOG(WARNING, "create AddIndexOP data error. table %s pid %u", name.c_str(), pid); - return -1; + if (CreateOPData(api::kAddIndexOP, value, op_data, name, db, 0) < 0) { + return {-1, absl::StrCat("create AddIndexOP data failed. table ", name)}; } - if (CreateAddIndexOPTask(op_data) < 0) { - PDLOG(WARNING, "create AddIndexOP task failed. table[%s] pid[%u]", name.c_str(), pid); - return -1; + auto status = CreateAddIndexOPTask(op_data); + if (!status.OK()) { + return {-1, absl::StrCat("create AddIndexOP task failed. table ", name, " msg ", status.GetMsg())}; } if (AddOPData(op_data, FLAGS_name_server_task_max_concurrency) < 0) { - PDLOG(WARNING, "add op data failed. name[%s] pid[%u]", name.c_str(), pid); - return -1; + return {-1, absl::StrCat("add op data failed. name ", name)}; } - PDLOG(INFO, "create AddIndexOP op ok. op_id[%lu] name[%s] pid[%u]", op_data->op_info_.op_id(), name.c_str(), pid); - return 0; + PDLOG(INFO, "create AddIndexOP op ok. op_id[%lu] name[%s]", op_data->GetOpId(), name.c_str()); + return {}; } -int NameServerImpl::CreateAddIndexOPTask(std::shared_ptr op_data) { +base::Status NameServerImpl::CreateAddIndexOPTask(std::shared_ptr op_data) { AddIndexMeta add_index_meta; if (!add_index_meta.ParseFromString(op_data->op_info_.data())) { - PDLOG(WARNING, "parse AddIndexMeta failed. data[%s]", op_data->op_info_.data().c_str()); - return -1; + return {-1, absl::StrCat("parse AddIndexMeta failed. data ", op_data->op_info_.data())}; } - std::string name = op_data->op_info_.name(); - std::string db = op_data->op_info_.db(); - uint32_t pid = op_data->op_info_.pid(); + const std::string& name = op_data->op_info_.name(); + const std::string& db = op_data->op_info_.db(); + auto column_key_vec = schema::IndexUtil::Convert2Vector(add_index_meta.column_keys()); + uint64_t op_index = op_data->op_info_.op_id(); + auto op_type = api::kAddIndexOP; + return FillAddIndexTask(op_index, op_type, name, db, column_key_vec, &op_data->task_list_); +} + +base::Status NameServerImpl::FillAddIndexTask(uint64_t op_index, api::OPType op_type, + const std::string& name, const std::string& db, + const std::vector<::openmldb::common::ColumnKey>& column_key, + std::list>* task_list) { std::shared_ptr<::openmldb::nameserver::TableInfo> table_info; if (!GetTableInfoUnlock(name, db, &table_info)) { - PDLOG(WARNING, "get table info failed! name[%s]", name.c_str()); - return -1; + return {-1, absl::StrCat("get table info failed, db ", db, " name ", name)}; } uint32_t tid = table_info->tid(); - std::string leader_endpoint; - std::string follower_endpoint; std::map pid_endpoint_map; + std::map pid_offset_map; std::vector endpoints; for (const auto& part : table_info->table_partition()) { for (const auto& meta : part.partition_meta()) { + const std::string& ep = meta.endpoint(); if (!meta.is_alive()) { - continue; + return {-1, absl::StrCat(ep, " is not alive")}; } - const std::string& ep = meta.endpoint(); - if (meta.is_leader()) { - if (part.pid() == pid) { - leader_endpoint = ep; - } else { - pid_endpoint_map.insert(std::make_pair(part.pid(), ep)); - } + auto it = tablets_.find(ep); + if (it == tablets_.end() || !it->second->Health()) { + return {-1, absl::StrCat(ep, " is not online")}; } - if (part.pid() == pid) { - if (!meta.is_leader() && follower_endpoint.empty()) { - follower_endpoint = ep; - } - endpoints.push_back(ep); + if (meta.is_leader()) { + pid_endpoint_map.emplace(part.pid(), ep); + pid_offset_map.emplace(part.pid(), meta.offset()); } + endpoints.push_back(ep); } } - if (leader_endpoint.empty()) { - LOG(WARNING) << "get leader failed. table[" << name << "] pid[" << pid << "]"; - return -1; - } - auto it = tablets_.find(leader_endpoint); - if (it == tablets_.end() || !it->second->Health()) { - LOG(WARNING) << "leader[" << leader_endpoint << "] is not online"; - return -1; + if (static_cast(pid_endpoint_map.size()) != table_info->table_partition_size()) { + return {-1, "get leader failed"}; } - uint64_t op_index = op_data->op_info_.op_id(); - auto op_type = kAddIndexOP; - std::shared_ptr task; - const openmldb::common::ColumnKey& ck = add_index_meta.column_key(); - const auto ck_idx = add_index_meta.idx(); - if (add_index_meta.skip_data()) { - task = CreateTask(std::make_shared(op_index, op_type, tid, pid, endpoints, ck)); - if (!task) { - LOG(WARNING) << "create add index task failed. tid[" << tid << "] pid[" << pid << "]"; - return -1; - } - op_data->task_list_.push_back(task); - boost::function fun = boost::bind(&NameServerImpl::AddIndexToTableInfo, this, name, db, ck, ck_idx); - task = CreateTask(std::make_shared(op_index, op_type, tid, fun)); - if (!task) { - LOG(WARNING) << "create add index task failed. tid[" << tid << "] pid[" << pid << "]"; - } - op_data->task_list_.push_back(task); - return 0; - } - const int part_size = table_info->table_partition_size(); - task = CreateTask(std::make_shared( - op_index, op_type, leader_endpoint, tid, pid, part_size, ck, ck_idx)); + int part_size = table_info->table_partition_size(); + auto task = CreateTask(op_index, op_type, *table_info, column_key); if (!task) { - LOG(WARNING) << "create dump index task failed. tid[" << tid << "] pid[" << pid << "] endpoint[" - << leader_endpoint << "]"; - return -1; + return {-1, "create add index task failed"}; } - op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, leader_endpoint, tid, pid, pid_endpoint_map)); + task_list->push_back(task); + task = CreateTask(op_index, op_type, name, db, column_key); if (!task) { - LOG(WARNING) << "create send index data task failed. tid[" << tid << "] pid [" << pid << "] endpoint[" - << leader_endpoint << "]"; - return -1; + return {-1, "create add index to table info task failed"}; } - op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, tid, pid, endpoints, ck)); - if (!task) { - LOG(WARNING) << "create add index task failed. tid[" << tid << "] pid[" << pid << "]"; - return -1; - } - op_data->task_list_.push_back(task); - task = CreateTask( - std::make_shared(op_index, op_type, tid, pid, endpoints, part_size, ck, ck_idx)); + task_list->push_back(task); + task = CreateTask(op_index, op_type, tid, part_size, + column_key, pid_offset_map, pid_endpoint_map); if (!task) { - LOG(WARNING) << "Create extract index data task failed. tid[" << tid << "] pid[" << pid << "]"; - return -1; - } - op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, leader_endpoint, tid, pid, part_size)); - if (!task) { - LOG(WARNING) << "create load index data task failed. tid[" << tid << "] pid[" << pid << "] endpoint[" - << leader_endpoint << "]"; - return -1; + return {-1, "create extract index task failed"}; } - op_data->task_list_.push_back(task); - task = CreateTask(std::make_shared(op_index, op_type, - name, db, pid, follower_endpoint, FLAGS_check_binlog_sync_progress_delta)); + task_list->push_back(task); + task = CreateTask(op_index, op_type, tid, pid_endpoint_map); if (!task) { - LOG(WARNING) << "create CheckBinlogSyncProgressTask failed. name[" << name << "] pid[" << pid << "]"; - return -1; + return {-1, "create send index task failed"}; } - op_data->task_list_.push_back(task); - boost::function fun = boost::bind(&NameServerImpl::AddIndexToTableInfo, this, name, db, ck, ck_idx); - task = CreateTask(std::make_shared(op_index, op_type, tid, fun)); + task_list->push_back(task); + task = CreateTask(op_index, op_type, tid, part_size, pid_endpoint_map); if (!task) { - LOG(WARNING) << "creawte table sync task failed. name[" << name << "] pid[" << pid << "]"; - return -1; + return {-1, "create load index task failed"}; } - op_data->task_list_.push_back(task); - return 0; -} - -void NameServerImpl::RunSyncTaskFun(uint32_t tid, const boost::function& fun, - std::shared_ptr<::openmldb::api::TaskInfo> task_info) { - std::string value; - std::string table_sync_node = zk_path_.op_sync_path_ + "/" + std::to_string(tid); - do { - uint32_t task_num = 0; - { - std::lock_guard lock(mu_); - if (!zk_client_->GetNodeValue(table_sync_node, value)) { - PDLOG(WARNING, "get sync value failed. table %u node %s", tid, table_sync_node.c_str()); - break; - } - try { - task_num = boost::lexical_cast(value); - } catch (std::exception const& e) { - PDLOG(WARNING, "convert to uint failed. table %u value %s", tid, value.c_str()); - break; - } - task_num--; - if (task_num > 0) { - std::string new_value = std::to_string(task_num); - if (!zk_client_->SetNodeValue(table_sync_node, new_value)) { - PDLOG(WARNING, "set sync value failed. table %u node %s", tid, table_sync_node.c_str()); - break; - } - } - } - if (task_num == 0) { - if (!fun()) { - PDLOG(WARNING, "execute fun failed. table %u", tid); - break; - } - if (!zk_client_->DeleteNode(table_sync_node)) { - PDLOG(WARNING, "delete sync value failed. table %u node %s", tid, table_sync_node.c_str()); - } - PDLOG(INFO, "execute fun success. table %u", tid); - } - task_info->set_status(::openmldb::api::TaskStatus::kDone); - return; - } while (0); - task_info->set_status(::openmldb::api::TaskStatus::kFailed); + task_list->push_back(task); + return {}; } void NameServerImpl::RunSubTask(std::shared_ptr task) { for (const auto& cur_task : task->sub_task_) { - cur_task->task_info_->set_status(::openmldb::api::TaskStatus::kDoing); + PDLOG(INFO, "task starts running. op_id %lu task type %s %s", + cur_task->GetOpId(), cur_task->GetReadableType().c_str(), cur_task->GetAdditionalMsg().c_str()); + cur_task->SetStatus(::openmldb::api::TaskStatus::kDoing); cur_task->fun_(); } } +void NameServerImpl::RunSeqTask(std::shared_ptr task) { + if (task->seq_task_.empty()) { + PDLOG(INFO, "update task status from %s to kDone. op_id %lu task_type %s %s", + task->GetReadableStatus().c_str(), task->GetOpId(), + task->GetReadableType().c_str(), task->GetAdditionalMsg().c_str()); + task->SetStatus(::openmldb::api::TaskStatus::kDone); + return; + } + auto cur_task = task->seq_task_.front(); + auto task_status = cur_task->GetStatus(); + if (task_status == ::openmldb::api::TaskStatus::kInited) { + PDLOG(INFO, "seq task starts running. op_id %lu task type %s %s", + cur_task->GetOpId(), cur_task->GetReadableType().c_str(), cur_task->GetAdditionalMsg().c_str()); + cur_task->SetStatus(::openmldb::api::TaskStatus::kDoing); + cur_task->fun_(); + } else if (task_status == ::openmldb::api::TaskStatus::kFailed || + task_status == ::openmldb::api::TaskStatus::kCanceled) { + PDLOG(INFO, "update task status from %s to %s. op_id %lu task_type %s %s", + task->GetReadableStatus().c_str(), cur_task->GetReadableStatus().c_str(), + task->GetOpId(), task->GetReadableType().c_str(), task->GetAdditionalMsg().c_str()); + task->SetStatus(task_status); + return; + } else if (task_status == ::openmldb::api::TaskStatus::kDone) { + task->seq_task_.pop_front(); + } + task_thread_pool_.DelayTask(SEQ_TASK_CHECK_INTERVAL, boost::bind(&NameServerImpl::RunSeqTask, this, task)); +} + void NameServerImpl::CreateDatabase(RpcController* controller, const CreateDatabaseRequest* request, GeneralResponse* response, Closure* done) { brpc::ClosureGuard done_guard(done); @@ -9467,17 +9178,14 @@ void NameServerImpl::CreateProcedure(RpcController* controller, const api::Creat PDLOG(WARNING, "cur nameserver is not leader"); return; } - auto sp_info = std::make_shared(); - sp_info->CopyFrom(request->sp_info()); - const std::string& sp_db_name = sp_info->db_name(); - const std::string& sp_name = sp_info->sp_name(); - const std::string sp_data_path = zk_path_.db_sp_data_path_ + "/" + sp_db_name + "." + sp_name; + const std::string& sp_db_name = request->sp_info().db_name(); + const std::string& sp_name = request->sp_info().sp_name(); { std::lock_guard lock(mu_); if (databases_.find(sp_db_name) == databases_.end()) { response->set_code(::openmldb::base::ReturnCode::kDatabaseNotFound); response->set_msg("database not found"); - PDLOG(WARNING, "database[%s] not found", sp_db_name); + PDLOG(WARNING, "database[%s] not found", sp_db_name.c_str()); return; } else { const auto& sp_table_map = db_sp_table_map_[sp_db_name]; @@ -9490,11 +9198,18 @@ void NameServerImpl::CreateProcedure(RpcController* controller, const api::Creat } } } + auto status = CreateProcedureInternal(*request); + base::SetResponseStatus(status, response); +} + +base::Status NameServerImpl::CreateProcedureInternal(const api::CreateProcedureRequest& sp_request) { + auto sp_info = std::make_shared(sp_request.sp_info()); + const std::string& sp_db_name = sp_info->db_name(); + const std::string& sp_name = sp_info->sp_name(); + const std::string sp_data_path = absl::StrCat(zk_path_.db_sp_data_path_ , "/", sp_db_name, ".", sp_name); + auto status = CreateProcedureOnTablet(sp_request); do { - std::string err_msg; - if (!CreateProcedureOnTablet(*request, err_msg)) { - response->set_code(::openmldb::base::ReturnCode::kCreateProcedureFailedOnTablet); - response->set_msg(err_msg); + if (!status.OK()) { break; } if (IsClusterMode()) { @@ -9505,8 +9220,7 @@ void NameServerImpl::CreateProcedure(RpcController* controller, const api::Creat if (!zk_client_->CreateNode(sp_data_path, compressed)) { PDLOG(WARNING, "create db store procedure node[%s] failed! value[%s] value size[%lu]", sp_data_path.c_str(), sp_value.c_str(), compressed.length()); - response->set_code(::openmldb::base::ReturnCode::kCreateZkFailed); - response->set_msg("create zk node failed"); + status = {base::ReturnCode::kCreateZkFailed, "create zk node failed"}; break; } } @@ -9515,23 +9229,22 @@ void NameServerImpl::CreateProcedure(RpcController* controller, const api::Creat auto& sp_table_map = db_sp_table_map_[sp_db_name]; for (const auto& depend_table : sp_info->tables()) { auto& table_sp_map = db_table_sp_map_[depend_table.db_name()]; - sp_table_map[sp_name].push_back(std::make_pair(depend_table.db_name(), depend_table.table_name())); - table_sp_map[depend_table.table_name()].push_back(std::make_pair(sp_db_name, sp_name)); + sp_table_map[sp_name].emplace_back(depend_table.db_name(), depend_table.table_name()); + table_sp_map[depend_table.table_name()].emplace_back(sp_db_name, sp_name); } db_sp_info_map_[sp_db_name][sp_name] = sp_info; } NotifyTableChanged(::openmldb::type::NotifyType::kTable); PDLOG(INFO, "create db store procedure success! db_name [%s] sp_name [%s] sql [%s]", sp_db_name.c_str(), sp_name.c_str(), sp_info->sql().c_str()); - response->set_code(::openmldb::base::ReturnCode::kOk); - response->set_msg("ok"); - return; } while (0); - DropProcedureOnTablet(sp_db_name, sp_name); + if (!status.OK()) { + DropProcedureOnTablet(sp_db_name, sp_name); + } + return status; } -bool NameServerImpl::CreateProcedureOnTablet(const ::openmldb::api::CreateProcedureRequest& sp_request, - std::string& err_msg) { +base::Status NameServerImpl::CreateProcedureOnTablet(const ::openmldb::api::CreateProcedureRequest& sp_request) { std::vector> tb_client_vec; { std::lock_guard lock(mu_); @@ -9546,24 +9259,22 @@ bool NameServerImpl::CreateProcedureOnTablet(const ::openmldb::api::CreateProced DLOG(INFO) << "request timeout in ms: " << sp_request.timeout_ms(); const auto& sp_info = sp_request.sp_info(); for (auto tb_client : tb_client_vec) { - std::string msg; - if (!tb_client->CreateProcedure(sp_request, msg)) { + auto status = tb_client->CreateProcedure(sp_request); + if (!status.OK()) { + std::string err_msg; char temp_msg[100]; snprintf(temp_msg, sizeof(temp_msg), - "create procedure on tablet failed." - "db_name[%s], sp_name[%s], endpoint[%s]. ", + "create procedure on tablet failed. db_name[%s], sp_name[%s], endpoint[%s]. ", sp_info.db_name().c_str(), sp_info.sp_name().c_str(), tb_client->GetEndpoint().c_str()); - err_msg.append(temp_msg); - err_msg.append("msg: "); - err_msg.append(msg); + absl::StrAppend(&err_msg, temp_msg, "msg: ", status.GetMsg()); LOG(WARNING) << err_msg; - return false; + return {base::ReturnCode::kCreateProcedureFailedOnTablet, err_msg}; } DLOG(INFO) << "create procedure on tablet success. db_name: " << sp_info.db_name() << ", " << "sp_name: " << sp_info.sp_name() << ", " << "sql: " << sp_info.sql() << "endpoint: " << tb_client->GetEndpoint(); } - return true; + return {}; } void NameServerImpl::DropProcedureOnTablet(const std::string& db_name, const std::string& sp_name) { @@ -10349,7 +10060,7 @@ void NameServerImpl::FreeSdkConnection() { } } -std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr& task_meta) { +std::shared_ptr NameServerImpl::CreateTaskInternal(const TaskMeta* task_meta) { auto task_type = task_meta->task_info->task_type(); std::shared_ptr client; std::string endpoint = task_meta->task_info->endpoint(); @@ -10364,7 +10075,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr auto task = std::make_shared(endpoint, task_info); switch (task_type) { case ::openmldb::api::TaskType::kMakeSnapshot: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::MakeSnapshot, client, meta->tid, meta->pid, meta->end_offset, task_info); @@ -10372,21 +10083,21 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kPauseSnapshot: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::PauseSnapshot, client, meta->tid, meta->pid, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kRecoverSnapshot: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::RecoverSnapshot, client, meta->tid, meta->pid, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kSendSnapshot: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::SendSnapshot, client, meta->tid, meta->remote_tid, meta->pid, meta->des_endpoint, task_info); @@ -10394,7 +10105,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kLoadTable: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); ::openmldb::api::TableMeta table_meta; table_meta.set_name(meta->name); table_meta.set_tid(meta->tid); @@ -10412,7 +10123,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kAddReplica: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun; if (meta->is_remote) { if (meta->task_id != INVALID_PARENT_ID) { @@ -10428,26 +10139,21 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kDelReplica: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::DelReplica, client, meta->tid, meta->pid, meta->des_endpoint, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kDropTable: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); boost::function fun = boost::bind(&TabletClient::DropTable, client, meta->tid, meta->pid, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } - case ::openmldb::api::TaskType::kTableSyncTask: { - auto meta = std::dynamic_pointer_cast(task_meta); - task->fun_ = boost::bind(&NameServerImpl::RunSyncTaskFun, this, meta->tid, meta->fun, task_info); - break; - } case ::openmldb::api::TaskType::kAddTableInfo: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); if (meta->is_remote) { task->fun_ = boost::bind(&NameServerImpl::AddTableInfo, this, meta->alias, meta->endpoint, meta->name, meta->db, @@ -10459,7 +10165,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kDelTableInfo: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); if (meta->has_flag) { task->fun_ = boost::bind(&NameServerImpl::DelTableInfo, this, meta->name, meta->db, meta->endpoint, meta->pid, task_info, meta->flag); @@ -10470,83 +10176,108 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kUpdateTableInfo: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); task->fun_ = boost::bind(&NameServerImpl::UpdateTableInfo, this, meta->src_endpoint, meta->name, meta->db, meta->pid, meta->des_endpoint, task_info); break; } - case ::openmldb::api::TaskType::kDumpIndexData: { - auto meta = std::dynamic_pointer_cast(task_meta); + case ::openmldb::api::TaskType::kSendIndexRequest: { + auto meta = dynamic_cast(task_meta); boost::function fun = - boost::bind(&TabletClient::DumpIndexData, client, meta->tid, meta->pid, - meta->partition_num, meta->column_key, meta->idx, task_info); + boost::bind(&TabletClient::SendIndexData, client, meta->tid, meta->pid, + meta->pid_endpoint_map, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kSendIndexData: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); + for (const auto& kv : meta->pid_endpoint_map) { + auto sub_task = CreateTask( + meta->task_info->op_id(), meta->task_info->op_type(), kv.second, + meta->tid, kv.first, meta->pid_endpoint_map); + task->sub_task_.push_back(sub_task); + PDLOG(INFO, "add subtask kSendIndexData. op_id[%lu] tid[%u] pid[%u] endpoint[%s]", + meta->task_info->op_id(), meta->tid, kv.first, kv.second.c_str()); + } + task->fun_ = boost::bind(&NameServerImpl::RunSubTask, this, task); + break; + } + case ::openmldb::api::TaskType::kLoadIndexRequest: { + auto meta = dynamic_cast(task_meta); boost::function fun = - boost::bind(&TabletClient::SendIndexData, client, meta->tid, meta->pid, - meta->pid_endpoint_map, task_info); + boost::bind(&TabletClient::LoadIndexData, client, meta->tid, meta->pid, + meta->partition_num, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kLoadIndexData: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); + for (const auto& kv : meta->pid_endpoint_map) { + auto sub_task = CreateTask( + meta->task_info->op_id(), meta->task_info->op_type(), kv.second, + meta->tid, kv.first, meta->pid_endpoint_map.size()); + task->sub_task_.push_back(sub_task); + PDLOG(INFO, "add subtask kLoadIndexData. op_id[%lu] tid[%u] pid[%u] endpoint[%s]", + meta->task_info->op_id(), meta->tid, kv.first, kv.second.c_str()); + } + task->fun_ = boost::bind(&NameServerImpl::RunSubTask, this, task); + break; + } + case ::openmldb::api::TaskType::kExtractIndexRequest: { + auto meta = dynamic_cast(task_meta); boost::function fun = - boost::bind(&TabletClient::LoadIndexData, client, meta->tid, meta->pid, - meta->partition_num, task_info); + boost::bind(&TabletClient::ExtractIndexData, client, meta->tid, meta->pid, + meta->partition_num, meta->column_key, meta->offset, true, task_info); task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } case ::openmldb::api::TaskType::kExtractIndexData: { - auto meta = std::dynamic_pointer_cast(task_meta); - for (const auto& endpoint : meta->endpoints) { - auto tablet = GetHealthTabletInfoNoLock(endpoint); - if (!tablet) { - return {}; - } - auto sub_task = std::make_shared(endpoint, std::make_shared<::openmldb::api::TaskInfo>()); - sub_task->task_info_->set_op_id(meta->task_info->op_id()); - sub_task->task_info_->set_op_type(meta->task_info->op_type()); - sub_task->task_info_->set_task_type(::openmldb::api::TaskType::kExtractIndexData); - sub_task->task_info_->set_status(::openmldb::api::TaskStatus::kInited); - sub_task->task_info_->set_endpoint(endpoint); - boost::function fun = boost::bind(&TabletClient::ExtractIndexData, tablet->client_, - meta->tid, meta->pid, meta->partition_num, meta->column_key, meta->idx, sub_task->task_info_); - sub_task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, sub_task->task_info_); + auto meta = dynamic_cast(task_meta); + for (const auto& kv : meta->pid_endpoint_map) { + auto iter = meta->pid_offset_map.find(kv.first); + auto sub_task = CreateTask( + meta->task_info->op_id(), meta->task_info->op_type(), kv.second, + meta->tid, kv.first, meta->partition_num, meta->column_key, + iter->second); task->sub_task_.push_back(sub_task); PDLOG(INFO, "add subtask kExtractIndexData. op_id[%lu] tid[%u] pid[%u] endpoint[%s]", - meta->task_info->op_id(), meta->tid, meta->pid, endpoint.c_str()); + meta->task_info->op_id(), meta->tid, kv.first, kv.second.c_str()); } task->fun_ = boost::bind(&NameServerImpl::RunSubTask, this, task); break; } + case ::openmldb::api::TaskType::kAddIndexToTabletRequest: { + auto meta = dynamic_cast(task_meta); + boost::function fun = + boost::bind(&TabletClient::AddMultiIndex, client, meta->tid, meta->pid, + meta->column_key, task_info); + task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); + break; + } case ::openmldb::api::TaskType::kAddIndexToTablet: { - auto meta = std::dynamic_pointer_cast(task_meta); - for (const auto& endpoint : meta->endpoints) { - auto tablet = GetHealthTabletInfoNoLock(endpoint); - if (!tablet) { - return {}; + auto meta = dynamic_cast(task_meta); + for (const auto& part : meta->table_info.table_partition()) { + for (const auto& part_meta : part.partition_meta()) { + const std::string& ep = part_meta.endpoint(); + auto sub_task = CreateTask( + meta->task_info->op_id(), meta->task_info->op_type(), ep, + meta->table_info.tid(), part.pid(), meta->column_key); + task->sub_task_.push_back(sub_task); + PDLOG(INFO, "add subtask AddIndexToTablet. op_id[%lu] tid[%u] pid[%u] endpoint[%s]", + meta->task_info->op_id(), meta->table_info.tid(), part.pid(), ep.c_str()); } - auto sub_task = std::make_shared(endpoint, std::make_shared<::openmldb::api::TaskInfo>()); - sub_task->task_info_->set_op_id(meta->task_info->op_id()); - sub_task->task_info_->set_op_type(meta->task_info->op_type()); - sub_task->task_info_->set_task_type(::openmldb::api::TaskType::kAddIndexToTablet); - sub_task->task_info_->set_status(::openmldb::api::TaskStatus::kInited); - sub_task->task_info_->set_endpoint(endpoint); - boost::function fun = boost::bind(&TabletClient::AddIndex, tablet->client_, - meta->tid, meta->pid, meta->column_key, sub_task->task_info_); - sub_task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, sub_task->task_info_); - task->sub_task_.push_back(sub_task); - PDLOG(INFO, "add subtask AddIndexToTablet. op_id[%lu] tid[%u] pid[%u] endpoint[%s]", - meta->task_info->op_id(), meta->tid, meta->pid, endpoint.c_str()); } task->fun_ = boost::bind(&NameServerImpl::RunSubTask, this, task); break; } + case ::openmldb::api::TaskType::kAddIndexToTableInfo: { + auto meta = dynamic_cast(task_meta); + task->fun_ = boost::bind(&NameServerImpl::AddIndexToTableInfo, this, + meta->name, meta->db, meta->column_key, task_info); + break; + } case ::openmldb::api::TaskType::kCheckBinlogSyncProgress: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); task->fun_ = boost::bind(&NameServerImpl::CheckBinlogSyncProgress, this, meta->name, meta->db, meta->pid, meta->follower, meta->offset_delta, task_info); break; @@ -10556,7 +10287,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kSelectLeader: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); task->fun_ = boost::bind(&NameServerImpl::SelectLeader, this, meta->name, meta->db, meta->tid, meta->pid, meta->follower_endpoint, task_info); break; @@ -10566,19 +10297,19 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kRecoverTable: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); task->fun_ = boost::bind(&NameServerImpl::RecoverEndpointTable, this, meta->name, meta->db, meta->pid, meta->endpoint, meta->offset_delta, meta->concurrency, task_info); break; } case ::openmldb::api::TaskType::kUpdatePartitionStatus: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); task->fun_ = boost::bind(&NameServerImpl::UpdatePartitionStatus, this, meta->name, meta->db, meta->endpoint, meta->pid, meta->is_leader, meta->is_alive, task_info); break; } case ::openmldb::api::TaskType::kCreateTableRemote: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); auto cluster = GetHealthCluster(meta->alias); if (!cluster) { PDLOG(WARNING, "replica[%s] not available op_index[%lu]", @@ -10594,7 +10325,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kDropTableRemote: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); auto cluster = GetHealthCluster(meta->alias); if (!cluster) { PDLOG(WARNING, "replica[%s] not available op_index[%lu]", @@ -10610,7 +10341,7 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr break; } case ::openmldb::api::TaskType::kAddReplicaNSRemote: { - auto meta = std::dynamic_pointer_cast(task_meta); + auto meta = dynamic_cast(task_meta); auto cluster = GetHealthCluster(meta->alias); if (!cluster) { PDLOG(WARNING, "replica[%s] not available op_index[%lu]", @@ -10626,11 +10357,145 @@ std::shared_ptr NameServerImpl::CreateTask(const std::shared_ptr task->fun_ = boost::bind(&NameServerImpl::WrapTaskFun, this, fun, task_info); break; } + case ::openmldb::api::TaskType::kAddTableIndex: { + auto meta = dynamic_cast(task_meta); + auto status = FillAddIndexTask(meta->task_info->op_id(), meta->task_info->op_type(), + meta->name, meta->db, meta->column_key, &task->seq_task_); + if (!status.OK()) { + PDLOG(WARNING, "FillAddIndexTask failed. op_id %lu msg %s", + meta->task_info->op_id(), status.GetMsg().c_str()); + return {}; + } + task->fun_ = boost::bind(&NameServerImpl::RunSeqTask, this, task); + break; + } + case ::openmldb::api::TaskType::kAddMultiTableIndex: { + auto meta = dynamic_cast(task_meta); + for (const auto& cur_table_index : meta->table_index) { + auto sub_task = CreateTask( + meta->task_info->op_id(), meta->task_info->op_type(), + cur_table_index.name(), cur_table_index.db(), + schema::IndexUtil::Convert2Vector(cur_table_index.column_key())); + if (!sub_task) { + return {}; + } + task->sub_task_.push_back(sub_task); + PDLOG(INFO, "add subtask kAddTableIndex. op_id[%lu] table name %s db %s", + meta->task_info->op_id(), cur_table_index.name().c_str(), cur_table_index.db().c_str()); + } + task->fun_ = boost::bind(&NameServerImpl::RunSubTask, this, task); + break; + } + case ::openmldb::api::TaskType::kCreateProcedure: { + auto meta = dynamic_cast(task_meta); + api::CreateProcedureRequest request; + request.mutable_sp_info()->CopyFrom(meta->sp_info); + boost::function wrap_fun = boost::bind(&NameServerImpl::CreateProcedureInternal, + this, request); + task->fun_ = boost::bind(&NameServerImpl::WrapNormalTaskFun, this, wrap_fun, task_info); + break; + } + case ::openmldb::api::TaskType::kDumpIndexData: // deprecated case ::openmldb::api::TaskType::kUpdateTableAlive: // deprecated + case ::openmldb::api::TaskType::kTableSyncTask: // deprecated break; } return task; } +std::shared_ptr NameServerImpl::GetProcedure(const std::string& db, const std::string& name) { + std::lock_guard lock(mu_); + auto iter = db_sp_info_map_.find(db); + if (iter != db_sp_info_map_.end()) { + auto sp_iter = iter->second.find(name); + if (sp_iter != iter->second.end()) { + return sp_iter->second; + } + } + return {}; +} + +bool NameServerImpl::IsExistDataBase(const std::string& db) { + std::lock_guard lock(mu_); + return databases_.find(db) != databases_.end(); +} + +void NameServerImpl::DeploySQL(RpcController* controller, const DeploySQLRequest* request, + DeploySQLResponse* response, Closure* done) { + brpc::ClosureGuard done_guard(done); + if (!running_.load(std::memory_order_acquire)) { + response->set_code(::openmldb::base::ReturnCode::kNameserverIsNotLeader); + response->set_msg("nameserver is not leader"); + PDLOG(WARNING, "cur nameserver is not leader"); + return; + } + const auto& sp_info = request->sp_info(); + const auto& db = sp_info.db_name(); + const auto& deploy_name = sp_info.sp_name(); + if (!IsExistDataBase(db)) { + base::SetResponseStatus(ReturnCode::kDatabaseNotFound, "database not found", response); + PDLOG(WARNING, "database[%s] not found", db.c_str()); + return; + } + if (auto procedure = GetProcedure(db, deploy_name); + procedure && procedure->type() == ::openmldb::type::ProcedureType::kReqDeployment) { + base::SetResponseStatus(ReturnCode::kProcedureAlreadyExists, "deployment already exists", response); + PDLOG(WARNING, "deployment[%s] already exists in db[%s]", deploy_name.c_str(), db.c_str()); + return; + } + for (const auto& index : request->index()) { + std::shared_ptr table_info; + std::string cur_db = index.has_db() && !index.db().empty() ? index.db() : db; + const auto& table_name = index.name(); + if (!GetTableInfo(table_name, cur_db, &table_info)) { + base::SetResponseStatus(ReturnCode::kTableIsNotExist, "table does not exist!", response); + PDLOG(WARNING, "table %s.%s does not exit", cur_db.c_str(), table_name.c_str()); + return; + } + for (const auto& column_key : index.column_key()) { + if (schema::IndexUtil::IsExist(column_key, table_info->column_key())) { + base::SetResponseStatus(ReturnCode::kIndexAlreadyExists, "index already exist!", response); + PDLOG(WARNING, "index already exist in table %s", table_name.c_str()); + return; + } + } + } + uint64_t op_id = 0; + std::lock_guard lock(mu_); + auto status = CreateDeployOP(*request, &op_id); + if (!status.OK()) { + PDLOG(WARNING, "%s", status.GetMsg().c_str()); + } + response->set_op_id(op_id); + SetResponseStatus(status, response); +} + +base::Status NameServerImpl::CreateDeployOP(const DeploySQLRequest& request, uint64_t* op_id) { + std::shared_ptr op_data; + const auto& sp_info = request.sp_info(); + const auto& deploy_name = sp_info.sp_name(); + std::string value; + auto op_type = api::OPType::kDeployOP; + if (CreateOPData(op_type, value, op_data, sp_info.main_table(), sp_info.db_name(), 0) < 0) { + return {-1, absl::StrCat("create AddIndexOP data error. deploy name ", deploy_name)}; + } + auto task = CreateTask(op_data->GetOpId(), op_type, request.index()); + if (!task) { + return {-1, absl::StrCat("Create kAddMultiTableIndex task failed. deploy name ", deploy_name)}; + } + op_data->task_list_.push_back(task); + task = CreateTask(op_data->GetOpId(), op_type, sp_info); + if (!task) { + return {-1, absl::StrCat("Create CreateProcedureTaskMeta task failed. deploy name ", deploy_name)}; + } + op_data->task_list_.push_back(task); + if (AddOPData(op_data) < 0) { + return {-1, absl::StrCat("add op data failed. deploy name ", sp_info.sp_name())}; + } + PDLOG(INFO, "create DeployOP success. op id %lu deploy name %s", op_data->GetOpId(), deploy_name.c_str()); + *op_id = op_data->GetOpId(); + return {}; +} + } // namespace nameserver } // namespace openmldb diff --git a/src/nameserver/name_server_impl.h b/src/nameserver/name_server_impl.h index 86f921ba42b..e5f41923647 100644 --- a/src/nameserver/name_server_impl.h +++ b/src/nameserver/name_server_impl.h @@ -98,7 +98,6 @@ struct ZkPath { std::string zone_data_path_; std::string op_index_node_; std::string op_data_path_; - std::string op_sync_path_; std::string globalvar_changed_notify_node_; std::string external_function_path_; }; @@ -151,6 +150,9 @@ class NameServerImpl : public NameServer { void CreateProcedure(RpcController* controller, const api::CreateProcedureRequest* request, GeneralResponse* response, Closure* done); + void DeploySQL(RpcController* controller, const DeploySQLRequest* request, + DeploySQLResponse* response, Closure* done); + void DropTableInternel(const DropTableRequest& request, GeneralResponse& response, // NOLINT std::shared_ptr<::openmldb::nameserver::TableInfo> table_info, std::shared_ptr<::openmldb::api::TaskInfo> task_ptr); @@ -312,8 +314,7 @@ class NameServerImpl : public NameServer { int UpdateTaskStatusRemote(bool is_recover_op); int UpdateTask(const std::list>& op_list, const std::string& endpoint, - const std::string& msg, bool is_recover_op, - ::openmldb::api::TaskStatusResponse& response); // NOLINT + bool is_recover_op, const ::openmldb::api::TaskStatusResponse& response); int UpdateTaskStatus(bool is_recover_op); @@ -341,7 +342,8 @@ class NameServerImpl : public NameServer { bool RegisterName(); - bool CreateProcedureOnTablet(const api::CreateProcedureRequest& sp_request, std::string& err_msg); // NOLINT + base::Status CreateProcedureOnTablet(const api::CreateProcedureRequest& sp_request); + base::Status CreateProcedureInternal(const api::CreateProcedureRequest& sp_request); void DropProcedure(RpcController* controller, const api::DropProcedureRequest* request, GeneralResponse* response, Closure* done); @@ -450,7 +452,13 @@ class NameServerImpl : public NameServer { int UpdateEndpointTableAlive(const std::string& endpoint, bool is_alive); - std::shared_ptr CreateTask(const std::shared_ptr& task_meta); + template + std::shared_ptr CreateTask(Arg &&...arg) { + T meta(std::forward(arg)...); + return CreateTaskInternal(&meta); + } + + std::shared_ptr CreateTaskInternal(const TaskMeta* task_meta); std::shared_ptr CreateLoadTableRemoteTask(const std::string& alias, const std::string& name, const std::string& db, const std::string& endpoint, uint32_t pid, @@ -471,10 +479,10 @@ class NameServerImpl : public NameServer { const ::openmldb::common::ColumnKey& column_key); bool GetTableInfo(const std::string& table_name, const std::string& db_name, - std::shared_ptr* table_info); + std::shared_ptr* table_info); bool GetTableInfoUnlock(const std::string& table_name, const std::string& db_name, - std::shared_ptr* table_info); + std::shared_ptr* table_info); int AddOPTask(const ::openmldb::api::TaskInfo& task_info, ::openmldb::api::TaskType task_type, std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, // NOLINT @@ -539,11 +547,17 @@ class NameServerImpl : public NameServer { uint64_t parent_id = INVALID_PARENT_ID, uint32_t concurrency = FLAGS_name_server_task_concurrency_for_replica_cluster); - int CreateAddIndexOP(const std::string& name, const std::string& db, uint32_t pid, - const std::vector& new_cols, - const ::openmldb::common::ColumnKey& column_key, uint32_t idx); + base::Status CreateDeployOP(const DeploySQLRequest& request, uint64_t* op_id); + + base::Status CreateAddIndexOP(const std::string& name, const std::string& db, + const std::vector<::openmldb::common::ColumnKey>& column_key); - int CreateAddIndexOPTask(std::shared_ptr op_data); + base::Status CreateAddIndexOPTask(std::shared_ptr op_data); + + base::Status FillAddIndexTask(uint64_t op_index, api::OPType op_type, + const std::string& name, const std::string& db, + const std::vector<::openmldb::common::ColumnKey>& column_key, + std::list>* task_list); int DropTableRemoteOP(const std::string& name, const std::string& db, const std::string& alias, uint64_t parent_id = INVALID_PARENT_ID, @@ -559,14 +573,16 @@ class NameServerImpl : public NameServer { std::shared_ptr<::openmldb::api::TaskInfo> task_info); bool AddIndexToTableInfo(const std::string& name, const std::string& db, - const ::openmldb::common::ColumnKey& column_key, uint32_t index_pos); + const std::vector<::openmldb::common::ColumnKey>& column_key, + std::shared_ptr<::openmldb::api::TaskInfo> task_info); void WrapTaskFun(const boost::function& fun, std::shared_ptr<::openmldb::api::TaskInfo> task_info); - void RunSyncTaskFun(uint32_t tid, const boost::function& fun, - std::shared_ptr<::openmldb::api::TaskInfo> task_info); + void WrapNormalTaskFun(const boost::function& fun, + std::shared_ptr<::openmldb::api::TaskInfo> task_info); void RunSubTask(std::shared_ptr task); + void RunSeqTask(std::shared_ptr task); // get tablet info std::shared_ptr GetTabletInfo(const std::string& endpoint); @@ -661,6 +677,10 @@ class NameServerImpl : public NameServer { ::openmldb::base::Status CheckZoneInfo(const ::openmldb::nameserver::ZoneInfo& zone_info); + std::shared_ptr GetProcedure(const std::string& db, const std::string& name); + + bool IsExistDataBase(const std::string& db); + private: std::mutex mu_; Tablets tablets_; diff --git a/src/nameserver/name_server_test.cc b/src/nameserver/name_server_test.cc index 244b1115f63..f1ad0f86eab 100644 --- a/src/nameserver/name_server_test.cc +++ b/src/nameserver/name_server_test.cc @@ -44,6 +44,7 @@ DECLARE_int32(zk_keep_alive_check_interval); DECLARE_int32(make_snapshot_threshold_offset); DECLARE_uint32(name_server_task_max_concurrency); DECLARE_uint32(system_table_replica_num); +DECLARE_uint32(sync_deploy_stats_timeout); DECLARE_bool(auto_failover); using brpc::Server; @@ -774,7 +775,7 @@ void InitTablet(int port, vector services, vector tb = std::make_shared(); if (!tb->Init("")) { @@ -1293,5 +1294,6 @@ int main(int argc, char** argv) { FLAGS_ssd_root_path = tmp_path.GetTempPath("ssd"); FLAGS_hdd_root_path = tmp_path.GetTempPath("hdd"); FLAGS_system_table_replica_num = 0; + FLAGS_sync_deploy_stats_timeout = 1000000; return RUN_ALL_TESTS(); } diff --git a/src/nameserver/task.cc b/src/nameserver/task.cc new file mode 100644 index 00000000000..4f292e91b40 --- /dev/null +++ b/src/nameserver/task.cc @@ -0,0 +1,172 @@ +/* + * Copyright 2022 4Paradigm + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nameserver/task.h" + +#include "absl/strings/str_cat.h" +#include "base/glog_wrapper.h" + +namespace openmldb { +namespace nameserver { + +bool Task::IsFinished() const { + return task_info_->status() == ::openmldb::api::kDone || + task_info_->status() == ::openmldb::api::kFailed || task_info_->status() == ::openmldb::api::kCanceled; +} + +void Task::UpdateTaskStatus(const ::google::protobuf::RepeatedPtrField<::openmldb::api::TaskInfo>& tasks, + const std::string& endpoint, bool is_recover) { + if (IsFinished()) { + return; + } + for (const auto& cur_task : tasks) { + UpdateStatus(cur_task, endpoint); + } + SetState(endpoint, is_recover); + check_num_++; +} + +bool Task::UpdateStatus(const ::openmldb::api::TaskInfo& task, const std::string& endpoint) { + if (GetOpId() != task.op_id()) { + return false; + } + if (IsFinished()) { + return false; + } + if (!seq_task_.empty()) { + seq_task_.front()->UpdateStatus(task, endpoint); + } else if (!sub_task_.empty()) { + for (auto& cur_task : sub_task_) { + cur_task->UpdateStatus(task, endpoint); + } + } else { + if (task_info_->task_type() != task.task_type()) { + return false; + } + if (task_info_->has_endpoint() && task_info_->endpoint() != endpoint) { + return false; + } + if (task_info_->has_tid() && (!task.has_tid() || task.tid() != task_info_->tid())) { + return false; + } + if (task_info_->has_pid() && (!task.has_pid() || task.pid() != task_info_->pid())) { + return false; + } + if (task.status() != ::openmldb::api::kInited) { + if (task_info_->status() != task.status()) { + PDLOG(INFO, "update task status from %s to %s. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetReadableStatus(task).c_str(), + task.op_id(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + task_info_->set_status(task.status()); + } + } + traversed_ = true; + return true; + } + return false; +} + +void Task::SetState(const std::string& endpoint, bool is_recover) { + if (IsFinished()) { + return; + } + if (!seq_task_.empty()) { + seq_task_.front()->SetState(endpoint, is_recover); + return; + } + if (sub_task_.empty()) { + if (endpoint_ == endpoint && !traversed_ && (is_recover || (task_info_->is_rpc_send() && check_num_ > 5))) { + // if the task does not exist in tablet, set to failed + task_info_->set_status(::openmldb::api::kFailed); + } + traversed_ = false; + return; + } + uint32_t done_cnt = 0; + for (auto& cur_task : sub_task_) { + cur_task->SetState(endpoint, is_recover); + if (cur_task->GetStatus() == ::openmldb::api::kDone) { + done_cnt++; + } else if (cur_task->GetStatus() == ::openmldb::api::kFailed) { + PDLOG(INFO, "update task status from %s to kFailed. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kFailed); + break; + } else if (cur_task->GetStatus() == ::openmldb::api::kCanceled) { + PDLOG(INFO, "update task status from %s to kCanceled. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kCanceled); + break; + } + } + if (done_cnt == sub_task_.size()) { + PDLOG(INFO, "update task status from %s to kDone. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kDone); + } +} + +void Task::UpdateStatusFromSubTask() { + if (IsFinished()) { + return; + } + if (!seq_task_.empty()) { + return seq_task_.front()->UpdateStatusFromSubTask(); + } + if (sub_task_.empty()) { + return; + } + uint32_t done_cnt = 0; + for (auto& cur_task : sub_task_) { + cur_task->UpdateStatusFromSubTask(); + if (cur_task->GetStatus() == ::openmldb::api::kDone) { + done_cnt++; + } else if (cur_task->GetStatus() == ::openmldb::api::kFailed) { + PDLOG(INFO, "update task status from %s to kFailed. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kFailed); + return; + } else if (cur_task->GetStatus() == ::openmldb::api::kCanceled) { + PDLOG(INFO, "update task status from %s to kCanceled. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kCanceled); + return; + } + } + if (done_cnt == sub_task_.size()) { + PDLOG(INFO, "update task status from %s to kDone. op_id %lu task_type %s %s", + GetReadableStatus().c_str(), GetOpId(), GetReadableType().c_str(), GetAdditionalMsg().c_str()); + SetStatus(::openmldb::api::kDone); + } +} + +std::string Task::GetAdditionalMsg(const ::openmldb::api::TaskInfo& task_info) { + std::string additional_msg; + if (task_info.has_tid()) { + absl::StrAppend(&additional_msg, "tid ", task_info.tid()); + } + if (task_info.has_pid()) { + absl::StrAppend(&additional_msg, " pid ", task_info.pid()); + } + return additional_msg; +} + +std::string Task::GetAdditionalMsg() { + return GetAdditionalMsg(*task_info_); +} + +} // namespace nameserver +} // namespace openmldb diff --git a/src/nameserver/task.h b/src/nameserver/task.h index 3ef60bd0ffc..28cfa523f12 100644 --- a/src/nameserver/task.h +++ b/src/nameserver/task.h @@ -24,7 +24,7 @@ #include #include -#include "boost/bind.hpp" +#include "boost/function.hpp" #include "proto/common.pb.h" #include "proto/name_server.pb.h" #include "proto/tablet.pb.h" @@ -35,17 +35,50 @@ namespace nameserver { using TaskFun = boost::function; constexpr uint64_t INVALID_PARENT_ID = UINT64_MAX; -struct Task { +class Task { + public: Task(std::string endpoint, std::shared_ptr<::openmldb::api::TaskInfo> task_info) : endpoint_(std::move(endpoint)), task_info_(std::move(task_info)) {} - ~Task() = default; + bool IsFinished() const; + void UpdateTaskStatus(const ::google::protobuf::RepeatedPtrField<::openmldb::api::TaskInfo>& tasks, + const std::string& endpoint, bool is_recover); + void UpdateStatusFromSubTask(); + std::string GetReadableType() const { return ::openmldb::api::TaskType_Name(task_info_->task_type()); } + api::TaskType GetType() const { return task_info_->task_type(); } + void SetStatus(api::TaskStatus status) { task_info_->set_status(status); } + api::TaskStatus GetStatus() const { return task_info_->status(); } + std::string GetReadableStatus() const { return ::openmldb::api::TaskStatus_Name(task_info_->status()); } + static std::string GetReadableStatus(const ::openmldb::api::TaskInfo& task_info) { + return ::openmldb::api::TaskStatus_Name(task_info.status()); + } + uint64_t GetOpId() const { return task_info_->op_id(); } + std::string GetReadableOpType() const { return ::openmldb::api::OPType_Name(task_info_->op_type()); } + std::string GetAdditionalMsg(); // for log info + static std::string GetAdditionalMsg(const ::openmldb::api::TaskInfo& task_info); + std::string endpoint_; std::shared_ptr<::openmldb::api::TaskInfo> task_info_; - std::vector> sub_task_; + std::list> seq_task_; // execute one by one + std::list> sub_task_; // execute parallel TaskFun fun_; + + private: + bool UpdateStatus(const ::openmldb::api::TaskInfo& task_info, const std::string& endpoint); + void SetState(const std::string& endpoint, bool is_recover); + + private: + int check_num_ = 0; + bool traversed_ = false; }; -struct OPData { +class OPData { + public: + uint64_t GetOpId() const { return op_info_.op_id(); } + api::OPType GetType() const { return op_info_.op_type(); } + std::string GetReadableType() const { return ::openmldb::api::OPType_Name(op_info_.op_type()); } + void SetTaskStatus(api::TaskStatus status) { op_info_.set_task_status(status); } + api::TaskStatus GetTaskStatus() const { return op_info_.task_status(); } + ::openmldb::api::OPInfo op_info_; std::list> task_list_; }; @@ -167,77 +200,134 @@ class DropTableTaskMeta : public TaskMeta { uint32_t pid; }; -class TableSyncTaskMeta : public TaskMeta { +class SendIndexRequestTaskMeta : public TaskMeta { public: - TableSyncTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, uint32_t tid_i, boost::function fun_i) : - TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kTableSyncTask, ""), tid(tid_i), fun(fun_i) {} + SendIndexRequestTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, + uint32_t tid_i, uint32_t pid_i, const std::map& pid_endpoint_map_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kSendIndexRequest, endpoint), + tid(tid_i), pid(pid_i), pid_endpoint_map(pid_endpoint_map_i) { + task_info->set_tid(tid); + task_info->set_pid(pid); + } uint32_t tid; - boost::function fun; + uint32_t pid; + std::map pid_endpoint_map; }; -class DumpIndexDataTaskMeta : public TaskMeta { +class SendIndexDataTaskMeta : public TaskMeta { public: - DumpIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, - uint32_t tid_i, uint32_t pid_i, uint32_t partition_num_i, - const ::openmldb::common::ColumnKey& column_key_i, uint32_t idx_i) : - TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kDumpIndexData, endpoint), - tid(tid_i), pid(pid_i), partition_num(partition_num_i), column_key(column_key_i), idx(idx_i) {} + SendIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + uint32_t tid_i, const std::map& pid_endpoint_map_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kSendIndexData, ""), + tid(tid_i), pid_endpoint_map(pid_endpoint_map_i) { + task_info->set_tid(tid); + } + uint32_t tid; + std::map pid_endpoint_map; +}; + +class LoadIndexRequestTaskMeta : public TaskMeta { + public: + LoadIndexRequestTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, + uint32_t tid_i, uint32_t pid_i, uint32_t partition_num_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kLoadIndexRequest, endpoint), + tid(tid_i), pid(pid_i), partition_num(partition_num_i) { + task_info->set_tid(tid); + task_info->set_pid(pid); + } uint32_t tid; uint32_t pid; uint32_t partition_num; - ::openmldb::common::ColumnKey column_key; - uint32_t idx; }; -class SendIndexDataTaskMeta : public TaskMeta { +class LoadIndexDataTaskMeta : public TaskMeta { public: - SendIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, - uint32_t tid_i, uint32_t pid_i, const std::map& pid_endpoint_map_i) : - TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kSendIndexData, endpoint), - tid(tid_i), pid(pid_i), pid_endpoint_map(pid_endpoint_map_i) {} + LoadIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + uint32_t tid_i, uint32_t partition_num_i, + const std::map& pid_endpoint_map_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kLoadIndexData, ""), + tid(tid_i), pid_endpoint_map(pid_endpoint_map_i) { + task_info->set_tid(tid); + } uint32_t tid; - uint32_t pid; std::map pid_endpoint_map; }; -class LoadIndexDataTaskMeta : public TaskMeta { +class ExtractIndexRequestTaskMeta : public TaskMeta { public: - LoadIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, - uint32_t tid_i, uint32_t pid_i, uint32_t partition_num_i) : - TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kLoadIndexData, endpoint), - tid(tid_i), pid(pid_i), partition_num(partition_num_i) {} + ExtractIndexRequestTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, + uint32_t tid_i, uint32_t pid_i, uint32_t partition_num_i, + const std::vector<::openmldb::common::ColumnKey>& column_key_i, + uint64_t offset_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kExtractIndexRequest, endpoint), + tid(tid_i), pid(pid_i), partition_num(partition_num_i), column_key(column_key_i), offset(offset_i) { + task_info->set_tid(tid); + task_info->set_pid(pid); + } uint32_t tid; uint32_t pid; uint32_t partition_num; + std::vector<::openmldb::common::ColumnKey> column_key; + uint64_t offset; }; class ExtractIndexDataTaskMeta : public TaskMeta { public: ExtractIndexDataTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, - uint32_t tid_i, uint32_t pid_i, const std::vector& endpoints_i, - uint32_t partition_num_i, const ::openmldb::common::ColumnKey& column_key_i, uint32_t idx_i) : + uint32_t tid_i, uint32_t partition_num_i, + const std::vector<::openmldb::common::ColumnKey>& column_key_i, + const std::map& pid_offset_map_i, + const std::map& pid_endpoint_map_i) : TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kExtractIndexData, ""), - tid(tid_i), pid(pid_i), endpoints(endpoints_i), partition_num(partition_num_i), - column_key(column_key_i), idx(idx_i) {} + tid(tid_i), partition_num(partition_num_i), column_key(column_key_i), + pid_offset_map(pid_offset_map_i), pid_endpoint_map(pid_endpoint_map_i) { + task_info->set_tid(tid); + } uint32_t tid; - uint32_t pid; - std::vector endpoints; uint32_t partition_num; - ::openmldb::common::ColumnKey column_key; - uint32_t idx; + std::vector<::openmldb::common::ColumnKey> column_key; + std::map pid_offset_map; + std::map pid_endpoint_map; }; class AddIndexToTabletTaskMeta : public TaskMeta { public: AddIndexToTabletTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, - uint32_t tid_i, uint32_t pid_i, const std::vector& endpoints_i, - const ::openmldb::common::ColumnKey& column_key_i) : + const ::openmldb::nameserver::TableInfo& table_info_i, + const std::vector<::openmldb::common::ColumnKey>& column_key_i) : TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kAddIndexToTablet, ""), - tid(tid_i), pid(pid_i), endpoints(endpoints_i), column_key(column_key_i) {} + table_info(table_info_i), column_key(column_key_i) { + task_info->set_tid(table_info.tid()); + } + ::openmldb::nameserver::TableInfo table_info; + std::vector<::openmldb::common::ColumnKey> column_key; +}; + +class AddIndexToTabletRequestTaskMeta : public TaskMeta { + public: + AddIndexToTabletRequestTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, const std::string& endpoint, + uint32_t tid_i, uint32_t pid_i, + const std::vector<::openmldb::common::ColumnKey>& column_key_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kAddIndexToTabletRequest, endpoint), + tid(tid_i), pid(pid_i), column_key(column_key_i) { + task_info->set_tid(tid); + task_info->set_pid(pid); + } uint32_t tid; uint32_t pid; - std::vector endpoints; - ::openmldb::common::ColumnKey column_key; + std::vector<::openmldb::common::ColumnKey> column_key; +}; + +class AddIndexToTableInfoTaskMeta : public TaskMeta { + public: + AddIndexToTableInfoTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + const std::string& table_name, const std::string& db_name, + const std::vector<::openmldb::common::ColumnKey>& column_key_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kAddIndexToTableInfo, ""), + name(table_name), db(db_name), column_key(column_key_i) {} + std::string name; + std::string db; + std::vector<::openmldb::common::ColumnKey> column_key; }; class AddTableInfoTaskMeta : public TaskMeta { @@ -400,6 +490,35 @@ class AddReplicaNSRemoteTaskMeta : public TaskMeta { std::vector endpoint_vec; uint32_t pid; }; + +class AddMultiTableIndexTaskMeta : public TaskMeta { + public: + AddMultiTableIndexTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + const ::google::protobuf::RepeatedPtrField& table_index_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kAddMultiTableIndex, ""), table_index(table_index_i) {} + ::google::protobuf::RepeatedPtrField table_index; +}; + +class AddTableIndexTaskMeta : public TaskMeta { + public: + AddTableIndexTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + const std::string& name_i, const std::string& db_i, + const std::vector<::openmldb::common::ColumnKey>& column_key_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kAddTableIndex, ""), + name(name_i), db(db_i), column_key(column_key_i) {} + std::string name; + std::string db; + std::vector<::openmldb::common::ColumnKey> column_key; +}; + +class CreateProcedureTaskMeta : public TaskMeta { + public: + CreateProcedureTaskMeta(uint64_t op_id, ::openmldb::api::OPType op_type, + const api::ProcedureInfo& sp_info_i) : + TaskMeta(op_id, op_type, ::openmldb::api::TaskType::kCreateProcedure, ""), sp_info(sp_info_i) {} + api::ProcedureInfo sp_info; +}; + } // namespace nameserver } // namespace openmldb #endif // SRC_NAMESERVER_TASK_H_ diff --git a/src/proto/name_server.proto b/src/proto/name_server.proto index d74346ec801..b0eb526d8e7 100755 --- a/src/proto/name_server.proto +++ b/src/proto/name_server.proto @@ -345,6 +345,7 @@ message ShowOPStatusRequest{ optional string name = 1; optional uint32 pid = 2; optional string db = 3; + optional uint64 op_id = 4; } message ConnectZKRequest {} @@ -424,6 +425,7 @@ message AddIndexMeta { optional uint32 idx = 4; optional string db = 5 [default = ""]; optional bool skip_data = 6 [default = false]; + repeated openmldb.common.ColumnKey column_keys = 7; } message AddIndexRequest { @@ -432,6 +434,7 @@ message AddIndexRequest { optional string db = 3 [default = ""]; repeated openmldb.common.ColumnDesc cols = 4; repeated openmldb.common.ColumnKey column_keys = 5; + optional bool skip_load_data = 6 [default = false]; } message DeleteIndexRequest { @@ -506,6 +509,23 @@ message ShowFunctionResponse { repeated openmldb.common.ExternalFun fun = 3; } +message TableIndex { + optional string db = 1; + optional string name = 2; + repeated openmldb.common.ColumnKey column_key = 3; +} + +message DeploySQLRequest { + optional openmldb.api.ProcedureInfo sp_info = 3; + repeated TableIndex index = 4; +} + +message DeploySQLResponse { + optional int32 code = 1; + optional string msg = 2; + optional uint64 op_id = 3; +} + service NameServer { rpc CreateTable(CreateTableRequest) returns (GeneralResponse); rpc DropTable(DropTableRequest) returns (GeneralResponse); @@ -561,4 +581,5 @@ service NameServer { rpc CreateProcedure(openmldb.api.CreateProcedureRequest) returns (GeneralResponse); rpc DropProcedure(openmldb.api.DropProcedureRequest) returns (GeneralResponse); rpc ShowProcedure(openmldb.api.ShowProcedureRequest) returns (openmldb.api.ShowProcedureResponse); + rpc DeploySQL(DeploySQLRequest) returns (DeploySQLResponse); } diff --git a/src/proto/tablet.proto b/src/proto/tablet.proto index 321719a8333..5b9b2933520 100755 --- a/src/proto/tablet.proto +++ b/src/proto/tablet.proto @@ -67,6 +67,7 @@ enum OPType { kDelReplicaRemoteOP = 18; kAddReplicaRemoteOP = 19; kAddIndexOP = 20; + kDeployOP = 21; } enum TaskType { @@ -91,12 +92,20 @@ enum TaskType { kCreateTableRemote = 19; kDropTableRemote = 20; kAddReplicaNSRemote = 21; - kDumpIndexData = 22; + kDumpIndexData = 22 [deprecated = true]; kSendIndexData = 23; kLoadIndexData = 24; kExtractIndexData = 25; kAddIndexToTablet = 26; - kTableSyncTask = 27; + kTableSyncTask = 27 [deprecated = true]; + kSendIndexRequest = 28; + kExtractIndexRequest = 29; + kLoadIndexRequest = 30; + kAddIndexToTabletRequest = 31; + kAddIndexToTableInfo = 32; + kAddMultiTableIndex = 33; + kAddTableIndex = 34; + kCreateProcedure = 35; } enum TaskStatus { @@ -120,7 +129,9 @@ message TaskInfo { optional string endpoint = 5; optional bool is_rpc_send = 6 [default = false]; repeated uint64 rep_cluster_op_id = 7; // for multi cluster - optional uint64 task_id = 8 [default = 0]; // for multi cluster + optional uint64 task_id = 8 [default = 0]; + optional uint32 tid = 9; + optional uint32 pid = 10; } message OPInfo { @@ -609,15 +620,6 @@ message DeleteIndexRequest { optional string idx_name = 3; } -message DumpIndexDataRequest { - optional uint32 tid = 1; - optional uint32 pid = 2; - optional uint32 partition_num = 3; - optional uint32 idx = 4; - optional openmldb.common.ColumnKey column_key = 5; - optional TaskInfo task_info = 6; -} - message LoadIndexDataRequest { optional uint32 tid = 1; optional uint32 pid = 2; @@ -625,20 +627,14 @@ message LoadIndexDataRequest { optional TaskInfo task_info = 4; } -message ExtractMultiIndexDataRequest { - optional uint32 tid = 1; - optional uint32 pid = 2; - optional uint32 partition_num = 3; - repeated openmldb.common.ColumnKey column_key = 4; -} - message ExtractIndexDataRequest { optional uint32 tid = 1; optional uint32 pid = 2; optional uint32 partition_num = 3; - optional uint32 idx = 4; - optional openmldb.common.ColumnKey column_key = 5; - optional TaskInfo task_info = 6; + repeated openmldb.common.ColumnKey column_key = 5; + optional uint64 offset = 6; + optional bool dump_data = 7 [default = true]; + optional TaskInfo task_info = 8; } message Columns { @@ -940,10 +936,8 @@ service TabletServer { rpc AddIndex(AddIndexRequest) returns (GeneralResponse); rpc SendIndexData(SendIndexDataRequest) returns (GeneralResponse); rpc DeleteIndex(DeleteIndexRequest) returns (GeneralResponse); - rpc DumpIndexData(DumpIndexDataRequest) returns (GeneralResponse); rpc LoadIndexData(LoadIndexDataRequest) returns (GeneralResponse); rpc ExtractIndexData(ExtractIndexDataRequest) returns (GeneralResponse); - rpc ExtractMultiIndexData(ExtractMultiIndexDataRequest) returns (GeneralResponse); rpc CancelOP(CancelOPRequest) returns (GeneralResponse); rpc UpdateRealEndpointMap(UpdateRealEndpointMapRequest) returns (GeneralResponse); diff --git a/src/schema/index_test.cc b/src/schema/index_test.cc index 41844699248..fe79d6ee44b 100644 --- a/src/schema/index_test.cc +++ b/src/schema/index_test.cc @@ -44,7 +44,7 @@ TEST_F(IndexTest, CheckUnique) { ASSERT_FALSE(IndexUtil::CheckUnique(indexs).OK()); } -TEST_F(IndexTest, CheckNewIndex) { +TEST_F(IndexTest, CheckExist) { openmldb::nameserver::TableInfo table_info; SchemaCodec::SetColumnDesc(table_info.add_column_desc(), "card", ::openmldb::type::kString); SchemaCodec::SetColumnDesc(table_info.add_column_desc(), "mcc", ::openmldb::type::kString); @@ -61,10 +61,13 @@ TEST_F(IndexTest, CheckNewIndex) { SchemaCodec::SetIndex(&test_index2, "test_index2", "card", "ts2", ::openmldb::type::kAbsoluteTime, 0, 0); ::openmldb::common::ColumnKey test_index3; SchemaCodec::SetIndex(&test_index3, "test_index3", "mcc", "ts2", ::openmldb::type::kAbsoluteTime, 0, 0); + ::openmldb::common::ColumnKey test_index4; + SchemaCodec::SetIndex(&test_index4, "index1", "aa", "ts2", ::openmldb::type::kAbsoluteTime, 0, 0); - ASSERT_FALSE(IndexUtil::CheckNewIndex(test_index1, table_info).OK()); - ASSERT_TRUE(IndexUtil::CheckNewIndex(test_index2, table_info).OK()); - ASSERT_TRUE(IndexUtil::CheckNewIndex(test_index3, table_info).OK()); + ASSERT_TRUE(IndexUtil::IsExist(test_index1, table_info.column_key())); + ASSERT_TRUE(IndexUtil::IsExist(test_index2, table_info.column_key())); + ASSERT_FALSE(IndexUtil::IsExist(test_index3, table_info.column_key())); + ASSERT_TRUE(IndexUtil::IsExist(test_index4, table_info.column_key())); } } // namespace schema diff --git a/src/schema/index_util.cc b/src/schema/index_util.cc index 1893ddd5aad..009b04d23cd 100644 --- a/src/schema/index_util.cc +++ b/src/schema/index_util.cc @@ -137,11 +137,9 @@ base::Status IndexUtil::CheckUnique(const PBIndex& index) { return {}; } -bool IndexUtil::CheckExist(const ::openmldb::common::ColumnKey& column_key, - const PBIndex& index, int32_t* pos) { - int32_t index_pos = 0; +bool IndexUtil::IsExist(const ::openmldb::common::ColumnKey& column_key, const PBIndex& index) { std::string id_str = GetIDStr(column_key); - for (; index_pos < index.size(); index_pos++) { + for (int32_t index_pos = 0; index_pos < index.size(); index_pos++) { if (index.Get(index_pos).index_name() == column_key.index_name()) { if (index.Get(index_pos).flag() == 0) { return true; @@ -152,10 +150,33 @@ bool IndexUtil::CheckExist(const ::openmldb::common::ColumnKey& column_key, return true; } } - *pos = index_pos; return false; } +int IndexUtil::GetPosition(const ::openmldb::common::ColumnKey& column_key, const PBIndex& index) { + std::string id_str = GetIDStr(column_key); + for (int32_t index_pos = 0; index_pos < index.size(); index_pos++) { + if (index.Get(index_pos).index_name() == column_key.index_name()) { + if (index.Get(index_pos).flag() == 0) { + return index_pos; + } + break; + } + if (id_str == GetIDStr(index.Get(index_pos))) { + return index_pos; + } + } + return -1; +} + +std::vector<::openmldb::common::ColumnKey> IndexUtil::Convert2Vector(const PBIndex& index) { + std::vector<::openmldb::common::ColumnKey> vec; + for (const auto& column_key : index) { + vec.push_back(column_key); + } + return vec; +} + std::string IndexUtil::GetIDStr(const ::openmldb::common::ColumnKey& column_key) { std::string id_str; for (const auto& cur_col : column_key.col_name()) { @@ -167,26 +188,5 @@ std::string IndexUtil::GetIDStr(const ::openmldb::common::ColumnKey& column_key) return id_str; } -base::Status IndexUtil::CheckNewIndex(const ::openmldb::common::ColumnKey& column_key, - const openmldb::nameserver::TableInfo& table_info) { - if (table_info.column_key_size() == 0) { - return {base::ReturnCode::kError, "has no index"}; - } - std::map col_map; - for (const auto& column_desc : table_info.column_desc()) { - col_map.emplace(column_desc.name(), column_desc); - } - for (const auto& column_desc : table_info.added_column_desc()) { - col_map.emplace(column_desc.name(), column_desc); - } - std::string id_str = GetIDStr(column_key); - for (const auto& cur_column_key : table_info.column_key()) { - if (id_str == GetIDStr(cur_column_key) && cur_column_key.flag() == 0) { - return {base::ReturnCode::kError, "duplicated index"}; - } - } - return {}; -} - } // namespace schema } // namespace openmldb diff --git a/src/schema/index_util.h b/src/schema/index_util.h index 83592862605..c5f6f90429d 100644 --- a/src/schema/index_util.h +++ b/src/schema/index_util.h @@ -19,6 +19,7 @@ #include #include +#include #include "base/status.h" #include "proto/common.pb.h" #include "proto/name_server.pb.h" @@ -35,11 +36,11 @@ class IndexUtil { static base::Status CheckIndex(const std::map& column_map, const PBIndex& index); - static base::Status CheckNewIndex(const ::openmldb::common::ColumnKey& column_key, - const openmldb::nameserver::TableInfo& table_info); + static bool IsExist(const ::openmldb::common::ColumnKey& column_key, const PBIndex& index); - static bool CheckExist(const ::openmldb::common::ColumnKey& column_key, - const PBIndex& index, int32_t* pos); + static int GetPosition(const ::openmldb::common::ColumnKey& column_key, const PBIndex& index); + + static std::vector<::openmldb::common::ColumnKey> Convert2Vector(const PBIndex& index); static base::Status CheckUnique(const PBIndex& index); diff --git a/src/sdk/sql_cluster_router.cc b/src/sdk/sql_cluster_router.cc index 72ddd500475..46a3c59a9f8 100644 --- a/src/sdk/sql_cluster_router.cc +++ b/src/sdk/sql_cluster_router.cc @@ -59,6 +59,7 @@ DECLARE_string(bucket_size); DECLARE_uint32(replica_num); DECLARE_int32(sync_job_timeout); +DECLARE_int32(deploy_job_max_wait_time_ms); namespace openmldb { namespace sdk { @@ -66,7 +67,8 @@ namespace sdk { using hybridse::common::StatusCode; using hybridse::plan::PlanAPI; -constexpr const char* SKIP_INDEX_CHECK = "skip_index_check"; +constexpr const char* SKIP_INDEX_CHECK_OPTION = "skip_index_check"; +constexpr const char* SYNC_OPTION = "sync"; class ExplainInfoImpl : public ExplainInfo { public: @@ -2536,9 +2538,18 @@ std::shared_ptr SQLClusterRouter::ExecuteSQL( "Can not deploy in offline mode, please set @@SESSION.execute_mode='online'"}; return {}; } - - *status = HandleDeploy(db, dynamic_cast(node)); + auto deploy_node = dynamic_cast(node); + std::optional job_id; + *status = HandleDeploy(db, deploy_node, &job_id); if (status->IsOK()) { + if (job_id.has_value()) { + schema::PBSchema job_schema; + auto col = job_schema.Add(); + col->set_name("job_id"); + col->set_data_type(::openmldb::type::DataType::kBigInt); + std::vector value = {std::to_string(job_id.value())}; + return ResultSetSQL::MakeResultSet(job_schema, {value}, status); + } RefreshCatalog(); } return {}; @@ -3261,7 +3272,8 @@ hybridse::sdk::Status SQLClusterRouter::HandleCreateFunction(const hybridse::nod } hybridse::sdk::Status SQLClusterRouter::HandleDeploy(const std::string& db, - const hybridse::node::DeployPlanNode* deploy_node) { + const hybridse::node::DeployPlanNode* deploy_node, + std::optional* job_id) { hybridse::sdk::Status status; if (db.empty()) { return {StatusCode::kCmdError, "database is empty"}; @@ -3323,35 +3335,12 @@ hybridse::sdk::Status SQLClusterRouter::HandleDeploy(const std::string& db, sp_info.set_sql(str_stream.str()); sp_info.set_type(::openmldb::type::ProcedureType::kReqDeployment); - - auto index_status = HandleIndex(db, table_pair, select_sql, deploy_node); - if (!index_status.IsOK()) { - return index_status; - } - - auto lw_status = HandleLongWindows(deploy_node, table_pair, select_sql); - if (!lw_status.IsOK()) { - return lw_status; - } for (const auto& o : *deploy_node->Options()) { auto option = sp_info.add_options(); option->set_name(o.first); option->mutable_value()->set_value(o.second->GetExprString()); } - auto ob_status = cluster_sdk_->GetNsClient()->CreateProcedure(sp_info, options_->request_timeout); - if (!ob_status.OK()) { - APPEND_FROM_BASE_AND_WARN(&status, ob_status, "ns create procedure failed"); - return status; - } - return {}; -} - -hybridse::sdk::Status SQLClusterRouter::HandleIndex(const std::string& db, - const std::set>& table_pair, - const std::string& select_sql, - const hybridse::node::DeployPlanNode* deploy_node) { - // extract index from sql std::vector<::openmldb::nameserver::TableInfo> tables; auto ns = cluster_sdk_->GetNsClient(); // TODO(denglong): support multi db @@ -3359,20 +3348,21 @@ hybridse::sdk::Status SQLClusterRouter::HandleIndex(const std::string& db, if (!ret.OK()) { return {StatusCode::kCmdError, "get table failed " + ret.msg}; } - std::map> table_schema_map; std::map table_map; + uint64_t record_cnt = 0; for (const auto& table : tables) { for (const auto& pair : table_pair) { if (table.name() == pair.second) { - table_schema_map.emplace(table.name(), table.column_desc()); table_map.emplace(table.name(), table); + for (int idx = 0; idx < table.table_partition_size(); idx++) { + record_cnt += table.table_partition(idx).record_cnt(); + } break; } } } - auto index_map = base::DDLParser::ExtractIndexes(select_sql, table_schema_map); bool skip_index_check = false; - auto iter = deploy_node->Options()->find(SKIP_INDEX_CHECK); + auto iter = deploy_node->Options()->find(SKIP_INDEX_CHECK_OPTION); if (iter != deploy_node->Options()->end()) { std::string skip_index_value = iter->second->GetExprString(); if (absl::EqualsIgnoreCase(absl::string_view(skip_index_value), absl::string_view("true"))) { @@ -3380,23 +3370,88 @@ hybridse::sdk::Status SQLClusterRouter::HandleIndex(const std::string& db, } } std::map> new_index_map; - auto get_index_status = GetNewIndex(table_map, index_map, skip_index_check, &new_index_map); - if (!get_index_status.IsOK()) { - return get_index_status; + auto index_status = GetNewIndex(table_map, select_sql, skip_index_check, &new_index_map); + if (!index_status.IsOK()) { + return index_status; } - - auto add_index_status = AddNewIndex(db, table_map, new_index_map); - if (!add_index_status.IsOK()) { - return add_index_status; + if (!new_index_map.empty()) { + if (cluster_sdk_->IsClusterMode() && record_cnt > 0) { + auto lw_status = HandleLongWindows(deploy_node, table_pair, select_sql); + if (!lw_status.IsOK()) { + return lw_status; + } + uint64_t id = 0; + auto deploy_status = ns->DeploySQL(sp_info, new_index_map, &id); + if (!deploy_status.OK()) { + return {deploy_status.GetCode(), deploy_status.GetMsg()}; + } + bool sync = true; + auto iter = deploy_node->Options()->find(SYNC_OPTION); + if (iter != deploy_node->Options()->end()) { + std::string skip_index_value = iter->second->GetExprString(); + if (absl::EqualsIgnoreCase(absl::string_view(skip_index_value), absl::string_view("false"))) { + sync = false; + } + } + if (sync) { + if (record_cnt > 1000000) { + LOG(INFO) << "There is data in the table, it may take a few minutes to load the data"; + } else if (record_cnt > 100000) { + LOG(INFO) << "There is data in the table, it may take a few seconds to load the data"; + } + uint64_t start_ts = ::baidu::common::timer::get_micros() / 1000; + while (true) { + nameserver::ShowOPStatusResponse response; + auto status = ns->ShowOPStatus(id, &response); + if (!status.OK()) { + return {status.GetCode(), status.GetMsg()}; + } + if (response.op_status_size() < 1) { + return {-1, absl::StrCat("op does not exist. id ", id)}; + } + if (response.op_status(0).status() == "kDone") { + return {}; + } else if (response.op_status(0).status() == "kFailed" || + response.op_status(0).status() == "kCanceled") { + return {-1, absl::StrCat("op status is ", response.op_status(0).status())}; + } + uint64_t cur_ts = ::baidu::common::timer::get_micros() / 1000; + if (cur_ts - start_ts > static_cast(FLAGS_deploy_job_max_wait_time_ms)) { + return {-1, "exceed max wait time"}; + } + sleep(1); + } + } else { + job_id->emplace(id); + } + return {}; + } else { + auto add_index_status = AddNewIndex(db, table_map, new_index_map); + if (!add_index_status.IsOK()) { + return add_index_status; + } + } + } + auto lw_status = HandleLongWindows(deploy_node, table_pair, select_sql); + if (!lw_status.IsOK()) { + return lw_status; + } + auto ob_status = ns->CreateProcedure(sp_info, options_->request_timeout); + if (!ob_status.OK()) { + APPEND_FROM_BASE_AND_WARN(&status, ob_status, "ns create procedure failed"); + return status; } return {}; } hybridse::sdk::Status SQLClusterRouter::GetNewIndex( - const std::map& table_map, - const std::map>& index_map, - bool skip_index_check, - std::map>* new_index_map) { + const std::map& table_map, const std::string& select_sql, + bool skip_index_check, std::map>* new_index_map) { + std::map> table_schema_map; + for (const auto& kv : table_map) { + table_schema_map.emplace(kv.first, kv.second.column_desc()); + } + auto index_map = base::DDLParser::ExtractIndexes(select_sql, table_schema_map); auto ns = cluster_sdk_->GetNsClient(); for (auto& kv : index_map) { std::string table_name = kv.first; @@ -3488,25 +3543,13 @@ hybridse::sdk::Status SQLClusterRouter::GetNewIndex( } } } else { - column_key.set_index_name("INDEX_" + std::to_string(cur_index_num + add_index_num) + "_" + - std::to_string(::baidu::common::timer::now_time())); + column_key.set_index_name(absl::StrCat("INDEX_", cur_index_num + add_index_num, "_", + ::baidu::common::timer::now_time())); add_index_num++; new_indexs.emplace_back(column_key); } } if (!new_indexs.empty()) { - if (cluster_sdk_->IsClusterMode()) { - // TODO(zhanghaohit): record_cnt is updated by ns periodically, causing a delay to get the latest value - uint64_t record_cnt = 0; - for (int idx = 0; idx < table.table_partition_size(); idx++) { - record_cnt += table.table_partition(idx).record_cnt(); - } - if (record_cnt > 0) { - return {StatusCode::kUnSupport, - "table " + table_name + - " has online data, cannot deploy. please drop this table and create a new one"}; - } - } new_index_map->emplace(table_name, std::move(new_indexs)); } } @@ -3517,52 +3560,12 @@ hybridse::sdk::Status SQLClusterRouter::AddNewIndex( const std::string& db, const std::map& table_map, const std::map>& new_index_map) { auto ns = cluster_sdk_->GetNsClient(); - if (cluster_sdk_->IsClusterMode()) { - for (auto& kv : new_index_map) { - auto status = ns->AddMultiIndex(db, kv.first, kv.second); - if (!status.OK()) { - return {StatusCode::kCmdError, "table [" + db + "." + kv.first + "] add index failed. " + status.msg}; - } - } - } else { - auto tablet_accessor = cluster_sdk_->GetTablet(); - if (!tablet_accessor) { - return {StatusCode::kCmdError, "cannot connect tablet"}; - } - auto tablet_client = tablet_accessor->GetClient(); - if (!tablet_client) { - return {StatusCode::kCmdError, "tablet client is null"}; - } - // add index - for (auto& kv : new_index_map) { - auto it = table_map.find(kv.first); - for (auto& column_key : kv.second) { - std::vector cols; - for (const auto& col_name : column_key.col_name()) { - for (const auto& col : it->second.column_desc()) { - if (col.name() == col_name) { - cols.push_back(col); - break; - } - } - } - std::string msg; - if (!ns->AddIndex(kv.first, column_key, &cols, msg)) { - return {StatusCode::kCmdError, "table " + kv.first + " add index failed"}; - } - } - } - // load new index data to table - for (auto& kv : new_index_map) { - auto it = table_map.find(kv.first); - if (it == table_map.end()) { - continue; - } - uint32_t tid = it->second.tid(); - uint32_t pid = 0; - if (!tablet_client->ExtractMultiIndexData(tid, pid, it->second.table_partition_size(), kv.second)) { - return {StatusCode::kCmdError, "table " + kv.first + " load data failed"}; - } + for (auto& kv : new_index_map) { + bool skip_load_data = cluster_sdk_->IsClusterMode(); + auto status = ns->AddMultiIndex(db, kv.first, kv.second, skip_load_data); + if (!status.OK()) { + return {StatusCode::kCmdError, + absl::StrCat("table [", db, ".", kv.first, "] add index failed. ", status.msg)}; } } return {}; diff --git a/src/sdk/sql_cluster_router.h b/src/sdk/sql_cluster_router.h index 8cf4cc9a5fe..359fb9ae2fa 100644 --- a/src/sdk/sql_cluster_router.h +++ b/src/sdk/sql_cluster_router.h @@ -327,21 +327,15 @@ class SQLClusterRouter : public SQLRouter { const std::vector& str_col_idx, const std::string& null_value, const std::vector& cols); - hybridse::sdk::Status HandleDeploy(const std::string& db, const hybridse::node::DeployPlanNode* deploy_node); + hybridse::sdk::Status HandleDeploy(const std::string& db, const hybridse::node::DeployPlanNode* deploy_node, + std::optional* job_id); hybridse::sdk::Status HandleDelete(const std::string& db, const std::string& table_name, const hybridse::node::ExprNode* condition); - hybridse::sdk::Status HandleIndex(const std::string& db, - const std::set>& table_pair, - const std::string& select_sql, - const hybridse::node::DeployPlanNode* deploy_node); - hybridse::sdk::Status GetNewIndex( - const std::map& table_map, - const std::map>& index_map, - bool skip_index_check, - std::map>* new_index_map); + const std::map& table_map, const std::string& select_sql, + bool skip_index_check, std::map>* new_index_map); hybridse::sdk::Status AddNewIndex( const std::string& db, const std::map& table_map, diff --git a/src/storage/mem_table_snapshot.cc b/src/storage/mem_table_snapshot.cc index 31424afcbe6..2b023f4825b 100644 --- a/src/storage/mem_table_snapshot.cc +++ b/src/storage/mem_table_snapshot.cc @@ -16,7 +16,6 @@ #include "storage/mem_table_snapshot.h" -#include #ifdef DISALLOW_COPY_AND_ASSIGN #undef DISALLOW_COPY_AND_ASSIGN #endif @@ -26,6 +25,7 @@ #include #include +#include "absl/cleanup/cleanup.h" #include "base/file_util.h" #include "base/glog_wrapper.h" #include "base/hash.h" @@ -37,13 +37,11 @@ #include "common/thread_pool.h" #include "common/timer.h" #include "gflags/gflags.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" #include "log/log_reader.h" #include "log/sequential_file.h" #include "proto/tablet.pb.h" -using google::protobuf::RepeatedPtrField; -using ::openmldb::codec::SchemaCodec; - DECLARE_uint64(gc_on_table_recover_count); DECLARE_int32(binlog_name_length); DECLARE_uint32(make_snapshot_max_deleted_keys); @@ -55,16 +53,245 @@ DECLARE_string(snapshot_compression); namespace openmldb { namespace storage { -const std::string SNAPSHOT_SUBFIX = ".sdb"; // NOLINT -const uint32_t KEY_NUM_DISPLAY = 1000000; // NOLINT -const std::string MANIFEST = "MANIFEST"; // NOLINT +constexpr const char* SNAPSHOT_SUBFIX = ".sdb"; +constexpr uint32_t KEY_NUM_DISPLAY = 1000000; +constexpr const char* MANIFEST = "MANIFEST"; + +bool IsCompressed(const std::string& path) { + if (path.find(openmldb::log::ZLIB_COMPRESS_SUFFIX) != std::string::npos || + path.find(openmldb::log::SNAPPY_COMPRESS_SUFFIX) != std::string::npos) { + return true; + } + return false; +} + +std::shared_ptr DataReader::CreateDataReader(const std::string& snapshot_path, LogParts* log_part, + const std::string& log_path, DataReaderType type) { + return CreateDataReader(snapshot_path, log_part, log_path, type, 0); +} + +std::shared_ptr DataReader::CreateDataReader(LogParts* log_part, const std::string& log_path, + uint64_t start_offset, uint64_t end_offset) { + auto reader = std::make_shared("", log_part, log_path, DataReaderType::kBinlog, + start_offset, end_offset); + if (!reader->Init()) { + reader.reset(); + } + return reader; +} + +std::shared_ptr DataReader::CreateDataReader(const std::string& snapshot_path, LogParts* log_part, + const std::string& log_path, DataReaderType type, uint64_t end_offset) { + auto reader = std::make_shared(snapshot_path, log_part, log_path, type, 0, end_offset); + if (!reader->Init()) { + reader.reset(); + } + return reader; +} + +bool DataReader::Init() { + uint64_t snapshot_offset = 0; + if (read_type_ == DataReaderType::kSnapshot || read_type_ == DataReaderType::kSnapshotAndBinlog) { + ::openmldb::api::Manifest manifest; + int ret = Snapshot::GetLocalManifest(snapshot_path_ + MANIFEST, manifest); + if (ret == -1) { + return false; + } else if (ret == 0) { + snapshot_offset = manifest.offset(); + std::string path = absl::StrCat(snapshot_path_, "/", manifest.name()); + FILE* fd = fopen(path.c_str(), "rb"); + if (fd == nullptr) { + PDLOG(WARNING, "fail to open path %s for error %s", path.c_str(), strerror(errno)); + return false; + } + seq_file_.reset(::openmldb::log::NewSeqFile(path, fd)); + bool compressed = IsCompressed(path); + snapshot_reader_ = std::make_shared<::openmldb::log::Reader>( + seq_file_.get(), nullptr, false, 0, compressed); + read_snapshot_ = true; + } + } + if (read_type_ == DataReaderType::kBinlog || read_type_ == DataReaderType::kSnapshotAndBinlog) { + if (log_part_ == nullptr) { + return false; + } + binlog_reader_ = std::make_shared<::openmldb::log::LogReader>(log_part_, log_path_, false); + cur_offset_ = std::max(start_offset_, snapshot_offset); + binlog_reader_->SetOffset(cur_offset_); + read_binlog_ = true; + } + return true; +} + +bool DataReader::ReadFromSnapshot() { + if (!read_snapshot_) { + return false; + } + do { + buffer_.clear(); + record_.clear(); + auto status = snapshot_reader_->ReadRecord(&record_, &buffer_); + if (status.IsWaitRecord() || status.IsEof()) { + PDLOG(INFO, "read snapshot completed, succ_cnt %lu, failed_cnt %lu, path %s", + succ_cnt_, failed_cnt_, snapshot_path_.c_str()); + succ_cnt_ = 0; + failed_cnt_ = 0; + read_snapshot_ = false; + return false; + } + if (!status.ok()) { + PDLOG(WARNING, "fail to read snapshot record. path %s, error %s", + snapshot_path_.c_str(), status.ToString().c_str()); + failed_cnt_++; + continue; + } + entry_buff_.assign(record_.data(), record_.size()); + if (!entry_.ParseFromString(entry_buff_)) { + PDLOG(WARNING, "fail to parse record. path %s", snapshot_path_); + failed_cnt_++; + continue; + } + succ_cnt_++; + break; + } while (true); + return true; +} + +bool DataReader::ReadFromBinlog() { + if (!read_binlog_) { + return false; + } + int last_log_index = binlog_reader_->GetLogIndex(); + do { + if (end_offset_ > 0 && cur_offset_ >= end_offset_) { + break; + } + buffer_.clear(); + record_.clear(); + auto status = binlog_reader_->ReadNextRecord(&record_, &buffer_); + if (status.IsWaitRecord()) { + int end_log_index = binlog_reader_->GetEndLogIndex(); + int cur_log_index = binlog_reader_->GetLogIndex(); + if (end_log_index >= 0 && end_log_index > cur_log_index) { + binlog_reader_->RollRLogFile(); + PDLOG(INFO, "read new binlog file. path[%s] cur_log_index[%d] " + "end_log_index[%d] cur_offset[%lu]", + log_path_.c_str(), cur_log_index, end_log_index, cur_offset_); + continue; + } + PDLOG(INFO, "read binlog completed, succ_cnt %lu, failed_cnt %lu, path %s", + succ_cnt_, failed_cnt_, log_path_.c_str()); + break; + } else if (status.IsEof()) { + if (binlog_reader_->GetLogIndex() != last_log_index) { + continue; + } + } else if (!status.ok()) { + failed_cnt_++; + continue; + } + entry_buff_.assign(record_.data(), record_.size()); + if (!entry_.ParseFromString(entry_buff_)) { + PDLOG(WARNING, "fail to parse record. path %s", log_path_.c_str()); + failed_cnt_++; + continue; + } + if (cur_offset_ >= entry_.log_index()) { + DEBUGLOG("cur offset is %lu, skip %lu", cur_offset_, entry_.log_index()); + continue; + } + if (cur_offset_ + 1 != entry_.log_index()) { + PDLOG(WARNING, "missing log entry. cur_offset %lu, new entry offset %lu, path %s", + cur_offset_, entry_.log_index(), log_path_.c_str()); + continue; + } + succ_cnt_++; + cur_offset_ = entry_.log_index(); + return true; + } while (true); + read_binlog_ = false; + return false; +} + +bool DataReader::HasNext() { + if (read_snapshot_ && ReadFromSnapshot()) { + return true; + } + if (read_binlog_ && ReadFromBinlog()) { + return true; + } + return false; +} + +bool TableIndexInfo::Init() { + for (int32_t i = 0; i < table_meta_.column_desc_size(); i++) { + column_idx_map_.emplace(table_meta_.column_desc(i).name(), i); + } + uint32_t base_size = table_meta_.column_desc_size(); + for (int32_t i = 0; i < table_meta_.added_column_desc_size(); ++i) { + column_idx_map_.emplace(table_meta_.added_column_desc(i).name(), i + base_size); + } + std::set index_col_set; + for (int32_t i = 0; i < table_meta_.column_key_size(); i++) { + const auto& ck = table_meta_.column_key(i); + if (ck.flag()) { + continue; + } + for (const auto& add_ck : add_indexs_) { + if (ck.index_name() == add_ck.index_name()) { + add_index_idx_vec_.push_back(i); + break; + } + } + std::vector cols; + for (const auto& name : ck.col_name()) { + auto iter = column_idx_map_.find(name); + if (iter != column_idx_map_.end()) { + cols.push_back(iter->second); + index_col_set.insert(iter->second); + } else { + PDLOG(WARNING, "fail to find column_desc %s", name.c_str()); + return false; + } + } + index_cols_map_.emplace(i, std::move(cols)); + } + if (add_index_idx_vec_.size() != add_indexs_.size()) { + return false; + } + std::map col_idx_map; + for (auto idx : index_col_set) { + col_idx_map.emplace(idx, all_index_cols_.size()); + all_index_cols_.push_back(idx); + } + for (const auto& kv : index_cols_map_) { + std::vector vec; + for (auto idx : kv.second) { + vec.push_back(col_idx_map[idx]); + } + real_index_cols_map_.emplace(kv.first, std::move(vec)); + } + return true; +} + +bool TableIndexInfo::HasIndex(uint32_t idx) const { + return index_cols_map_.find(idx) != index_cols_map_.end(); +} + +const std::vector& TableIndexInfo::GetIndexCols(uint32_t idx) { + return index_cols_map_[idx]; +} + +const std::vector& TableIndexInfo::GetRealIndexCols(uint32_t idx) { + return real_index_cols_map_[idx]; +} MemTableSnapshot::MemTableSnapshot(uint32_t tid, uint32_t pid, LogParts* log_part, const std::string& db_root_path) : Snapshot(tid, pid), log_part_(log_part), db_root_path_(db_root_path) {} bool MemTableSnapshot::Init() { - snapshot_path_ = db_root_path_ + "/" + std::to_string(tid_) + "_" + std::to_string(pid_) + "/snapshot/"; - log_path_ = db_root_path_ + "/" + std::to_string(tid_) + "_" + std::to_string(pid_) + "/binlog/"; + snapshot_path_ = absl::StrCat(db_root_path_, "/", tid_, "_", pid_, "/snapshot/"); + log_path_ = absl::StrCat(db_root_path_, "/", tid_, "_", pid_, "/binlog/"); if (!::openmldb::base::MkdirRecur(snapshot_path_)) { PDLOG(WARNING, "fail to create db meta path %s", snapshot_path_.c_str()); return false; @@ -195,8 +422,7 @@ void MemTableSnapshot::Put(std::string& path, std::shared_ptr& table, std } int MemTableSnapshot::TTLSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, - WriteHandle* wh, uint64_t& count, uint64_t& expired_key_num, - uint64_t& deleted_key_num) { + const std::shared_ptr& wh, MemSnapshotMeta* snapshot_meta) { std::string full_path = snapshot_path_ + manifest.name(); FILE* fd = fopen(full_path.c_str(), "rb"); if (fd == NULL) { @@ -237,13 +463,13 @@ int MemTableSnapshot::TTLSnapshot(std::shared_ptr
table, const ::openmldb } int ret = RemoveDeletedKey(entry, deleted_index, &tmp_buf); if (ret == 1) { - deleted_key_num++; + snapshot_meta->deleted_key_num++; continue; } else if (ret == 2) { record.reset(tmp_buf.data(), tmp_buf.size()); } if (table->IsExpire(entry)) { - expired_key_num++; + snapshot_meta->expired_key_num++; continue; } status = wh->Write(record); @@ -252,23 +478,25 @@ int MemTableSnapshot::TTLSnapshot(std::shared_ptr
table, const ::openmldb has_error = true; break; } - if ((count + expired_key_num + deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "tackled key num[%lu] total[%lu]", count + expired_key_num, manifest.count()); + if ((snapshot_meta->count + snapshot_meta->expired_key_num + snapshot_meta->deleted_key_num) + % KEY_NUM_DISPLAY == 0) { + PDLOG(INFO, "tackled key num[%lu] total[%lu]", + snapshot_meta->count + snapshot_meta->expired_key_num, manifest.count()); } - count++; + snapshot_meta->count++; } delete seq_file; - if (expired_key_num + count + deleted_key_num != manifest.count()) { - PDLOG(WARNING, - "key num not match! total key num[%lu] load key num[%lu] ttl key " - "num[%lu]", - manifest.count(), count, expired_key_num); + if (snapshot_meta->expired_key_num + snapshot_meta->count + snapshot_meta->deleted_key_num + != manifest.count()) { + PDLOG(WARNING, "key num not match! total key num[%lu] load key num[%lu] ttl key num[%lu]", + manifest.count(), snapshot_meta->count, snapshot_meta->expired_key_num); has_error = true; } if (has_error) { return -1; } - PDLOG(INFO, "load snapshot success. load key num[%lu] ttl key num[%lu]", count, expired_key_num); + PDLOG(INFO, "load snapshot success. load key num[%lu] ttl key num[%lu]", + snapshot_meta->count, snapshot_meta->expired_key_num); return 0; } @@ -280,9 +508,7 @@ uint64_t MemTableSnapshot::CollectDeletedKey(uint64_t end_offset) { std::string buffer; while (true) { if (deleted_keys_.size() >= FLAGS_make_snapshot_max_deleted_keys) { - PDLOG(WARNING, - "deleted_keys map size reach the " - "make_snapshot_max_deleted_keys %u, tid %u pid %u", + PDLOG(WARNING, "deleted_keys map size reach the make_snapshot_max_deleted_keys %u, tid %u pid %u", FLAGS_make_snapshot_max_deleted_keys, tid_, pid_); break; } @@ -313,9 +539,9 @@ uint64_t MemTableSnapshot::CollectDeletedKey(uint64_t end_offset) { PDLOG(WARNING, "no dimesion. tid %u pid %u offset %lu", tid_, pid_, cur_offset); continue; } - std::string combined_key = entry.dimensions(0).key() + "|" + std::to_string(entry.dimensions(0).idx()); - deleted_keys_[combined_key] = cur_offset; + std::string combined_key = absl::StrCat(entry.dimensions(0).key(), "|", entry.dimensions(0).idx()); DEBUGLOG("insert key %s offset %lu. tid %u pid %u", combined_key.c_str(), cur_offset, tid_, pid_); + deleted_keys_.insert_or_assign(std::move(combined_key), cur_offset); } } else if (status.IsEof()) { continue; @@ -325,8 +551,7 @@ uint64_t MemTableSnapshot::CollectDeletedKey(uint64_t end_offset) { // judge end_log_index greater than cur_log_index if (end_log_index >= 0 && end_log_index > cur_log_index) { log_reader.RollRLogFile(); - PDLOG(WARNING, - "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " + PDLOG(WARNING, "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " "end_log_index[%d] cur_offset[%lu]", tid_, pid_, cur_log_index, end_log_index, cur_offset); continue; @@ -353,37 +578,27 @@ int MemTableSnapshot::MakeSnapshot(std::shared_ptr
table, uint64_t& out_o } making_snapshot_.store(true, std::memory_order_release); std::string now_time = ::openmldb::base::GetNowTime(); - std::string snapshot_name = now_time.substr(0, now_time.length() - 2) + ".sdb"; - if (FLAGS_snapshot_compression != "off") { - snapshot_name.append("."); - snapshot_name.append(FLAGS_snapshot_compression); - } - std::string snapshot_name_tmp = snapshot_name + ".tmp"; - std::string full_path = snapshot_path_ + snapshot_name; - std::string tmp_file_path = snapshot_path_ + snapshot_name_tmp; - FILE* fd = fopen(tmp_file_path.c_str(), "ab+"); - if (fd == NULL) { - PDLOG(WARNING, "fail to create file %s", tmp_file_path.c_str()); + MemSnapshotMeta snapshot_meta(GenSnapshotName(), snapshot_path_, FLAGS_snapshot_compression); + auto wh = ::openmldb::log::CreateWriteHandle(FLAGS_snapshot_compression, + snapshot_meta.snapshot_name, snapshot_meta.tmp_file_path); + if (!wh) { + PDLOG(WARNING, "fail to create file %s", snapshot_meta.tmp_file_path.c_str()); making_snapshot_.store(false, std::memory_order_release); return -1; } uint64_t collected_offset = CollectDeletedKey(end_offset); uint64_t start_time = ::baidu::common::timer::now_time(); - WriteHandle* wh = new WriteHandle(FLAGS_snapshot_compression, snapshot_name_tmp, fd); ::openmldb::api::Manifest manifest; bool has_error = false; - uint64_t write_count = 0; - uint64_t expired_key_num = 0; - uint64_t deleted_key_num = 0; - uint64_t last_term = term; + snapshot_meta.term = term; int result = GetLocalManifest(snapshot_path_ + MANIFEST, manifest); if (result == 0) { // filter old snapshot - if (TTLSnapshot(table, manifest, wh, write_count, expired_key_num, deleted_key_num) < 0) { + if (TTLSnapshot(table, manifest, wh, &snapshot_meta) < 0) { has_error = true; } - last_term = manifest.term(); - DEBUGLOG("old manifest term is %lu", last_term); + snapshot_meta.term = manifest.term(); + DEBUGLOG("old manifest term is %lu", snapshot_meta.term); } else if (result < 0) { // parse manifest error has_error = true; @@ -426,29 +641,31 @@ int MemTableSnapshot::MakeSnapshot(std::shared_ptr
table, uint64_t& out_o continue; } if (entry.has_term()) { - last_term = entry.term(); + snapshot_meta.term = entry.term(); } int ret = RemoveDeletedKey(entry, deleted_index, &tmp_buf); if (ret == 1) { - deleted_key_num++; + snapshot_meta.deleted_key_num++; continue; } else if (ret == 2) { record.reset(tmp_buf.data(), tmp_buf.size()); } if (table->IsExpire(entry)) { - expired_key_num++; + snapshot_meta.expired_key_num++; continue; } ::openmldb::log::Status status = wh->Write(record); if (!status.ok()) { - PDLOG(WARNING, "fail to write snapshot. path[%s] status[%s]", tmp_file_path.c_str(), - status.ToString().c_str()); + PDLOG(WARNING, "fail to write snapshot. path[%s] status[%s]", + snapshot_meta.tmp_file_path.c_str(), status.ToString().c_str()); has_error = true; break; } - write_count++; - if ((write_count + expired_key_num + deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "has write key num[%lu] expired key num[%lu]", write_count, expired_key_num); + snapshot_meta.count++; + if ((snapshot_meta.count + snapshot_meta.expired_key_num + snapshot_meta.deleted_key_num) + % KEY_NUM_DISPLAY == 0) { + PDLOG(INFO, "has write key num[%lu] expired key num[%lu]", + snapshot_meta.count, snapshot_meta.expired_key_num); } } else if (status.IsEof()) { continue; @@ -458,10 +675,9 @@ int MemTableSnapshot::MakeSnapshot(std::shared_ptr
table, uint64_t& out_o // judge end_log_index greater than cur_log_index if (end_log_index >= 0 && end_log_index > cur_log_index) { log_reader.RollRLogFile(); - PDLOG(WARNING, - "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " - "end_log_index[%d] cur_offset[%lu]", - tid_, pid_, cur_log_index, end_log_index, cur_offset); + PDLOG(WARNING, "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " + "end_log_index[%d] cur_offset[%lu]", + tid_, pid_, cur_log_index, end_log_index, cur_offset); continue; } DEBUGLOG("has read all record!"); @@ -472,44 +688,28 @@ int MemTableSnapshot::MakeSnapshot(std::shared_ptr
table, uint64_t& out_o break; } } - if (wh != NULL) { - wh->EndLog(); - delete wh; - wh = NULL; - } + deleted_keys_.clear(); + wh->EndLog(); + wh.reset(); int ret = 0; if (has_error) { - unlink(tmp_file_path.c_str()); + unlink(snapshot_meta.tmp_file_path.c_str()); ret = -1; } else { - if (rename(tmp_file_path.c_str(), full_path.c_str()) == 0) { - if (GenManifest(snapshot_name, write_count, cur_offset, last_term) == 0) { - // delete old snapshot - if (manifest.has_name() && manifest.name() != snapshot_name) { - DEBUGLOG("old snapshot[%s] has deleted", manifest.name().c_str()); - unlink((snapshot_path_ + manifest.name()).c_str()); - } - uint64_t consumed = ::baidu::common::timer::now_time() - start_time; - PDLOG(INFO, - "make snapshot[%s] success. update offset from %lu to %lu." - "use %lu second. write key %lu expired key %lu deleted key " - "%lu", - snapshot_name.c_str(), offset_, cur_offset, consumed, write_count, expired_key_num, - deleted_key_num); - offset_ = cur_offset; - out_offset = cur_offset; - } else { - PDLOG(WARNING, "GenManifest failed. delete snapshot file[%s]", full_path.c_str()); - unlink(full_path.c_str()); - ret = -1; - } - } else { - PDLOG(WARNING, "rename[%s] failed", snapshot_name.c_str()); - unlink(tmp_file_path.c_str()); + snapshot_meta.offset = cur_offset; + uint64_t old_offset = offset_; + auto status = WriteSnapshot(snapshot_meta); + if (!status.OK()) { ret = -1; + PDLOG(WARNING, "write snapshot failed. tid %u pid %u msg is %s ", tid_, pid_, status.GetMsg().c_str()); } + uint64_t consumed = ::baidu::common::timer::now_time() - start_time; + PDLOG(INFO, "make snapshot[%s] success. update offset from %lu to %lu." + "use %lu second. write key %lu expired key %lu deleted key %lu", + snapshot_meta.snapshot_name.c_str(), old_offset, snapshot_meta.offset, consumed, + snapshot_meta.count, snapshot_meta.expired_key_num, snapshot_meta.deleted_key_num); + out_offset = snapshot_meta.offset; } - deleted_keys_.clear(); making_snapshot_.store(false, std::memory_order_release); return ret; } @@ -518,7 +718,7 @@ int MemTableSnapshot::RemoveDeletedKey(const ::openmldb::api::LogEntry& entry, c std::string* buffer) { uint64_t cur_offset = entry.log_index(); if (entry.dimensions_size() == 0) { - std::string combined_key = entry.pk() + "|0"; + std::string combined_key = absl::StrCat(entry.pk(), "|0"); auto iter = deleted_keys_.find(combined_key); if (iter != deleted_keys_.end() && cur_offset <= iter->second) { DEBUGLOG("delete key %s offset %lu", entry.pk().c_str(), entry.log_index()); @@ -527,7 +727,7 @@ int MemTableSnapshot::RemoveDeletedKey(const ::openmldb::api::LogEntry& entry, c } else { std::set deleted_pos_set; for (int pos = 0; pos < entry.dimensions_size(); pos++) { - std::string combined_key = entry.dimensions(pos).key() + "|" + std::to_string(entry.dimensions(pos).idx()); + std::string combined_key = absl::StrCat(entry.dimensions(pos).key(), "|", entry.dimensions(pos).idx()); auto iter = deleted_keys_.find(combined_key); if ((iter != deleted_keys_.end() && cur_offset <= iter->second) || deleted_index.count(entry.dimensions(pos).idx())) { @@ -555,24 +755,6 @@ int MemTableSnapshot::RemoveDeletedKey(const ::openmldb::api::LogEntry& entry, c return 0; } -base::Status MemTableSnapshot::GetAllDecoder(std::shared_ptr
table, - std::map* decoder_map) { - if (decoder_map == nullptr) { - return base::Status(base::ReturnCode::kError, "null ptr"); - } - auto schema_map = table->GetAllVersionSchema(); - if (schema_map.empty()) { - return base::Status(base::ReturnCode::kError, "schema map is empty"); - } - decoder_map->clear(); - for (const auto& kv : schema_map) { - if (kv.second) { - decoder_map->emplace(kv.first, codec::RowView(*kv.second)); - } - } - return {}; -} - /** * return code: * -1 : error @@ -590,7 +772,7 @@ int MemTableSnapshot::CheckDeleteAndUpdate(std::shared_ptr
table, openmld // deleted key std::set deleted_pos_set; for (int pos = 0; pos < entry->dimensions_size(); pos++) { - std::string combined_key = entry->dimensions(pos).key() + "|" + std::to_string(entry->dimensions(pos).idx()); + std::string combined_key = absl::StrCat(entry->dimensions(pos).key(), "|", entry->dimensions(pos).idx()); if (deleted_keys_.find(combined_key) != deleted_keys_.end() || !table->GetIndex(entry->dimensions(pos).idx())->IsReady()) { deleted_pos_set.insert(pos); @@ -614,1186 +796,222 @@ int MemTableSnapshot::CheckDeleteAndUpdate(std::shared_ptr
table, openmld return 0; } -base::Status MemTableSnapshot::GetIndexKey(std::shared_ptr
table, - const std::shared_ptr& index, const base::Slice& data, - std::map* decoder_map, std::string* index_key) { - if (table == nullptr || decoder_map == nullptr || index_key == nullptr) { - return base::Status(base::ReturnCode::kError, "null ptr"); +std::string MemTableSnapshot::GenSnapshotName() { + std::string now_time = ::openmldb::base::GetNowTime(); + std::string snapshot_name = now_time.substr(0, now_time.length() - 2) + ".sdb"; + if (FLAGS_snapshot_compression != "off") { + snapshot_name.append("."); + snapshot_name.append(FLAGS_snapshot_compression); + } + return snapshot_name; +} + +::openmldb::base::Status MemTableSnapshot::DecodeData(const std::shared_ptr
& table, + const openmldb::api::LogEntry& entry, + const std::vector& cols, std::vector* row) { + if (row == nullptr) { + return {-1, "row is nullptr"}; + } + row->clear(); + std::string buff; + openmldb::base::Slice data; + if (table->GetCompressType() == openmldb::type::kSnappy) { + snappy::Uncompress(entry.value().data(), entry.value().size(), &buff); + data.reset(buff.data(), buff.size()); + } else { + data.reset(entry.value().data(), entry.value().size()); } const int8_t* raw = reinterpret_cast(data.data()); uint8_t version = openmldb::codec::RowView::GetSchemaVersion(raw); - auto schema = table->GetVersionSchema(version); - auto it = decoder_map->find(version); - if (it == decoder_map->end() || schema == nullptr) { - return base::Status(base::ReturnCode::kError, "schema version does not exist"); + auto decoder = table->GetVersionDecoder(version); + if (decoder == nullptr) { + return ::openmldb::base::Status(-1, "get decoder failed. version is " + std::to_string(version)); } - index_key->clear(); - for (const auto& col : index->GetColumns()) { - if ((int32_t)col.GetId() >= schema->size()) { - return base::Status(base::ReturnCode::kError, "cannot found col"); - } - std::string val; - int ret = it->second.GetStrValue(raw, col.GetId(), &val); - if (ret < 0) { - return base::Status(base::ReturnCode::kError, "decode error"); - } else if (ret == 1) { - val = ::openmldb::codec::NONETOKEN; - } - if (index_key->empty()) { - *index_key = std::move(val); + if (!openmldb::codec::RowCodec::DecodeRow(*decoder, raw, cols, true, row)) { + return ::openmldb::base::Status(-1, "decode failed"); + } + return {}; +} + +::openmldb::base::Status MemTableSnapshot::WriteSnapshot(const MemSnapshotMeta& snapshot_meta) { + ::openmldb::api::Manifest old_manifest; + if (GetLocalManifest(snapshot_path_ + MANIFEST, old_manifest) < 0) { + return {-1, absl::StrCat("get old manifest failed. snapshot path is ", snapshot_path_)}; + } + if (rename(snapshot_meta.tmp_file_path.c_str(), snapshot_meta.full_path.c_str()) == 0) { + if (GenManifest(snapshot_meta) == 0) { + // delete old snapshot + if (old_manifest.has_name() && old_manifest.name() != snapshot_meta.snapshot_name) { + DEBUGLOG("old snapshot[%s] has deleted", old_manifest.name().c_str()); + unlink((snapshot_path_ + old_manifest.name()).c_str()); + } + offset_ = snapshot_meta.offset; } else { - *index_key += "|" + val; + unlink(snapshot_meta.full_path.c_str()); + return {-1, absl::StrCat("GenManifest failed. delete snapshot file ", snapshot_meta.full_path)}; } + } else { + unlink(snapshot_meta.tmp_file_path.c_str()); + return {-1, absl::StrCat("rename ", snapshot_meta.snapshot_name, " failed")}; } return {}; } -base::Status MemTableSnapshot::ExtractIndexFromSnapshot(std::shared_ptr
table, - const ::openmldb::api::Manifest& manifest, WriteHandle* wh, - const std::vector<::openmldb::common::ColumnKey>& add_indexs, uint32_t partition_num, - uint64_t* count, uint64_t* expired_key_num, uint64_t* deleted_key_num) { - if (wh == nullptr || count == nullptr || expired_key_num == nullptr || deleted_key_num == nullptr) { - return base::Status(base::ReturnCode::kError, "null ptr"); - } +::openmldb::base::Status MemTableSnapshot::ExtractIndexData(const std::shared_ptr
& table, + const std::vector<::openmldb::common::ColumnKey>& add_indexs, + const std::vector>& whs, + uint64_t offset, bool dump_data) { uint32_t tid = table->GetId(); uint32_t pid = table->GetPid(); - std::map decoder_map; - auto ret = GetAllDecoder(table, &decoder_map); - if (!ret.OK()) { - return ret; + if (making_snapshot_.exchange(true, std::memory_order_consume)) { + PDLOG(INFO, "snapshot is doing now. tid %u, pid %u", tid, pid); + return {-1, "snapshot is doing now"}; } - std::vector> index_vec; - for (const auto& index : add_indexs) { - auto index_def = table->GetIndex(index.index_name()); - if (!index_def) { - return base::Status(base::ReturnCode::kError, "fail to get index " + index.index_name()); - } - index_vec.push_back(index_def); + absl::Cleanup clean = [this]() { + this->making_snapshot_.store(false, std::memory_order_release); + deleted_keys_.clear(); + }; + TableIndexInfo table_index_info(*(table->GetTableMeta()), add_indexs); + if (!table_index_info.Init()) { + return {-1, "parse TableIndexInfo failed"}; } - std::string full_path = snapshot_path_ + manifest.name(); - FILE* fd = fopen(full_path.c_str(), "rb"); - if (fd == NULL) { - PDLOG(WARNING, "fail to open path %s for error %s", full_path.c_str(), strerror(errno)); - return base::Status(base::ReturnCode::kError, "fail to open file"); + MemSnapshotMeta snapshot_meta(GenSnapshotName(), snapshot_path_, FLAGS_snapshot_compression); + auto wh = ::openmldb::log::CreateWriteHandle(FLAGS_snapshot_compression, + snapshot_meta.snapshot_name, snapshot_meta.tmp_file_path); + if (!wh) { + return {-1, "create WriteHandle failed"}; } - ::openmldb::log::SequentialFile* seq_file = ::openmldb::log::NewSeqFile(manifest.name(), fd); - bool compressed = IsCompressed(full_path); - ::openmldb::log::Reader reader(seq_file, NULL, false, 0, compressed); - std::string buffer; - ::openmldb::api::LogEntry entry; - bool has_error = false; - uint64_t extract_count = 0; - uint64_t write_count = 0; - DLOG(INFO) << "extract index data from snapshot"; - while (true) { - ::openmldb::base::Slice record; - ::openmldb::log::Status status = reader.ReadRecord(&record, &buffer); - if (status.IsEof()) { - break; + CollectDeletedKey(offset); + + auto data_reader = DataReader::CreateDataReader(snapshot_path_, log_part_, log_path_, + DataReaderType::kSnapshotAndBinlog, offset); + if (!data_reader) { + return {-1, "create DataReader failed"}; + } + std::string buff; + ::openmldb::base::Status status; + uint64_t read_cnt = 0; + uint64_t dump_cnt = 0; + uint64_t extract_cnt = 0; + uint64_t cur_offset = 0; + while (data_reader->HasNext()) { + auto& entry = data_reader->GetValue(); + read_cnt++; + cur_offset = entry.log_index(); + if (entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete) { + continue; } - if (!status.ok()) { - PDLOG(WARNING, "fail to read record for tid %u, pid %u with error %s", - tid_, pid_, status.ToString().c_str()); - has_error = true; - break; + if (entry.has_term()) { + snapshot_meta.term = entry.term(); } - if (!entry.ParseFromString(record.ToString())) { - PDLOG(WARNING, "fail parse record for tid %u, pid %u with value %s", - tid_, pid_, ::openmldb::base::DebugString(record.ToString()).c_str()); - has_error = true; - break; + if (table->IsExpire(entry)) { + snapshot_meta.expired_key_num++; + continue; } - std::string tmp_buf; if (!deleted_keys_.empty()) { - int check_ret = CheckDeleteAndUpdate(table, &entry); - if (check_ret == 1) { - (*deleted_key_num)++; + int ret = CheckDeleteAndUpdate(table, &entry); + if (ret == 1) { + snapshot_meta.deleted_key_num++; continue; - } else if (check_ret == 2) { - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); } } - // delete timeout key - if (table->IsExpire(entry)) { - (*expired_key_num)++; - continue; + bool has_main_index = false; + for (int pos = 0; pos < entry.dimensions_size(); pos++) { + if (entry.dimensions(pos).idx() == 0) { + if (!deleted_keys_.empty()) { + std::string combined_key = absl::StrCat(entry.dimensions(pos).key(), "|0"); + if (deleted_keys_.find(combined_key) == deleted_keys_.end()) { + has_main_index = true; + } + } else { + has_main_index = true; + } + break; + } } - if (!(entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete)) { - std::string buff; - openmldb::base::Slice data; - if (table->GetCompressType() == openmldb::type::kSnappy) { - snappy::Uncompress(entry.value().data(), entry.value().size(), &buff); - data.reset(buff.data(), buff.size()); - } else { - data.reset(entry.value().data(), entry.value().size()); + if (!has_main_index) { + if (!wh->Write(::openmldb::base::Slice(data_reader->GetStrValue())).ok()) { + status = {-1, "fail to write snapshot"}; + break; } - std::map add_key_idx_map; - for (const auto& index : index_vec) { - std::string index_key; - auto ret = GetIndexKey(table, index, data, &decoder_map, &index_key); - if (ret.OK() && !index_key.empty()) { - uint32_t index_pid = ::openmldb::base::hash64(index_key) % partition_num; - if (index_pid == pid) { - add_key_idx_map.emplace(index->GetId(), index_key); - } + continue; + } + std::vector index_row; + auto staus = DecodeData(table, entry, table_index_info.GetAllIndexCols(), &index_row); + std::map> dimension_map; + for (auto idx : table_index_info.GetAddIndexIdx()) { + std::string index_key; + for (auto pos : table_index_info.GetRealIndexCols(idx)) { + if (index_key.empty()) { + index_key = index_row.at(pos); + } else { + absl::StrAppend(&index_key, "|", index_row.at(pos)); } } - if (!add_key_idx_map.empty()) { - for (const auto& kv : add_key_idx_map) { - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_idx(kv.first); - dim->set_key(kv.second); + ::openmldb::api::Dimension dim; + dim.set_idx(idx); + dim.set_key(index_key); + uint32_t index_pid = ::openmldb::base::hash64(index_key) % whs.size(); + auto iter = dimension_map.emplace(index_pid, std::vector<::openmldb::api::Dimension>()).first; + iter->second.push_back(dim); + } + std::string tmp_buf; + auto iter = dimension_map.find(pid); + if (iter != dimension_map.end()) { + for (const auto& dim : iter->second) { + ::openmldb::api::Dimension* new_dim = entry.add_dimensions(); + new_dim->CopyFrom(dim); + } + entry.SerializeToString(&tmp_buf); + if (!wh->Write(::openmldb::base::Slice(tmp_buf)).ok()) { + status = {-1, "fail to write snapshot"}; + break; + } + entry.clear_dimensions(); + for (const auto& dim : iter->second) { + ::openmldb::api::Dimension* new_dim = entry.add_dimensions(); + new_dim->CopyFrom(dim); + } + DLOG(INFO) << "extract: dim size " << entry.dimensions_size() << " key " << entry.dimensions(0).key(); + extract_cnt++; + table->Put(entry); + } + if (dump_data) { + for (const auto& kv : dimension_map) { + if (kv.first == pid) { + continue; } - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); entry.clear_dimensions(); - for (const auto& kv : add_key_idx_map) { - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_idx(kv.first); - dim->set_key(kv.second); + for (const auto& dim : kv.second) { + ::openmldb::api::Dimension* new_dim = entry.add_dimensions(); + new_dim->CopyFrom(dim); + } + tmp_buf.clear(); + entry.SerializeToString(&tmp_buf); + if (!whs[kv.first]->Write(::openmldb::base::Slice(tmp_buf)).ok()) { + status = {-1, "fail to dump index entry"}; + break; } - table->Put(entry); - extract_count++; + DLOG(INFO) << "dump " << pid << " dim size " << entry.dimensions_size() + << " key " << entry.dimensions(0).key(); + dump_cnt++; } } - status = wh->Write(record); - if (!status.ok()) { - PDLOG(WARNING, "fail to extract index from snapshot. status[%s] tid[%u] pid[%u]", - status.ToString().c_str(), tid, pid); - has_error = true; + if (!status.OK()) { break; } - if ((*count + *expired_key_num + *deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "tackled key num[%lu] total[%lu] tid[%u] pid[%u]", - *count + *expired_key_num, manifest.count(), tid, pid); - } - (*count)++; - } - delete seq_file; - if (*expired_key_num + write_count + *deleted_key_num != manifest.count()) { - PDLOG(WARNING, "key num not match! total key[%lu] load key[%lu] ttl key[%lu] delete key [%lu], tid %u pid %u", - manifest.count(), *count, *expired_key_num, *deleted_key_num, tid, pid); - has_error = true; } - if (has_error) { - return base::Status(base::ReturnCode::kError, "extract error"); - } - PDLOG(INFO, "extract index from snapshot success! extract count [%lu], tid %u pid %u", extract_count, tid, pid); - return {}; -} - -int MemTableSnapshot::ExtractIndexFromSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, - WriteHandle* wh, const ::openmldb::common::ColumnKey& column_key, - uint32_t idx, uint32_t partition_num, uint32_t max_idx, - const std::vector& index_cols, uint64_t& count, - uint64_t& expired_key_num, uint64_t& deleted_key_num) { - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - std::string full_path = snapshot_path_ + manifest.name(); - FILE* fd = fopen(full_path.c_str(), "rb"); - if (fd == NULL) { - PDLOG(WARNING, "fail to open path %s for error %s", full_path.c_str(), strerror(errno)); - return -1; + LOG(INFO) << "read cnt " << read_cnt << " dump cnt " << dump_cnt << " extract cnt " << extract_cnt + << " tid " << tid << " pid " << pid; + wh->EndLog(); + wh.reset(); + if (!status.OK()) { + unlink(snapshot_meta.tmp_file_path.c_str()); + } else { + snapshot_meta.offset = std::max(cur_offset, offset_); + WriteSnapshot(snapshot_meta); } - ::openmldb::log::SequentialFile* seq_file = ::openmldb::log::NewSeqFile(manifest.name(), fd); - bool compressed = IsCompressed(full_path); - ::openmldb::log::Reader reader(seq_file, NULL, false, 0, compressed); - std::string buffer; - ::openmldb::api::LogEntry entry; - bool has_error = false; - uint64_t extract_count = 0; - uint64_t schame_size_less_count = 0; - uint64_t other_error_count = 0; - DLOG(INFO) << "extract index data from snapshot"; - while (true) { - ::openmldb::base::Slice record; - ::openmldb::log::Status status = reader.ReadRecord(&record, &buffer); - if (status.IsEof()) { - break; - } - if (!status.ok()) { - PDLOG(WARNING, "fail to read record for tid %u, pid %u with error %s", tid_, pid_, - status.ToString().c_str()); - has_error = true; - break; - } - if (!entry.ParseFromString(record.ToString())) { - PDLOG(WARNING, "fail parse record for tid %u, pid %u with value %s", tid_, pid_, - ::openmldb::base::DebugString(record.ToString()).c_str()); - has_error = true; - break; - } - // deleted key - std::string tmp_buf; - if (entry.dimensions_size() == 0) { - std::string combined_key = entry.pk() + "|0"; - if (deleted_keys_.find(combined_key) != deleted_keys_.end()) { - deleted_key_num++; - continue; - } - } else { - std::set deleted_pos_set; - for (int pos = 0; pos < entry.dimensions_size(); pos++) { - std::string combined_key = - entry.dimensions(pos).key() + "|" + std::to_string(entry.dimensions(pos).idx()); - if (deleted_keys_.find(combined_key) != deleted_keys_.end() || - !table->GetIndex(entry.dimensions(pos).idx())->IsReady()) { - deleted_pos_set.insert(pos); - } - } - if (!deleted_pos_set.empty()) { - if ((int)deleted_pos_set.size() == // NOLINT - entry.dimensions_size()) { - deleted_key_num++; - continue; - } else { - ::openmldb::api::LogEntry tmp_entry(entry); - entry.clear_dimensions(); - for (int pos = 0; pos < tmp_entry.dimensions_size(); pos++) { - if (deleted_pos_set.find(pos) == deleted_pos_set.end()) { - ::openmldb::api::Dimension* dimension = entry.add_dimensions(); - dimension->CopyFrom(tmp_entry.dimensions(pos)); - } - } - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); - } - } - } - // delete timeout key - if (table->IsExpire(entry)) { - expired_key_num++; - continue; - } - if (!(entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete)) { - // new column_key - std::vector row; - int ret = DecodeData(table, entry, max_idx, row); - if (ret == 2) { - count++; - wh->Write(record); - continue; - } else if (ret != 0) { - DLOG(INFO) << "skip current data"; - other_error_count++; - continue; - } - std::string cur_key; - for (uint32_t i : index_cols) { - if (cur_key.empty()) { - cur_key = row[i]; - } else { - cur_key += "|" + row[i]; - } - } - if (cur_key.empty()) { - other_error_count++; - DLOG(INFO) << "skip empty key"; - continue; - } - uint32_t index_pid = ::openmldb::base::hash64(cur_key) % partition_num; - // update entry and write entry into memory - if (index_pid == pid) { - if (entry.dimensions_size() == 1 && entry.dimensions(0).idx() == idx) { - other_error_count++; - DLOG(INFO) << "skip not default key " << cur_key; - continue; - } - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_key(cur_key); - dim->set_idx(idx); - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); - entry.clear_dimensions(); - dim = entry.add_dimensions(); - dim->set_key(cur_key); - dim->set_idx(idx); - table->Put(entry); - extract_count++; - } - } - status = wh->Write(record); - if (!status.ok()) { - PDLOG(WARNING, - "fail to extract index from snapshot. status[%s] tid[%u] " - "pid[%u]", - status.ToString().c_str(), tid, pid); - has_error = true; - break; - } - if ((count + expired_key_num + deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "tackled key num[%lu] total[%lu] tid[%u] pid[%u]", count + expired_key_num, manifest.count(), - tid, pid); - } - count++; - } - delete seq_file; - if (expired_key_num + count + deleted_key_num + schame_size_less_count + other_error_count != manifest.count()) { - LOG(WARNING) << "key num not match ! total key num[" << manifest.count() << "] load key num[" << count - << "] ttl key num[" << expired_key_num << "] schema size less num[" << schame_size_less_count - << "] other error count[" << other_error_count << "]" - << " tid[" << tid << "] pid[" << pid << "]"; - has_error = true; - } - if (has_error) { - return -1; - } - LOG(INFO) << "extract index from snapshot success. extract key num[" << extract_count << "] load key num[" << count - << "] ttl key num[" << expired_key_num << "] schema size less num[" << schame_size_less_count - << "] other error count[" << other_error_count << "]" - << " tid[" << tid << "] pid[" << pid << "]"; - return 0; -} - -std::string MemTableSnapshot::GenSnapshotName() { - std::string now_time = ::openmldb::base::GetNowTime(); - std::string snapshot_name = now_time.substr(0, now_time.length() - 2) + ".sdb"; - if (FLAGS_snapshot_compression != "off") { - snapshot_name.append("."); - snapshot_name.append(FLAGS_snapshot_compression); - } - return snapshot_name; -} - -base::Status MemTableSnapshot::ExtractIndexFromBinlog(std::shared_ptr
table, - WriteHandle* wh, const std::vector<::openmldb::common::ColumnKey>& add_indexs, - uint64_t collected_offset, uint32_t partition_num, uint64_t* offset, - uint64_t* last_term, uint64_t* count, uint64_t* expired_key_num, uint64_t* deleted_key_num) { - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - std::map decoder_map; - auto ret = GetAllDecoder(table, &decoder_map); - if (!ret.OK()) { - return ret; - } - std::vector> index_vec; - for (const auto& index : add_indexs) { - auto index_def = table->GetIndex(index.index_name()); - if (!index_def) { - return base::Status(base::ReturnCode::kError, "fail to get index " + index.index_name()); - } - index_vec.push_back(index_def); - } - ::openmldb::log::LogReader log_reader(log_part_, log_path_, false); - log_reader.SetOffset(offset_); - *offset = offset_; - std::string buffer; - uint64_t extract_count = 0; - DLOG(INFO) << "extract index data from binlog"; - while (*offset < collected_offset) { - buffer.clear(); - ::openmldb::base::Slice record; - ::openmldb::log::Status status = log_reader.ReadNextRecord(&record, &buffer); - if (status.ok()) { - ::openmldb::api::LogEntry entry; - if (!entry.ParseFromString(record.ToString())) { - LOG(WARNING) << "fail to parse LogEntry. record " << openmldb::base::DebugString(record.ToString()) - << " size " << record.ToString().size() << " tid " << tid << " pid " << pid; - return base::Status(base::ReturnCode::kError, "parse error"); - } - if (entry.log_index() <= *offset) { - continue; - } - if (*offset + 1 != entry.log_index()) { - LOG(WARNING) << "log missing expect offset " << *offset + 1 << " but " << entry.log_index() - << ". tid " << tid << " pid " << pid; - continue; - } - *offset = entry.log_index(); - if (entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete) { - continue; - } - if (entry.has_term()) { - *last_term = entry.term(); - } - std::string tmp_buf; - if (!deleted_keys_.empty()) { - int check_ret = CheckDeleteAndUpdate(table, &entry); - if (check_ret == 1) { - (*deleted_key_num)++; - continue; - } else if (check_ret == 2) { - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); - } - } - if (table->IsExpire(entry)) { - (*expired_key_num)++; - continue; - } - if (!(entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete)) { - std::string buff; - openmldb::base::Slice data; - if (table->GetCompressType() == openmldb::type::kSnappy) { - snappy::Uncompress(entry.value().data(), entry.value().size(), &buff); - data.reset(buff.data(), buff.size()); - } else { - data.reset(entry.value().data(), entry.value().size()); - } - std::map add_key_idx_map; - for (const auto& index : index_vec) { - std::string index_key; - auto ret = GetIndexKey(table, index, data, &decoder_map, &index_key); - if (ret.OK() && !index_key.empty()) { - uint32_t index_pid = ::openmldb::base::hash64(index_key) % partition_num; - if (index_pid == pid) { - add_key_idx_map.emplace(index->GetId(), index_key); - } - } - } - if (!add_key_idx_map.empty()) { - for (const auto& kv : add_key_idx_map) { - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_idx(kv.first); - dim->set_key(kv.second); - } - entry.SerializeToString(&tmp_buf); - entry.clear_dimensions(); - for (const auto& kv : add_key_idx_map) { - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_idx(kv.first); - dim->set_key(kv.second); - } - table->Put(entry); - extract_count++; - record.reset(tmp_buf.data(), tmp_buf.size()); - } - } - ::openmldb::log::Status status = wh->Write(record); - if (!status.ok()) { - PDLOG(WARNING, "fail to write snapshot. tid[%u] pid[%u] status[%s]", - tid, pid, status.ToString().c_str()); - return base::Status(base::ReturnCode::kError, "fail to write snapshot"); - } - (*count)++; - if ((*count + *expired_key_num + *deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "has write key num[%lu] expired key num[%lu]", *count, *expired_key_num); - } - } else if (status.IsEof()) { - continue; - } else if (status.IsWaitRecord()) { - int end_log_index = log_reader.GetEndLogIndex(); - int cur_log_index = log_reader.GetLogIndex(); - // judge end_log_index greater than cur_log_index - if (end_log_index >= 0 && end_log_index > cur_log_index) { - log_reader.RollRLogFile(); - PDLOG(WARNING, "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " - "end_log_index[%d] cur_offset[%lu]", - tid, pid, cur_log_index, end_log_index, *offset); - continue; - } - DEBUGLOG("has read all record!"); - return {}; - } else { - PDLOG(WARNING, "fail to get record. status is %s", status.ToString().c_str()); - return base::Status(base::ReturnCode::kError, "fail to get record"); - } - } - return {}; -} - -int MemTableSnapshot::ExtractIndexData(std::shared_ptr
table, - const std::vector<::openmldb::common::ColumnKey>& indexs, - uint32_t partition_num, uint64_t* out_offset) { - if (out_offset == NULL) { - return -1; - } - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - if (making_snapshot_.exchange(true, std::memory_order_consume)) { - PDLOG(INFO, "snapshot is doing now. tid %u, pid %u", tid, pid); - return -1; - } - std::string snapshot_name = GenSnapshotName(); - std::string snapshot_name_tmp = snapshot_name + ".tmp"; - std::string full_path = snapshot_path_ + snapshot_name; - std::string tmp_file_path = snapshot_path_ + snapshot_name_tmp; - FILE* fd = fopen(tmp_file_path.c_str(), "ab+"); - if (fd == NULL) { - PDLOG(WARNING, "fail to create file %s. tid %u, pid %u", tmp_file_path.c_str(), tid, pid); - making_snapshot_.store(false, std::memory_order_release); - return -1; - } - uint64_t collected_offset = CollectDeletedKey(0); - uint64_t start_time = ::baidu::common::timer::now_time(); - WriteHandle* wh = new WriteHandle(FLAGS_snapshot_compression, snapshot_name_tmp, fd); - ::openmldb::api::Manifest manifest; - bool has_error = false; - uint64_t write_count = 0; - uint64_t expired_key_num = 0; - uint64_t deleted_key_num = 0; - uint64_t last_term = 0; - - int result = GetLocalManifest(snapshot_path_ + MANIFEST, manifest); - if (result == 0) { - DLOG(INFO) << "begin extract index data from snapshot"; - if (!ExtractIndexFromSnapshot(table, manifest, wh, indexs, partition_num, - &write_count, &expired_key_num, &deleted_key_num).OK()) { - has_error = true; - } - last_term = manifest.term(); - DLOG(INFO) << "old manifest term is " << last_term; - } else if (result < 0) { - // parse manifest error - has_error = true; - } - uint64_t cur_offset = offset_; - if (!has_error) { - auto ret = ExtractIndexFromBinlog(table, wh, indexs, collected_offset, partition_num, - &cur_offset, &last_term, &write_count, &expired_key_num, &deleted_key_num); - if (!ret.OK()) { - LOG(WARNING) << ret.msg; - has_error = true; - } - } - - if (wh != NULL) { - wh->EndLog(); - delete wh; - wh = NULL; - } - int ret = 0; - if (has_error) { - unlink(tmp_file_path.c_str()); - ret = -1; - } else { - if (rename(tmp_file_path.c_str(), full_path.c_str()) == 0) { - if (GenManifest(snapshot_name, write_count, cur_offset, last_term) == 0) { - // delete old snapshot - if (manifest.has_name() && manifest.name() != snapshot_name) { - DEBUGLOG("old snapshot[%s] has deleted", manifest.name().c_str()); - unlink((snapshot_path_ + manifest.name()).c_str()); - } - uint64_t consumed = ::baidu::common::timer::now_time() - start_time; - PDLOG(INFO, - "make snapshot[%s] success. update offset from %lu to %lu." - "use %lu second. write key %lu expired key %lu deleted key %lu", - snapshot_name.c_str(), offset_, cur_offset, consumed, write_count, expired_key_num, - deleted_key_num); - offset_ = cur_offset; - *out_offset = cur_offset; - } else { - PDLOG(WARNING, "GenManifest failed. delete snapshot file[%s]", full_path.c_str()); - unlink(full_path.c_str()); - ret = -1; - } - } else { - PDLOG(WARNING, "rename[%s] failed", snapshot_name.c_str()); - unlink(tmp_file_path.c_str()); - ret = -1; - } - } - deleted_keys_.clear(); - making_snapshot_.store(false, std::memory_order_release); - return ret; -} - -int MemTableSnapshot::ExtractIndexData(std::shared_ptr
table, const ::openmldb::common::ColumnKey& column_key, - uint32_t idx, uint32_t partition_num, uint64_t& out_offset) { - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - if (making_snapshot_.exchange(true, std::memory_order_consume)) { - PDLOG(INFO, "snapshot is doing now. tid %u, pid %u", tid, pid); - return -1; - } - std::string now_time = ::openmldb::base::GetNowTime(); - std::string snapshot_name = now_time.substr(0, now_time.length() - 2) + ".sdb"; - if (FLAGS_snapshot_compression != "off") { - snapshot_name.append("."); - snapshot_name.append(FLAGS_snapshot_compression); - } - std::string snapshot_name_tmp = snapshot_name + ".tmp"; - std::string full_path = snapshot_path_ + snapshot_name; - std::string tmp_file_path = snapshot_path_ + snapshot_name_tmp; - FILE* fd = fopen(tmp_file_path.c_str(), "ab+"); - if (fd == NULL) { - PDLOG(WARNING, "fail to create file %s. tid %u, pid %u", tmp_file_path.c_str(), tid, pid); - making_snapshot_.store(false, std::memory_order_release); - return -1; - } - uint64_t collected_offset = CollectDeletedKey(0); - uint64_t start_time = ::baidu::common::timer::now_time(); - WriteHandle* wh = new WriteHandle(FLAGS_snapshot_compression, snapshot_name_tmp, fd); - ::openmldb::api::Manifest manifest; - bool has_error = false; - uint64_t write_count = 0; - uint64_t expired_key_num = 0; - uint64_t deleted_key_num = 0; - uint64_t last_term = 0; - - std::map column_desc_map; - auto table_meta = table->GetTableMeta(); - for (int32_t i = 0; i < table_meta->column_desc_size(); ++i) { - column_desc_map.insert(std::make_pair(table_meta->column_desc(i).name(), i)); - } - uint32_t base_size = table_meta->column_desc_size(); - for (int32_t i = 0; i < table_meta->added_column_desc_size(); ++i) { - column_desc_map.insert(std::make_pair(table_meta->added_column_desc(i).name(), i + base_size)); - } - std::vector index_cols; - uint32_t max_idx = 0; - // get columns in new column_key - for (const auto& name : column_key.col_name()) { - if (column_desc_map.find(name) != column_desc_map.end()) { - uint32_t idx = column_desc_map[name]; - index_cols.push_back(idx); - if (idx > max_idx) { - max_idx = idx; - } - } else { - PDLOG(WARNING, "fail to find column_desc %s. tid %u, pid %u", name.c_str(), tid, pid); - making_snapshot_.store(false, std::memory_order_release); - return -1; - } - } - - int result = GetLocalManifest(snapshot_path_ + MANIFEST, manifest); - if (result == 0) { - DLOG(INFO) << "begin extract index data from snapshot"; - if (ExtractIndexFromSnapshot(table, manifest, wh, column_key, idx, partition_num, max_idx, index_cols, - write_count, expired_key_num, deleted_key_num) < 0) { - has_error = true; - } - last_term = manifest.term(); - DLOG(INFO) << "old manifest term is " << last_term; - } else if (result < 0) { - // parse manifest error - has_error = true; - } - - ::openmldb::log::LogReader log_reader(log_part_, log_path_, false); - log_reader.SetOffset(offset_); - uint64_t cur_offset = offset_; - std::string buffer; - uint64_t extract_count = 0; - DLOG(INFO) << "extract index data from binlog"; - while (!has_error && cur_offset < collected_offset) { - buffer.clear(); - ::openmldb::base::Slice record; - ::openmldb::log::Status status = log_reader.ReadNextRecord(&record, &buffer); - if (status.ok()) { - ::openmldb::api::LogEntry entry; - if (!entry.ParseFromString(record.ToString())) { - LOG(WARNING) << "fail to parse LogEntry. record " << openmldb::base::DebugString(record.ToString()) - << " size " << record.ToString().size() << " tid " << tid << " pid " << pid; - has_error = true; - break; - } - if (entry.log_index() <= cur_offset) { - continue; - } - if (cur_offset + 1 != entry.log_index()) { - LOG(WARNING) << "log missing expect offset " << cur_offset + 1 << " but " << entry.log_index() - << ". tid " << tid << " pid " << pid; - continue; - } - cur_offset = entry.log_index(); - if (entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete) { - continue; - } - if (entry.has_term()) { - last_term = entry.term(); - } - std::string tmp_buf; - if (entry.dimensions_size() == 0) { - std::string combined_key = entry.pk() + "|0"; - auto iter = deleted_keys_.find(combined_key); - if (iter != deleted_keys_.end() && cur_offset <= iter->second) { - DEBUGLOG("delete key %s offset %lu", entry.pk().c_str(), entry.log_index()); - deleted_key_num++; - continue; - } - } else { - std::set deleted_pos_set; - for (int pos = 0; pos < entry.dimensions_size(); pos++) { - std::string combined_key = - entry.dimensions(pos).key() + "|" + std::to_string(entry.dimensions(pos).idx()); - auto iter = deleted_keys_.find(combined_key); - if ((iter != deleted_keys_.end() && cur_offset <= iter->second) || - !table->GetIndex(entry.dimensions(pos).idx())->IsReady()) { - deleted_pos_set.insert(pos); - } - } - if (!deleted_pos_set.empty()) { - if ((int)deleted_pos_set.size() == entry.dimensions_size()) { // NOLINT - deleted_key_num++; - continue; - } else { - ::openmldb::api::LogEntry tmp_entry(entry); - entry.clear_dimensions(); - for (int pos = 0; pos < tmp_entry.dimensions_size(); pos++) { - if (deleted_pos_set.find(pos) == deleted_pos_set.end()) { - ::openmldb::api::Dimension* dimension = entry.add_dimensions(); - dimension->CopyFrom(tmp_entry.dimensions(pos)); - } - } - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); - } - } - } - if (table->IsExpire(entry)) { - expired_key_num++; - continue; - } - if (!(entry.has_method_type() && entry.method_type() == ::openmldb::api::MethodType::kDelete)) { - // new column_key - std::vector row; - int ret = DecodeData(table, entry, max_idx, row); - if (ret == 2) { - wh->Write(record); - write_count++; - continue; - } else if (ret != 0) { - DLOG(INFO) << "skip current data"; - continue; - } - std::string cur_key; - for (uint32_t i : index_cols) { - if (cur_key.empty()) { - cur_key = row[i]; - } else { - cur_key += "|" + row[i]; - } - } - if (cur_key.empty()) { - DLOG(INFO) << "skip empty key"; - continue; - } - uint32_t index_pid = ::openmldb::base::hash64(cur_key) % partition_num; - // update entry and write entry into memory - if (index_pid == pid) { - if (entry.dimensions_size() == 1 && entry.dimensions(0).idx() == idx) { - DLOG(INFO) << "skip not default key " << cur_key; - continue; - } - ::openmldb::api::Dimension* dim = entry.add_dimensions(); - dim->set_key(cur_key); - dim->set_idx(idx); - entry.SerializeToString(&tmp_buf); - record.reset(tmp_buf.data(), tmp_buf.size()); - entry.clear_dimensions(); - dim = entry.add_dimensions(); - dim->set_key(cur_key); - dim->set_idx(idx); - table->Put(entry); - extract_count++; - } - } - ::openmldb::log::Status status = wh->Write(record); - if (!status.ok()) { - PDLOG(WARNING, "fail to write snapshot. path[%s] status[%s]", tmp_file_path.c_str(), - status.ToString().c_str()); - has_error = true; - break; - } - write_count++; - if ((write_count + expired_key_num + deleted_key_num) % KEY_NUM_DISPLAY == 0) { - PDLOG(INFO, "has write key num[%lu] expired key num[%lu]", write_count, expired_key_num); - } - } else if (status.IsEof()) { - continue; - } else if (status.IsWaitRecord()) { - int end_log_index = log_reader.GetEndLogIndex(); - int cur_log_index = log_reader.GetLogIndex(); - // judge end_log_index greater than cur_log_index - if (end_log_index >= 0 && end_log_index > cur_log_index) { - log_reader.RollRLogFile(); - PDLOG(WARNING, - "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " - "end_log_index[%d] cur_offset[%lu]", - tid_, pid_, cur_log_index, end_log_index, cur_offset); - continue; - } - DEBUGLOG("has read all record!"); - break; - } else { - PDLOG(WARNING, "fail to get record. status is %s", status.ToString().c_str()); - has_error = true; - break; - } - } - if (wh != NULL) { - wh->EndLog(); - delete wh; - wh = NULL; - } - int ret = 0; - if (has_error) { - unlink(tmp_file_path.c_str()); - ret = -1; - } else { - if (rename(tmp_file_path.c_str(), full_path.c_str()) == 0) { - if (GenManifest(snapshot_name, write_count, cur_offset, last_term) == 0) { - // delete old snapshot - if (manifest.has_name() && manifest.name() != snapshot_name) { - DEBUGLOG("old snapshot[%s] has deleted", manifest.name().c_str()); - unlink((snapshot_path_ + manifest.name()).c_str()); - } - uint64_t consumed = ::baidu::common::timer::now_time() - start_time; - PDLOG(INFO, - "make snapshot[%s] success. update offset from %lu to %lu." - "use %lu second. write key %lu expired key %lu deleted key " - "%lu", - snapshot_name.c_str(), offset_, cur_offset, consumed, write_count, expired_key_num, - deleted_key_num); - offset_ = cur_offset; - out_offset = cur_offset; - } else { - PDLOG(WARNING, "GenManifest failed. delete snapshot file[%s]", full_path.c_str()); - unlink(full_path.c_str()); - ret = -1; - } - } else { - PDLOG(WARNING, "rename[%s] failed", snapshot_name.c_str()); - unlink(tmp_file_path.c_str()); - ret = -1; - } - } - deleted_keys_.clear(); - making_snapshot_.store(false, std::memory_order_release); - return ret; -} - -bool MemTableSnapshot::PackNewIndexEntry(std::shared_ptr
table, - const std::vector>& index_cols, uint32_t max_idx, - uint32_t idx, uint32_t partition_num, ::openmldb::api::LogEntry* entry, - uint32_t* index_pid) { - if (entry->dimensions_size() == 0) { - std::string combined_key = entry->pk() + "|0"; - if (deleted_keys_.find(combined_key) != deleted_keys_.end()) { - return false; - } - } else { - bool has_main_index = false; - for (int pos = 0; pos < entry->dimensions_size(); pos++) { - if (entry->dimensions(pos).idx() == 0) { - std::string combined_key = entry->dimensions(pos).key() + "|0"; - if (deleted_keys_.find(combined_key) == deleted_keys_.end()) { - has_main_index = true; - } - break; - } - } - if (!has_main_index) { - return false; - } - } - std::vector row; - int ret = DecodeData(table, *entry, max_idx, row); - if (ret != 0 && ret != 2) { - DLOG(INFO) << "pack fail code is " << ret; - return false; - } - std::string key; - std::set pid_set; - for (uint32_t i = 0; i < index_cols.size(); ++i) { - std::string cur_key; - bool skip_calc = false; - for (uint32_t j : index_cols[i]) { - if (j >= row.size()) { - skip_calc = true; - break; - } - if (cur_key.empty()) { - cur_key = row[j]; - } else { - cur_key += "|" + row[j]; - } - } - if (skip_calc) { - continue; - } - if (cur_key.empty()) { - DLOG(INFO) << "key is emptry"; - continue; - } - - uint32_t pid = ::openmldb::base::hash64(cur_key) % partition_num; - if (i < index_cols.size() - 1) { - pid_set.insert(pid); - } else { - *index_pid = pid; - key = cur_key; - } - } - DLOG(INFO) << "pack end "; - if (key.empty()) { - DLOG(INFO) << "key is empty"; - return false; - } - if (pid_set.find(*index_pid) == pid_set.end()) { - entry->clear_dimensions(); - ::openmldb::api::Dimension* dim = entry->add_dimensions(); - dim->set_key(key); - dim->set_idx(idx); - return true; - } - return false; -} - -bool MemTableSnapshot::DumpSnapshotIndexData(std::shared_ptr
table, - const std::vector>& index_cols, uint32_t max_idx, - uint32_t idx, const std::vector<::openmldb::log::WriteHandle*>& whs, - uint64_t* snapshot_offset) { - uint32_t partition_num = whs.size(); - ::openmldb::api::Manifest manifest; - manifest.set_offset(0); - int ret = GetLocalManifest(snapshot_path_ + MANIFEST, manifest); - if (ret == -1) { - return false; - } - *snapshot_offset = manifest.offset(); - std::string path = snapshot_path_ + "/" + manifest.name(); - uint64_t succ_cnt = 0; - uint64_t failed_cnt = 0; - FILE* fd = fopen(path.c_str(), "rb"); - if (fd == NULL) { - PDLOG(WARNING, "fail to open path %s for error %s", path.c_str(), strerror(errno)); - return false; - } - ::openmldb::log::SequentialFile* seq_file = ::openmldb::log::NewSeqFile(path, fd); - bool compressed = IsCompressed(path); - ::openmldb::log::Reader reader(seq_file, NULL, false, 0, compressed); - ::openmldb::api::LogEntry entry; - std::string buffer; - std::string entry_buff; - DLOG(INFO) << "begin dump snapshot index data"; - while (true) { - buffer.clear(); - ::openmldb::base::Slice record; - ::openmldb::log::Status status = reader.ReadRecord(&record, &buffer); - if (status.IsWaitRecord() || status.IsEof()) { - PDLOG(INFO, - "read path %s for table tid %u pid %u completed, succ_cnt " - "%lu, failed_cnt %lu", - path.c_str(), tid_, pid_, succ_cnt, failed_cnt); - break; - } - if (!status.ok()) { - PDLOG(WARNING, "fail to read record for tid %u, pid %u with error %s", tid_, pid_, - status.ToString().c_str()); - failed_cnt++; - continue; - } - entry_buff.assign(record.data(), record.size()); - if (!entry.ParseFromString(entry_buff)) { - PDLOG(WARNING, "fail to parse record for tid %u, pid %u", tid_, pid_); - failed_cnt++; - continue; - } - uint32_t index_pid = 0; - if (!PackNewIndexEntry(table, index_cols, max_idx, idx, partition_num, &entry, &index_pid)) { - DLOG(INFO) << "pack new entry fail in snapshot"; - continue; - } - std::string entry_str; - entry.SerializeToString(&entry_str); - ::openmldb::base::Slice new_record(entry_str); - status = whs[index_pid]->Write(new_record); - if (!status.ok()) { - delete seq_file; - PDLOG(WARNING, - "fail to dump index entrylog in snapshot to pid[%u]. tid " - "%u pid %u", - index_pid, tid_, pid_); - return false; - } - succ_cnt++; - } - delete seq_file; - return true; -} - -bool MemTableSnapshot::DumpIndexData(std::shared_ptr
table, const ::openmldb::common::ColumnKey& column_key, - uint32_t idx, const std::vector<::openmldb::log::WriteHandle*>& whs) { - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - if (making_snapshot_.exchange(true, std::memory_order_consume)) { - PDLOG(INFO, "snapshot is doing now. tid %u, pid %u", tid, pid); - return false; - } - std::map column_desc_map; - auto table_meta = table->GetTableMeta(); - for (int32_t i = 0; i < table_meta->column_desc_size(); ++i) { - column_desc_map.insert(std::make_pair(table_meta->column_desc(i).name(), i)); - } - uint32_t base_size = table_meta->column_desc_size(); - for (int32_t i = 0; i < table_meta->added_column_desc_size(); ++i) { - column_desc_map.insert(std::make_pair(table_meta->added_column_desc(i).name(), i + base_size)); - } - std::vector> index_cols; - uint32_t max_idx = 0; - for (const auto& ck : table_meta->column_key()) { - std::vector cols; - if (ck.flag()) { - continue; - } - for (const auto& name : ck.col_name()) { - if (column_desc_map.find(name) != column_desc_map.end()) { - uint32_t col_idx = column_desc_map[name]; - cols.push_back(col_idx); - if (col_idx > max_idx) { - max_idx = col_idx; - } - } else { - PDLOG(WARNING, "fail to find column_desc %s", name.c_str()); - making_snapshot_.store(false, std::memory_order_release); - return false; - } - } - index_cols.push_back(cols); - } - std::vector cols; - for (const auto& name : column_key.col_name()) { - if (column_desc_map.find(name) != column_desc_map.end()) { - uint32_t col_idx = column_desc_map[name]; - cols.push_back(col_idx); - if (col_idx > max_idx) { - max_idx = col_idx; - } - } else { - PDLOG(WARNING, "fail to find column_desc %s", name.c_str()); - making_snapshot_.store(false, std::memory_order_release); - return false; - } - } - index_cols.push_back(cols); - uint64_t collected_offset = CollectDeletedKey(0); - uint64_t snapshot_offset = 0; - bool ret = true; - if (!DumpSnapshotIndexData(table, index_cols, max_idx, idx, whs, &snapshot_offset) || - !DumpBinlogIndexData(table, index_cols, max_idx, idx, whs, snapshot_offset, collected_offset)) { - ret = false; - } - making_snapshot_.store(false, std::memory_order_release); - return ret; -} - -bool MemTableSnapshot::DumpBinlogIndexData(std::shared_ptr
table, - const std::vector>& index_cols, uint32_t max_idx, - uint32_t idx, const std::vector<::openmldb::log::WriteHandle*>& whs, - uint64_t snapshot_offset, uint64_t collected_offset) { - ::openmldb::log::LogReader log_reader(log_part_, log_path_, false); - log_reader.SetOffset(snapshot_offset); - uint64_t cur_offset = snapshot_offset; - uint32_t partition_num = whs.size(); - ::openmldb::api::LogEntry entry; - uint64_t succ_cnt = 0; - uint64_t failed_cnt = 0; - uint64_t consumed = ::baidu::common::timer::now_time(); - int last_log_index = log_reader.GetLogIndex(); - std::string buffer; - std::string entry_buff; - DLOG(INFO) << "begin dump binlog index data"; - while (cur_offset < collected_offset) { - buffer.clear(); - ::openmldb::base::Slice record; - ::openmldb::log::Status status = log_reader.ReadNextRecord(&record, &buffer); - if (status.IsWaitRecord()) { - int end_log_index = log_reader.GetEndLogIndex(); - int cur_log_index = log_reader.GetLogIndex(); - if (end_log_index >= 0 && end_log_index > cur_log_index) { - log_reader.RollRLogFile(); - PDLOG(WARNING, - "read new binlog file. tid[%u] pid[%u] cur_log_index[%d] " - "end_log_index[%d] cur_offset[%lu]", - tid_, pid_, cur_log_index, end_log_index, cur_offset); - continue; - } - consumed = ::baidu::common::timer::now_time() - consumed; - PDLOG(INFO, - "table tid %u pid %u completed, succ_cnt %lu, failed_cnt " - "%lu, consumed %us", - tid_, pid_, succ_cnt, failed_cnt, consumed); - break; - } - if (status.IsEof()) { - if (log_reader.GetLogIndex() != last_log_index) { - last_log_index = log_reader.GetLogIndex(); - continue; - } - break; - } - if (!status.ok()) { - failed_cnt++; - continue; - } - entry_buff.assign(record.data(), record.size()); - if (!entry.ParseFromString(entry_buff)) { - PDLOG(WARNING, "fail parse record for tid %u, pid %u with value %s", tid_, pid_, - ::openmldb::base::DebugString(entry_buff).c_str()); - failed_cnt++; - continue; - } - if (cur_offset >= entry.log_index()) { - DEBUGLOG("offset %lu has been made snapshot", entry.log_index()); - continue; - } - - if (cur_offset + 1 != entry.log_index()) { - PDLOG(WARNING, - "missing log entry cur_offset %lu , new entry offset %lu for " - "tid %u, pid %u", - cur_offset, entry.log_index(), tid_, pid_); - } - uint32_t index_pid = 0; - if (!PackNewIndexEntry(table, index_cols, max_idx, idx, partition_num, &entry, &index_pid)) { - LOG(INFO) << "pack new entry fail in binlog"; - continue; - } - std::string entry_str; - entry.SerializeToString(&entry_str); - ::openmldb::base::Slice new_record(entry_str); - status = whs[index_pid]->Write(new_record); - if (!status.ok()) { - PDLOG(WARNING, "fail to dump index entrylog in binlog to pid[%u].", index_pid); - return false; - } - cur_offset = entry.log_index(); - succ_cnt++; - } - return true; -} - -int MemTableSnapshot::DecodeData(std::shared_ptr
table, const openmldb::api::LogEntry& entry, uint32_t max_idx, - std::vector& row) { - std::string buff; - openmldb::base::Slice data; - if (table->GetCompressType() == openmldb::type::kSnappy) { - snappy::Uncompress(entry.value().data(), entry.value().size(), &buff); - data.reset(buff.data(), buff.size()); - } else { - data.reset(entry.value().data(), entry.value().size()); - } - const int8_t* raw = reinterpret_cast(data.data()); - uint8_t version = openmldb::codec::RowView::GetSchemaVersion(raw); - int32_t data_size = data.size(); - std::shared_ptr schema = table->GetVersionSchema(version); - if (schema == nullptr) { - LOG(WARNING) << "fail get version " << unsigned(version) << " schema"; - return 1; - } - - bool ok = openmldb::codec::RowCodec::DecodeRow(*schema, raw, data_size, true, 0, max_idx + 1, row); - if (!ok) { - DLOG(WARNING) << "decode data error"; - return 3; - } - if (schema->size() < (int64_t)(max_idx + 1)) { - DLOG(WARNING) << "data size is " << schema->size() << " less than " << max_idx + 1; - return 2; - } - return 0; -} - -bool MemTableSnapshot::IsCompressed(const std::string& path) { - if (path.find(openmldb::log::ZLIB_COMPRESS_SUFFIX) != std::string::npos || - path.find(openmldb::log::SNAPPY_COMPRESS_SUFFIX) != std::string::npos) { - return true; - } - return false; + return status; } } // namespace storage diff --git a/src/storage/mem_table_snapshot.h b/src/storage/mem_table_snapshot.h index badeb019f96..29d3ff0adaa 100644 --- a/src/storage/mem_table_snapshot.h +++ b/src/storage/mem_table_snapshot.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -31,15 +32,107 @@ #include "proto/tablet.pb.h" #include "storage/snapshot.h" -using ::openmldb::api::LogEntry; namespace openmldb { namespace storage { using ::openmldb::log::WriteHandle; - typedef ::openmldb::base::Skiplist LogParts; -// table snapshot +struct MemSnapshotMeta : SnapshotMeta { + MemSnapshotMeta(const std::string& name, const std::string& snapshot_path, + const std::string& compression) : SnapshotMeta(name), snapshot_compression(compression) { + if (snapshot_compression != "off") { + snapshot_name.append("."); + snapshot_name.append(snapshot_compression); + } + snapshot_name_tmp = snapshot_name + ".tmp"; + full_path = snapshot_path + snapshot_name; + tmp_file_path = snapshot_path + snapshot_name_tmp; + } + + uint64_t expired_key_num = 0; + uint64_t deleted_key_num = 0; + std::string snapshot_compression; + std::string snapshot_name_tmp; + std::string full_path; + std::string tmp_file_path; +}; + +enum class DataReaderType { + kSnapshot = 1, + kBinlog = 2, + kSnapshotAndBinlog = 3 +}; + +class DataReader { + public: + DataReader(const std::string& snapshot_path, LogParts* log_part, + const std::string& log_path, DataReaderType type, uint64_t start_offset, uint64_t end_offset) + : snapshot_path_(snapshot_path), log_part_(log_part), log_path_(log_path), read_type_(type), + start_offset_(start_offset), end_offset_(end_offset) {} + DataReader(const DataReader&) = delete; + DataReader& operator=(const DataReader&) = delete; + + static std::shared_ptr CreateDataReader(const std::string& snapshot_path, LogParts* log_part, + const std::string& log_path, DataReaderType type); + static std::shared_ptr CreateDataReader(const std::string& snapshot_path, LogParts* log_part, + const std::string& log_path, DataReaderType type, uint64_t end_offset); + static std::shared_ptr CreateDataReader(LogParts* log_part, const std::string& log_path, + uint64_t start_offset, uint64_t end_offset); + + bool HasNext(); + ::openmldb::api::LogEntry& GetValue() { return entry_; } + const std::string& GetStrValue() { return entry_buff_; } + bool Init(); + + private: + bool ReadFromSnapshot(); + bool ReadFromBinlog(); + + private: + std::string snapshot_path_; + LogParts* log_part_; + std::string log_path_; + DataReaderType read_type_; + uint64_t start_offset_ = 0; + uint64_t end_offset_ = 0; + uint64_t cur_offset_ = 0; + bool read_snapshot_ = false; + bool read_binlog_ = false; + std::shared_ptr<::openmldb::log::SequentialFile> seq_file_; + std::shared_ptr<::openmldb::log::Reader> snapshot_reader_; + std::shared_ptr<::openmldb::log::LogReader> binlog_reader_; + std::string buffer_; + std::string entry_buff_; + ::openmldb::base::Slice record_; + ::openmldb::api::LogEntry entry_; + uint64_t succ_cnt_ = 0; + uint64_t failed_cnt_ = 0; +}; + + +class TableIndexInfo { + public: + TableIndexInfo(const ::openmldb::api::TableMeta& table_meta, + const std::vector<::openmldb::common::ColumnKey>& add_indexs) + : table_meta_(table_meta), add_indexs_(add_indexs) {} + bool Init(); + const std::vector& GetAllIndexCols() const { return all_index_cols_; } + const std::vector& GetAddIndexIdx() const { return add_index_idx_vec_; } + bool HasIndex(uint32_t idx) const; + const std::vector& GetIndexCols(uint32_t idx); + const std::vector& GetRealIndexCols(uint32_t idx); // the pos in all_index_cols_ + + private: + ::openmldb::api::TableMeta table_meta_; + std::vector<::openmldb::common::ColumnKey> add_indexs_; + std::map column_idx_map_; + std::vector all_index_cols_; + std::vector add_index_idx_vec_; + std::map> index_cols_map_; + std::map> real_index_cols_map_; +}; + class MemTableSnapshot : public Snapshot { public: MemTableSnapshot(uint32_t tid, uint32_t pid, LogParts* log_part, const std::string& db_root_path); @@ -57,61 +150,19 @@ class MemTableSnapshot : public Snapshot { uint64_t end_offset, uint64_t term = 0) override; - int TTLSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, WriteHandle* wh, - uint64_t& count, uint64_t& expired_key_num, // NOLINT - uint64_t& deleted_key_num); // NOLINT + int TTLSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, + const std::shared_ptr& wh, MemSnapshotMeta* snapshot_meta); void Put(std::string& path, std::shared_ptr
& table, // NOLINT std::vector recordPtr, std::atomic* succ_cnt, std::atomic* failed_cnt); - std::string GenSnapshotName(); - - base::Status GetAllDecoder(std::shared_ptr
table, std::map* decoder_map); - - base::Status GetIndexKey(std::shared_ptr
table, const std::shared_ptr& index, - const base::Slice& data, std::map* decoder_map, std::string* index_key); - - base::Status ExtractIndexFromSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, - WriteHandle* wh, const std::vector<::openmldb::common::ColumnKey>& add_indexs, - uint32_t partition_num, uint64_t* count, uint64_t* expired_key_num, uint64_t* deleted_key_num); + base::Status ExtractIndexData(const std::shared_ptr
& table, + const std::vector<::openmldb::common::ColumnKey>& add_indexs, + const std::vector>& whs, + uint64_t offset, bool dump_data); int CheckDeleteAndUpdate(std::shared_ptr
table, ::openmldb::api::LogEntry* new_entry); - base::Status ExtractIndexFromBinlog(std::shared_ptr
table, - WriteHandle* wh, const std::vector<::openmldb::common::ColumnKey>& add_indexs, - uint64_t collected_offset, uint32_t partition_num, uint64_t* offset, - uint64_t* last_term, uint64_t* count, uint64_t* expired_key_num, uint64_t* deleted_key_num); - - int ExtractIndexFromSnapshot(std::shared_ptr
table, const ::openmldb::api::Manifest& manifest, - WriteHandle* wh, - const ::openmldb::common::ColumnKey& column_key, // NOLINT - uint32_t idx, uint32_t partition_num, uint32_t max_idx, - const std::vector& index_cols, - uint64_t& count, // NOLINT - uint64_t& expired_key_num, uint64_t& deleted_key_num); // NOLINT - - bool DumpSnapshotIndexData(std::shared_ptr
table, const std::vector>& index_cols, - uint32_t max_idx, uint32_t idx, const std::vector<::openmldb::log::WriteHandle*>& whs, - uint64_t* snapshot_offset); - - bool DumpBinlogIndexData(std::shared_ptr
table, const std::vector>& index_cols, - uint32_t max_idx, uint32_t idx, const std::vector<::openmldb::log::WriteHandle*>& whs, - uint64_t snapshot_offset, uint64_t collected_offset); - - int ExtractIndexData(std::shared_ptr
table, const ::openmldb::common::ColumnKey& column_key, uint32_t idx, - uint32_t partition_num, - uint64_t& out_offset); // NOLINT - - int ExtractIndexData(std::shared_ptr
table, const std::vector<::openmldb::common::ColumnKey>& column_key, - uint32_t partition_num, uint64_t* out_offset); - - bool DumpIndexData(std::shared_ptr
table, const ::openmldb::common::ColumnKey& column_key, uint32_t idx, - const std::vector<::openmldb::log::WriteHandle*>& whs); - - bool PackNewIndexEntry(std::shared_ptr
table, const std::vector>& index_cols, - uint32_t max_idx, uint32_t idx, uint32_t partition_num, ::openmldb::api::LogEntry* entry, - uint32_t* index_pid); - int RemoveDeletedKey(const ::openmldb::api::LogEntry& entry, const std::set& deleted_index, std::string* buffer); @@ -122,10 +173,12 @@ class MemTableSnapshot : public Snapshot { uint64_t CollectDeletedKey(uint64_t end_offset); - int DecodeData(std::shared_ptr
table, const openmldb::api::LogEntry& entry, uint32_t maxIdx, - std::vector& row); // NOLINT + ::openmldb::base::Status DecodeData(const std::shared_ptr
& table, const openmldb::api::LogEntry& entry, + const std::vector& cols, std::vector* row); + + std::string GenSnapshotName(); - inline bool IsCompressed(const std::string& path); + ::openmldb::base::Status WriteSnapshot(const MemSnapshotMeta& snapshot_meta); private: LogParts* log_part_; diff --git a/src/storage/snapshot.cc b/src/storage/snapshot.cc index 27781a61652..b2cacec1de9 100644 --- a/src/storage/snapshot.cc +++ b/src/storage/snapshot.cc @@ -17,23 +17,29 @@ #include "storage/snapshot.h" #include -#include -#include #include #include #include +#include "absl/strings/str_cat.h" #include "base/glog_wrapper.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/text_format.h" namespace openmldb { namespace storage { -const std::string MANIFEST = "MANIFEST"; // NOLINT +constexpr const char* MANIFEST = "MANIFEST"; + +int Snapshot::GenManifest(const SnapshotMeta& snapshot_meta) { + return GenManifest(snapshot_meta.snapshot_name, snapshot_meta.count, + snapshot_meta.offset, snapshot_meta.term); +} int Snapshot::GenManifest(const std::string& snapshot_name, uint64_t key_count, uint64_t offset, uint64_t term) { DEBUGLOG("record offset[%lu]. add snapshot[%s] key_count[%lu]", offset, snapshot_name.c_str(), key_count); - std::string full_path = snapshot_path_ + MANIFEST; - std::string tmp_file = snapshot_path_ + MANIFEST + ".tmp"; + std::string full_path = absl::StrCat(snapshot_path_, MANIFEST); + std::string tmp_file = absl::StrCat(snapshot_path_, MANIFEST, ".tmp"); ::openmldb::api::Manifest manifest; std::string manifest_info; manifest.set_offset(offset); @@ -43,7 +49,7 @@ int Snapshot::GenManifest(const std::string& snapshot_name, uint64_t key_count, manifest_info.clear(); google::protobuf::TextFormat::PrintToString(manifest, &manifest_info); FILE* fd_write = fopen(tmp_file.c_str(), "w"); - if (fd_write == NULL) { + if (fd_write == nullptr) { PDLOG(WARNING, "fail to open file %s", tmp_file.c_str()); return -1; } @@ -58,7 +64,7 @@ int Snapshot::GenManifest(const std::string& snapshot_name, uint64_t key_count, } fclose(fd_write); if (!io_error && rename(tmp_file.c_str(), full_path.c_str()) == 0) { - DEBUGLOG("%s generate success. path[%s]", MANIFEST.c_str(), full_path.c_str()); + DEBUGLOG("%s generate success. path[%s]", MANIFEST, full_path.c_str()); return 0; } unlink(tmp_file.c_str()); @@ -68,7 +74,7 @@ int Snapshot::GenManifest(const std::string& snapshot_name, uint64_t key_count, int Snapshot::GetLocalManifest(const std::string& full_path, ::openmldb::api::Manifest& manifest) { int fd = open(full_path.c_str(), O_RDONLY); if (fd < 0) { - PDLOG(INFO, "[%s] does not exist", MANIFEST.c_str()); + PDLOG(INFO, "[%s] does not exist", MANIFEST); return 1; } else { google::protobuf::io::FileInputStream fileInput(fd); diff --git a/src/storage/snapshot.h b/src/storage/snapshot.h index 65dbe3adb45..3565502e9b9 100644 --- a/src/storage/snapshot.h +++ b/src/storage/snapshot.h @@ -26,6 +26,15 @@ namespace openmldb { namespace storage { +struct SnapshotMeta { + explicit SnapshotMeta(const std::string& name) : snapshot_name(name) {} + + uint64_t count = 0; + uint64_t term = 0; + uint64_t offset = 0; + std::string snapshot_name; +}; + class Snapshot { public: Snapshot(uint32_t tid, uint32_t pid) : tid_(tid), pid_(pid), offset_(0), making_snapshot_(false) {} @@ -39,6 +48,7 @@ class Snapshot { uint64_t& latest_offset) = 0; // NOLINT uint64_t GetOffset() { return offset_; } int GenManifest(const std::string& snapshot_name, uint64_t key_count, uint64_t offset, uint64_t term); + int GenManifest(const SnapshotMeta& snapshot_meta); static int GetLocalManifest(const std::string& full_path, ::openmldb::api::Manifest& manifest); // NOLINT std::string GetSnapshotPath() { return snapshot_path_; } diff --git a/src/tablet/tablet_impl.cc b/src/tablet/tablet_impl.cc index 3059bb64f6a..609d8db6786 100644 --- a/src/tablet/tablet_impl.cc +++ b/src/tablet/tablet_impl.cc @@ -57,6 +57,7 @@ #include "glog/logging.h" #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/text_format.h" +#include "nameserver/task.h" #include "schema/schema_adapter.h" #include "storage/binlog.h" #include "storage/disk_table_snapshot.h" @@ -2071,7 +2072,7 @@ void TabletImpl::AddReplica(RpcController* controller, const ::openmldb::api::Re brpc::ClosureGuard done_guard(done); std::shared_ptr<::openmldb::api::TaskInfo> task_ptr; if (request->has_task_info() && request->task_info().IsInitialized()) { - if (AddOPMultiTask(request->task_info(), ::openmldb::api::TaskType::kAddReplica, task_ptr) < 0) { + if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kAddReplica, task_ptr) < 0) { response->set_code(-1); response->set_msg("add task failed"); return; @@ -4053,7 +4054,7 @@ void TabletImpl::SetTaskStatus(std::shared_ptr<::openmldb::api::TaskInfo>& task_ task_ptr->set_status(status); } -int TabletImpl::GetTaskStatus(std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, +int TabletImpl::GetTaskStatus(const std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, ::openmldb::api::TaskStatus* status) { if (!task_ptr) { return -1; @@ -4066,7 +4067,7 @@ int TabletImpl::GetTaskStatus(std::shared_ptr<::openmldb::api::TaskInfo>& task_p int TabletImpl::AddOPTask(const ::openmldb::api::TaskInfo& task_info, ::openmldb::api::TaskType task_type, std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr) { std::lock_guard lock(mu_); - if (FindTask(task_info.op_id(), task_info.task_type())) { + if (IsExistTaskUnLock(task_info)) { PDLOG(WARNING, "task is running. op_id[%lu] op_type[%s] task_type[%s]", task_info.op_id(), ::openmldb::api::OPType_Name(task_info.op_type()).c_str(), ::openmldb::api::TaskType_Name(task_info.task_type()).c_str()); @@ -4077,72 +4078,43 @@ int TabletImpl::AddOPTask(const ::openmldb::api::TaskInfo& task_info, ::openmldb task_ptr->set_status(::openmldb::api::TaskStatus::kDoing); auto iter = task_map_.find(task_info.op_id()); if (iter == task_map_.end()) { - task_map_.insert(std::make_pair(task_info.op_id(), std::list>())); + iter = task_map_.emplace(task_info.op_id(), std::list>()).first; } - task_map_[task_info.op_id()].push_back(task_ptr); + iter->second.push_back(task_ptr); if (task_info.task_type() != task_type) { - PDLOG(WARNING, "task type is not match. type is[%s]", - ::openmldb::api::TaskType_Name(task_info.task_type()).c_str()); + PDLOG(WARNING, "task type is not match. type is[%s] expect type is [%s]", + ::openmldb::api::TaskType_Name(task_info.task_type()).c_str(), + ::openmldb::api::TaskType_Name(task_type).c_str()); task_ptr->set_status(::openmldb::api::TaskStatus::kFailed); return -1; } - PDLOG(INFO, "add task map success, op_id[%lu] op_type[%s] task_type[%s]", task_info.op_id(), + PDLOG(INFO, "add task map success, op_id %lu op_type %s task_type %s %s", task_info.op_id(), ::openmldb::api::OPType_Name(task_info.op_type()).c_str(), - ::openmldb::api::TaskType_Name(task_info.task_type()).c_str()); - return 0; -} - -std::shared_ptr<::openmldb::api::TaskInfo> TabletImpl::FindTask(uint64_t op_id, ::openmldb::api::TaskType task_type) { - auto iter = task_map_.find(op_id); - if (iter == task_map_.end()) { - return std::shared_ptr<::openmldb::api::TaskInfo>(); - } - for (auto& task : iter->second) { - if (task->op_id() == op_id && task->task_type() == task_type) { - return task; - } - } - return std::shared_ptr<::openmldb::api::TaskInfo>(); -} - -int TabletImpl::AddOPMultiTask(const ::openmldb::api::TaskInfo& task_info, ::openmldb::api::TaskType task_type, - std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr) { - std::lock_guard lock(mu_); - if (FindMultiTask(task_info)) { - PDLOG(WARNING, "task is running. op_id[%lu] op_type[%s] task_type[%s]", task_info.op_id(), - ::openmldb::api::OPType_Name(task_info.op_type()).c_str(), - ::openmldb::api::TaskType_Name(task_info.task_type()).c_str()); - return -1; - } - task_ptr.reset(task_info.New()); - task_ptr->CopyFrom(task_info); - task_ptr->set_status(::openmldb::api::TaskStatus::kDoing); - auto iter = task_map_.find(task_info.op_id()); - if (iter == task_map_.end()) { - task_map_.insert(std::make_pair(task_info.op_id(), std::list>())); - } - task_map_[task_info.op_id()].push_back(task_ptr); - if (task_info.task_type() != task_type) { - PDLOG(WARNING, "task type is not match. type is[%s]", - ::openmldb::api::TaskType_Name(task_info.task_type()).c_str()); - task_ptr->set_status(::openmldb::api::TaskStatus::kFailed); - return -1; - } + ::openmldb::api::TaskType_Name(task_info.task_type()).c_str(), + nameserver::Task::GetAdditionalMsg(task_info)); return 0; } -std::shared_ptr<::openmldb::api::TaskInfo> TabletImpl::FindMultiTask(const ::openmldb::api::TaskInfo& task_info) { - auto iter = task_map_.find(task_info.op_id()); +bool TabletImpl::IsExistTaskUnLock(const ::openmldb::api::TaskInfo& task) { + auto iter = task_map_.find(task.op_id()); if (iter == task_map_.end()) { - return std::shared_ptr<::openmldb::api::TaskInfo>(); + return false; } - for (auto& task : iter->second) { - if (task->op_id() == task_info.op_id() && task->task_type() == task_info.task_type() && - task->task_id() == task_info.task_id()) { - return task; + for (auto& cur_task : iter->second) { + if (cur_task->op_id() == task.op_id() && cur_task->task_type() == task.task_type()) { + if (task.has_tid() && (!cur_task->has_tid() || task.tid() != cur_task->tid())) { + continue; + } + if (task.has_pid() && (!cur_task->has_pid() || task.pid() != cur_task->pid())) { + continue; + } + if (task.has_task_id() && (!cur_task->has_task_id() || task.task_id() != cur_task->task_id())) { + continue; + } + return true; } } - return std::shared_ptr<::openmldb::api::TaskInfo>(); + return false; } void TabletImpl::GcTable(uint32_t tid, uint32_t pid, bool execute_once) { @@ -4714,7 +4686,7 @@ void TabletImpl::SendIndexData(RpcController* controller, const ::openmldb::api: brpc::ClosureGuard done_guard(done); std::shared_ptr<::openmldb::api::TaskInfo> task_ptr; if (request->has_task_info() && request->task_info().IsInitialized()) { - if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kSendIndexData, task_ptr) < 0) { + if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kSendIndexRequest, task_ptr) < 0) { response->set_code(-1); response->set_msg("add task failed"); return; @@ -4776,7 +4748,7 @@ void TabletImpl::SendIndexDataInternal(std::shared_ptr<::openmldb::storage::Tabl if (kv.first == pid) { continue; } - std::string index_file_name = std::to_string(pid) + "_" + std::to_string(kv.first) + "_index.data"; + std::string index_file_name = absl::StrCat(pid, "_", kv.first, "_index.data"); std::string src_file = index_path + index_file_name; if (!::openmldb::base::IsExists(src_file)) { PDLOG(WARNING, "file %s does not exist. tid[%u] pid[%u]", src_file.c_str(), tid, pid); @@ -4823,9 +4795,7 @@ void TabletImpl::SendIndexDataInternal(std::shared_ptr<::openmldb::storage::Tabl auto tmp_map = std::atomic_load_explicit(&real_ep_map_, std::memory_order_acquire); auto iter = tmp_map->find(kv.second); if (iter == tmp_map->end()) { - PDLOG(WARNING, - "name %s not found in real_ep_map." - "tid[%u] pid[%u]", + PDLOG(WARNING, "name %s not found in real_ep_map. tid[%u] pid[%u]", kv.second.c_str(), tid, pid); break; } @@ -4833,89 +4803,29 @@ void TabletImpl::SendIndexDataInternal(std::shared_ptr<::openmldb::storage::Tabl } FileSender sender(tid, kv.first, table->GetStorageMode(), real_endpoint); if (!sender.Init()) { - PDLOG(WARNING, - "Init FileSender failed. tid[%u] pid[%u] des_pid[%u] " - "endpoint[%s]", + PDLOG(WARNING, "Init FileSender failed. tid[%u] pid[%u] des_pid[%u] endpoint[%s]", tid, pid, kv.first, kv.second.c_str()); SetTaskStatus(task_ptr, ::openmldb::api::TaskStatus::kFailed); return; } if (sender.SendFile(index_file_name, std::string("index"), index_path + index_file_name) < 0) { - PDLOG(WARNING, "send file %s failed. tid[%u] pid[%u] des_pid[%u]", index_file_name.c_str(), tid, pid, - kv.first); + PDLOG(WARNING, "send file %s failed. tid[%u] pid[%u] des_pid[%u]", + index_file_name.c_str(), tid, pid, kv.first); SetTaskStatus(task_ptr, ::openmldb::api::TaskStatus::kFailed); return; } - PDLOG(INFO, - "send file %s to endpoint %s success. tid[%u] pid[%u] " - "des_pid[%u]", + PDLOG(INFO, "send file %s to endpoint %s success. tid[%u] pid[%u] des_pid[%u]", index_file_name.c_str(), kv.second.c_str(), tid, pid, kv.first); } } SetTaskStatus(task_ptr, ::openmldb::api::TaskStatus::kDone); } -void TabletImpl::DumpIndexData(RpcController* controller, const ::openmldb::api::DumpIndexDataRequest* request, - ::openmldb::api::GeneralResponse* response, Closure* done) { - brpc::ClosureGuard done_guard(done); - std::shared_ptr<::openmldb::api::TaskInfo> task_ptr; - if (request->has_task_info() && request->task_info().IsInitialized()) { - if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kDumpIndexData, task_ptr) < 0) { - response->set_code(-1); - response->set_msg("add task failed"); - return; - } - } - uint32_t tid = request->tid(); - uint32_t pid = request->pid(); - do { - std::shared_ptr
table; - std::shared_ptr snapshot; - { - std::lock_guard spin_lock(spin_mutex_); - table = GetTableUnLock(tid, pid); - if (!table) { - PDLOG(WARNING, "table does not exist. tid[%u] pid[%u]", tid, pid); - response->set_code(::openmldb::base::ReturnCode::kTableIsNotExist); - response->set_msg("table does not exist"); - break; - } - if (table->GetStorageMode() != ::openmldb::common::kMemory) { - response->set_code(::openmldb::base::ReturnCode::kOperatorNotSupport); - response->set_msg("only support mem_table"); - break; - } - if (table->GetTableStat() != ::openmldb::storage::kNormal) { - PDLOG(WARNING, "table state is %d, cannot dump index data. %u, pid %u", table->GetTableStat(), tid, - pid); - response->set_code(::openmldb::base::ReturnCode::kTableStatusIsNotKnormal); - response->set_msg("table status is not kNormal"); - break; - } - snapshot = GetSnapshotUnLock(tid, pid); - if (!snapshot) { - PDLOG(WARNING, "snapshot does not exist. tid[%u] pid[%u]", tid, pid); - response->set_code(::openmldb::base::ReturnCode::kSnapshotIsNotExist); - response->set_msg("table snapshot does not exist"); - break; - } - } - std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot = - std::static_pointer_cast<::openmldb::storage::MemTableSnapshot>(snapshot); - task_pool_.AddTask(boost::bind(&TabletImpl::DumpIndexDataInternal, this, table, memtable_snapshot, - request->partition_num(), request->column_key(), request->idx(), task_ptr)); - response->set_code(::openmldb::base::ReturnCode::kOk); - response->set_msg("ok"); - PDLOG(INFO, "dump index tid[%u] pid[%u]", tid, pid); - return; - } while (0); - SetTaskStatus(task_ptr, ::openmldb::api::TaskStatus::kFailed); -} - -void TabletImpl::DumpIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, - std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, - uint32_t partition_num, ::openmldb::common::ColumnKey& column_key, uint32_t idx, - std::shared_ptr<::openmldb::api::TaskInfo> task) { +void TabletImpl::ExtractIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, + std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, + const std::vector<::openmldb::common::ColumnKey>& column_keys, + uint32_t partition_num, uint64_t offset, bool dump_data, + std::shared_ptr<::openmldb::api::TaskInfo> task) { uint32_t tid = table->GetId(); uint32_t pid = table->GetPid(); std::string db_root_path; @@ -4931,34 +4841,32 @@ void TabletImpl::DumpIndexDataInternal(std::shared_ptr<::openmldb::storage::Tabl SetTaskStatus(task, ::openmldb::api::kFailed); return; } - std::vector<::openmldb::log::WriteHandle*> whs; - for (uint32_t i = 0; i < partition_num; i++) { - std::string index_file_name = std::to_string(pid) + "_" + std::to_string(i) + "_index.data"; - std::string index_data_path = index_path + index_file_name; - FILE* fd = fopen(index_data_path.c_str(), "wb+"); - if (fd == nullptr) { - LOG(WARNING) << "fail to create file " << index_data_path << ". tid " << tid << " pid " << pid; - SetTaskStatus(task, ::openmldb::api::kFailed); - for (auto& wh : whs) { - delete wh; - wh = nullptr; + std::vector> whs(partition_num); + if (dump_data) { + for (uint32_t i = 0; i < partition_num; i++) { + std::string index_file_name = absl::StrCat(pid, "_", i, "_index.data"); + std::string index_data_path = index_path + index_file_name; + FILE* fd = fopen(index_data_path.c_str(), "wb+"); + if (fd == nullptr) { + LOG(WARNING) << "fail to create file " << index_data_path << ". tid " << tid << " pid " << pid; + SetTaskStatus(task, ::openmldb::api::kFailed); + return; } - return; + whs[i] = std::make_shared<::openmldb::log::WriteHandle>("off", index_file_name, fd); } - ::openmldb::log::WriteHandle* wh = new ::openmldb::log::WriteHandle("off", index_file_name, fd); - whs.push_back(wh); } - if (memtable_snapshot->DumpIndexData(table, column_key, idx, whs)) { - PDLOG(INFO, "dump index on table tid[%u] pid[%u] succeed", tid, pid); + auto status = memtable_snapshot->ExtractIndexData(table, column_keys, whs, offset, dump_data); + if (status.OK()) { + PDLOG(INFO, "extract index on table tid[%u] pid[%u] succeed", tid, pid); SetTaskStatus(task, ::openmldb::api::kDone); } else { - PDLOG(WARNING, "fail to dump index on table tid[%u] pid[%u]", tid, pid); + PDLOG(WARNING, "fail to extract index on table tid[%u] pid[%u] msg[%s]", tid, pid, status.GetMsg().c_str()); SetTaskStatus(task, ::openmldb::api::kFailed); } for (auto& wh : whs) { - wh->EndLog(); - delete wh; - wh = nullptr; + if (wh) { + wh->EndLog(); + } } } @@ -4967,7 +4875,7 @@ void TabletImpl::LoadIndexData(RpcController* controller, const ::openmldb::api: brpc::ClosureGuard done_guard(done); std::shared_ptr<::openmldb::api::TaskInfo> task_ptr; if (request->has_task_info() && request->task_info().IsInitialized()) { - if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kLoadIndexData, task_ptr) < 0) { + if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kLoadIndexRequest, task_ptr) < 0) { response->set_code(-1); response->set_msg("add task failed"); return; @@ -4990,8 +4898,8 @@ void TabletImpl::LoadIndexData(RpcController* controller, const ::openmldb::api: break; } if (table->GetTableStat() != ::openmldb::storage::kNormal) { - PDLOG(WARNING, "table state is %d, cannot load index data. tid %u, pid %u", table->GetTableStat(), tid, - pid); + PDLOG(WARNING, "table state is %d, cannot load index data. tid %u, pid %u", + table->GetTableStat(), tid, pid); response->set_code(::openmldb::base::ReturnCode::kTableStatusIsNotKnormal); response->set_msg("table status is not kNormal"); break; @@ -5015,8 +4923,8 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ uint64_t last_time, std::shared_ptr<::openmldb::api::TaskInfo> task) { uint64_t cur_time = ::baidu::common::timer::get_micros() / 1000; if (cur_pid == pid) { - task_pool_.AddTask(boost::bind(&TabletImpl::LoadIndexDataInternal, this, tid, pid, cur_pid + 1, partition_num, - cur_time, task)); + task_pool_.AddTask(boost::bind(&TabletImpl::LoadIndexDataInternal, this, + tid, pid, cur_pid + 1, partition_num, cur_time, task)); return; } ::openmldb::api::TaskStatus status = ::openmldb::api::TaskStatus::kFailed; @@ -5045,15 +4953,16 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ return; } std::string index_path = GetDBPath(db_root_path, tid, pid) + "/index/"; - std::string index_file_path = index_path + std::to_string(cur_pid) + "_" + std::to_string(pid) + "_index.data"; + std::string index_file_path = absl::StrCat(index_path, cur_pid, "_", pid, "_index.data"); if (!::openmldb::base::IsExists(index_file_path)) { if (last_time + FLAGS_load_index_max_wait_time < cur_time) { PDLOG(WARNING, "wait time too long. tid %u pid %u file %s", tid, pid, index_file_path.c_str()); SetTaskStatus(task, ::openmldb::api::TaskStatus::kFailed); return; } - task_pool_.DelayTask(FLAGS_task_check_interval, boost::bind(&TabletImpl::LoadIndexDataInternal, this, tid, pid, - cur_pid, partition_num, last_time, task)); + task_pool_.DelayTask(FLAGS_task_check_interval, + boost::bind(&TabletImpl::LoadIndexDataInternal, this, tid, pid, + cur_pid, partition_num, last_time, task)); return; } FILE* fd = fopen(index_file_path.c_str(), "rb"); @@ -5062,8 +4971,8 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ SetTaskStatus(task, ::openmldb::api::TaskStatus::kFailed); return; } - ::openmldb::log::SequentialFile* seq_file = ::openmldb::log::NewSeqFile(index_file_path, fd); - ::openmldb::log::Reader reader(seq_file, nullptr, false, 0, false); + auto seq_file = std::unique_ptr<::openmldb::log::SequentialFile>(::openmldb::log::NewSeqFile(index_file_path, fd)); + ::openmldb::log::Reader reader(seq_file.get(), nullptr, false, 0, false); std::string buffer; uint64_t succ_cnt = 0; uint64_t failed_cnt = 0; @@ -5072,9 +4981,7 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ ::openmldb::base::Slice record; ::openmldb::log::Status status = reader.ReadRecord(&record, &buffer); if (status.IsWaitRecord() || status.IsEof()) { - PDLOG(INFO, - "read path %s for table tid %u pid %u completed. succ_cnt " - "%lu, failed_cnt %lu", + PDLOG(INFO, "read path %s for table tid %u pid %u completed. succ_cnt %lu, failed_cnt %lu", index_file_path.c_str(), tid, pid, succ_cnt, failed_cnt); break; } @@ -5093,7 +5000,6 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ replicator->AppendEntry(entry); succ_cnt++; } - delete seq_file; if (cur_pid == partition_num - 1 || (cur_pid + 1 == pid && pid == partition_num - 1)) { if (FLAGS_recycle_bin_enabled) { std::string recycle_bin_root_path; @@ -5102,12 +5008,11 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ LOG(WARNING) << "fail to get recycle bin root path. tid " << tid << " pid " << pid; openmldb::base::RemoveDirRecursive(index_path); } else { - std::string recycle_path = - recycle_bin_root_path + "/" + std::to_string(tid) + "_" + std::to_string(pid); + std::string recycle_path = absl::StrCat(recycle_bin_root_path, "/", tid, "_", pid); if (!openmldb::base::IsExists(recycle_path)) { openmldb::base::Mkdir(recycle_path); } - std::string dst = recycle_path + "/index" + openmldb::base::GetNowTime(); + std::string dst = absl::StrCat(recycle_path, "/index", openmldb::base::GetNowTime()); openmldb::base::Rename(index_path, dst); } } else { @@ -5122,60 +5027,12 @@ void TabletImpl::LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_ boost::bind(&TabletImpl::LoadIndexDataInternal, this, tid, pid, cur_pid + 1, partition_num, cur_time, task)); } -void TabletImpl::ExtractMultiIndexData(RpcController* controller, - const ::openmldb::api::ExtractMultiIndexDataRequest* request, - ::openmldb::api::GeneralResponse* response, Closure* done) { - brpc::ClosureGuard done_guard(done); - uint32_t tid = request->tid(); - uint32_t pid = request->pid(); - std::shared_ptr
table; - std::shared_ptr snapshot; - { - std::lock_guard spin_lock(spin_mutex_); - table = GetTableUnLock(tid, pid); - if (!table) { - PDLOG(WARNING, "table does not exist. tid %u pid %u", tid, pid); - base::SetResponseStatus(base::ReturnCode::kTableIsNotExist, "table does not exist", response); - return; - } - if (table->GetTableStat() != ::openmldb::storage::kNormal) { - PDLOG(WARNING, "table state is %d, cannot extract index data. tid %u, pid %u", table->GetTableStat(), tid, - pid); - base::SetResponseStatus(base::ReturnCode::kTableStatusIsNotKnormal, "table status is not kNormal", - response); - return; - } - snapshot = GetSnapshotUnLock(tid, pid); - if (!snapshot) { - PDLOG(WARNING, "snapshot does not exist. tid %u pid %u", tid, pid); - base::SetResponseStatus(base::ReturnCode::kSnapshotIsNotExist, "table snapshot does not exist", response); - return; - } - } - std::vector<::openmldb::common::ColumnKey> index_vec; - for (const auto& cur_column_key : request->column_key()) { - index_vec.push_back(cur_column_key); - } - uint64_t offset = 0; - auto memtable_snapshot = std::static_pointer_cast<::openmldb::storage::MemTableSnapshot>(snapshot); - if (memtable_snapshot->ExtractIndexData(table, index_vec, request->partition_num(), &offset) < 0) { - PDLOG(WARNING, "fail to extract index. tid %u pid %u", tid, pid); - return; - } - PDLOG(INFO, "extract index success. tid %u pid %u", tid, pid); - std::shared_ptr replicator = GetReplicator(tid, pid); - if (replicator) { - replicator->SetSnapshotLogPartIndex(offset); - } - base::SetResponseOK(response); -} - void TabletImpl::ExtractIndexData(RpcController* controller, const ::openmldb::api::ExtractIndexDataRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done) { brpc::ClosureGuard done_guard(done); std::shared_ptr<::openmldb::api::TaskInfo> task_ptr; if (request->has_task_info() && request->task_info().IsInitialized()) { - if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kExtractIndexData, task_ptr) < 0) { + if (AddOPTask(request->task_info(), ::openmldb::api::TaskType::kExtractIndexRequest, task_ptr) < 0) { base::SetResponseStatus(-1, "add task failed", response); return; } @@ -5214,35 +5071,24 @@ void TabletImpl::ExtractIndexData(RpcController* controller, const ::openmldb::a break; } } + std::vector<::openmldb::common::ColumnKey> index_vec; + for (const auto& cur_column_key : request->column_key()) { + index_vec.push_back(cur_column_key); + } auto memtable_snapshot = std::static_pointer_cast<::openmldb::storage::MemTableSnapshot>(snapshot); - task_pool_.AddTask(boost::bind(&TabletImpl::ExtractIndexDataInternal, this, table, memtable_snapshot, - request->column_key(), request->idx(), request->partition_num(), task_ptr)); + if (IsClusterMode()) { + task_pool_.AddTask(boost::bind(&TabletImpl::ExtractIndexDataInternal, this, table, memtable_snapshot, + index_vec, request->partition_num(), request->offset(), request->dump_data(), task_ptr)); + } else { + ExtractIndexDataInternal(table, memtable_snapshot, index_vec, request->partition_num(), + request->offset(), false, nullptr); + } base::SetResponseOK(response); return; } while (0); SetTaskStatus(task_ptr, ::openmldb::api::TaskStatus::kFailed); } -void TabletImpl::ExtractIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, - std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, - ::openmldb::common::ColumnKey& column_key, uint32_t idx, - uint32_t partition_num, std::shared_ptr<::openmldb::api::TaskInfo> task) { - uint64_t offset = 0; - uint32_t tid = table->GetId(); - uint32_t pid = table->GetPid(); - if (memtable_snapshot->ExtractIndexData(table, column_key, idx, partition_num, offset) < 0) { - PDLOG(WARNING, "fail to extract index. tid %u pid %u", tid, pid); - SetTaskStatus(task, ::openmldb::api::TaskStatus::kFailed); - return; - } - PDLOG(INFO, "extract index success. tid %u pid %u", tid, pid); - std::shared_ptr replicator = GetReplicator(tid, pid); - if (replicator) { - replicator->SetSnapshotLogPartIndex(offset); - } - SetTaskStatus(task, ::openmldb::api::TaskStatus::kDone); -} - void TabletImpl::AddIndex(RpcController* controller, const ::openmldb::api::AddIndexRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done) { brpc::ClosureGuard done_guard(done); diff --git a/src/tablet/tablet_impl.h b/src/tablet/tablet_impl.h index 57be9e7aef2..d48f192ae26 100644 --- a/src/tablet/tablet_impl.h +++ b/src/tablet/tablet_impl.h @@ -200,18 +200,12 @@ class TabletImpl : public ::openmldb::api::TabletServer { void DeleteIndex(RpcController* controller, const ::openmldb::api::DeleteIndexRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done); - void DumpIndexData(RpcController* controller, const ::openmldb::api::DumpIndexDataRequest* request, - ::openmldb::api::GeneralResponse* response, Closure* done); - void LoadIndexData(RpcController* controller, const ::openmldb::api::LoadIndexDataRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done); void ExtractIndexData(RpcController* controller, const ::openmldb::api::ExtractIndexDataRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done); - void ExtractMultiIndexData(RpcController* controller, const ::openmldb::api::ExtractMultiIndexDataRequest* request, - ::openmldb::api::GeneralResponse* response, Closure* done); - void AddIndex(RpcController* controller, const ::openmldb::api::AddIndexRequest* request, ::openmldb::api::GeneralResponse* response, Closure* done); @@ -327,20 +321,22 @@ class TabletImpl : public ::openmldb::api::TabletServer { void DumpIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, uint32_t partition_num, - ::openmldb::common::ColumnKey& column_key, // NOLINT - uint32_t idx, std::shared_ptr<::openmldb::api::TaskInfo> task); + const std::vector<::openmldb::common::ColumnKey>& column_keys, + uint64_t offset, std::shared_ptr<::openmldb::api::TaskInfo> task); void SendIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, const std::map& pid_endpoint_map, std::shared_ptr<::openmldb::api::TaskInfo> task); - void LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_pid, uint32_t partition_num, uint64_t last_time, - std::shared_ptr<::openmldb::api::TaskInfo> task); + void LoadIndexDataInternal(uint32_t tid, uint32_t pid, uint32_t cur_pid, + uint32_t partition_num, uint64_t last_time, + std::shared_ptr<::openmldb::api::TaskInfo> task); void ExtractIndexDataInternal(std::shared_ptr<::openmldb::storage::Table> table, - std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, - ::openmldb::common::ColumnKey& column_key, uint32_t idx, // NOLINT - uint32_t partition_num, std::shared_ptr<::openmldb::api::TaskInfo> task); + std::shared_ptr<::openmldb::storage::MemTableSnapshot> memtable_snapshot, + const std::vector<::openmldb::common::ColumnKey>& column_key, + uint32_t partition_num, uint64_t offset, bool contain_dump, + std::shared_ptr<::openmldb::api::TaskInfo> task); void SchedMakeSnapshot(); @@ -377,15 +373,10 @@ class TabletImpl : public ::openmldb::api::TabletServer { void SetTaskStatus(std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, // NOLINT ::openmldb::api::TaskStatus status); - int GetTaskStatus(std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, // NOLINT + int GetTaskStatus(const std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr, ::openmldb::api::TaskStatus* status); - std::shared_ptr<::openmldb::api::TaskInfo> FindTask(uint64_t op_id, ::openmldb::api::TaskType task_type); - - int AddOPMultiTask(const ::openmldb::api::TaskInfo& task_info, ::openmldb::api::TaskType task_type, - std::shared_ptr<::openmldb::api::TaskInfo>& task_ptr); // NOLINT - - std::shared_ptr<::openmldb::api::TaskInfo> FindMultiTask(const ::openmldb::api::TaskInfo& task_info); + bool IsExistTaskUnLock(const ::openmldb::api::TaskInfo& task); int CheckDimessionPut(const ::openmldb::api::PutRequest* request, uint32_t idx_cnt); diff --git a/src/tablet/tablet_impl_test.cc b/src/tablet/tablet_impl_test.cc index be48ba8354d..9bd004f4763 100644 --- a/src/tablet/tablet_impl_test.cc +++ b/src/tablet/tablet_impl_test.cc @@ -5019,129 +5019,6 @@ TEST_F(TabletImplTest, DelRecycleDisk) { } } -TEST_P(TabletImplTest, DumpIndex) { - ::openmldb::common::StorageMode storage_mode = GetParam(); - int old_offset = FLAGS_make_snapshot_threshold_offset; - FLAGS_make_snapshot_threshold_offset = 0; - uint32_t id = counter++; - MockClosure closure; - TabletImpl tablet; - tablet.Init(""); - ::openmldb::api::CreateTableRequest request; - ::openmldb::api::TableMeta* table_meta = request.mutable_table_meta(); - ::openmldb::common::ColumnDesc* desc = table_meta->add_column_desc(); - desc->set_name("card"); - desc->set_data_type(::openmldb::type::kString); - desc = table_meta->add_column_desc(); - desc->set_name("mcc"); - desc->set_data_type(::openmldb::type::kString); - desc = table_meta->add_column_desc(); - desc->set_name("price"); - desc->set_data_type(::openmldb::type::kBigInt); - desc = table_meta->add_column_desc(); - desc->set_name("ts1"); - desc->set_data_type(::openmldb::type::kBigInt); - desc = table_meta->add_column_desc(); - desc->set_name("ts2"); - desc->set_data_type(::openmldb::type::kBigInt); - SchemaCodec::SetIndex(table_meta->add_column_key(), "index1", "card", "ts1", ::openmldb::type::kAbsoluteTime, 0, 0); - SchemaCodec::SetIndex(table_meta->add_column_key(), "index2", "card", "ts2", ::openmldb::type::kAbsoluteTime, 0, 0); - - table_meta->set_name("t0"); - table_meta->set_tid(id); - table_meta->set_pid(1); - table_meta->set_storage_mode(storage_mode); - ::openmldb::api::CreateTableResponse response; - tablet.CreateTable(NULL, &request, &response, &closure); - ASSERT_EQ(0, response.code()); - - for (int i = 0; i < 10; i++) { - std::vector input; - input.push_back("card" + std::to_string(i)); - input.push_back("mcc" + std::to_string(i)); - input.push_back(std::to_string(i)); - input.push_back(std::to_string(i + 100)); - input.push_back(std::to_string(i + 10000)); - std::string value; - ::openmldb::codec::RowCodec::EncodeRow(input, table_meta->column_desc(), 1, value); - ::openmldb::api::PutRequest request; - request.set_value(value); - request.set_tid(id); - request.set_pid(1); - - ::openmldb::api::Dimension* d = request.add_dimensions(); - d->set_key(input[0]); - d->set_idx(0); - ::openmldb::api::TSDimension* tsd = request.add_ts_dimensions(); - tsd->set_ts(i + 100); - tsd->set_idx(0); - tsd = request.add_ts_dimensions(); - tsd->set_ts(i + 10000); - tsd->set_idx(1); - ::openmldb::api::PutResponse response; - MockClosure closure; - tablet.Put(NULL, &request, &response, &closure); - ASSERT_EQ(0, response.code()); - } - ::openmldb::api::GeneralRequest grq; - grq.set_tid(id); - grq.set_pid(1); - grq.set_storage_mode(storage_mode); - ::openmldb::api::GeneralResponse grp; - grp.set_code(-1); - tablet.MakeSnapshot(NULL, &grq, &grp, &closure); - sleep(2); - for (int i = 0; i < 10; i++) { - std::vector input; - input.push_back("card" + std::to_string(i)); - input.push_back("mcc" + std::to_string(i)); - input.push_back(std::to_string(i)); - input.push_back(std::to_string(i + 200)); - input.push_back(std::to_string(i + 20000)); - std::string value; - ::openmldb::codec::RowCodec::EncodeRow(input, table_meta->column_desc(), 1, value); - ::openmldb::api::PutRequest request; - request.set_value(value); - request.set_tid(id); - request.set_pid(1); - ::openmldb::api::Dimension* d = request.add_dimensions(); - d->set_key(input[0]); - d->set_idx(0); - ::openmldb::api::TSDimension* tsd = request.add_ts_dimensions(); - tsd->set_ts(i + 100); - tsd->set_idx(0); - tsd = request.add_ts_dimensions(); - tsd->set_ts(i + 10000); - tsd->set_idx(1); - ::openmldb::api::PutResponse response; - MockClosure closure; - tablet.Put(NULL, &request, &response, &closure); - ASSERT_EQ(0, response.code()); - } - { - ::openmldb::api::DumpIndexDataRequest dump_request; - dump_request.set_tid(id); - dump_request.set_pid(1); - dump_request.set_partition_num(8); - dump_request.set_idx(1); - auto column_key = dump_request.mutable_column_key(); - column_key->set_index_name("card|mcc"); - column_key->add_col_name("card"); - column_key->add_col_name("mcc"); - column_key->set_ts_name("ts2"); - ::openmldb::api::GeneralResponse dump_response; - tablet.DumpIndexData(NULL, &dump_request, &dump_response, &closure); - // Some functions in tablet_impl only support memtable now - // refer to issue #1438 - if (storage_mode == openmldb::common::kMemory) { - ASSERT_EQ(0, dump_response.code()); - } else { - ASSERT_EQ(701, dump_response.code()); - } - } - FLAGS_make_snapshot_threshold_offset = old_offset; -} - TEST_P(TabletImplTest, SendIndexData) { ::openmldb::common::StorageMode storage_mode = GetParam(); // only support Memtable now diff --git a/test/integration-test/openmldb-test-python/ha_cases/test_addindex.py b/test/integration-test/openmldb-test-python/ha_cases/test_addindex.py index b62e8b321e4..f813f799388 100644 --- a/test/integration-test/openmldb-test-python/ha_cases/test_addindex.py +++ b/test/integration-test/openmldb-test-python/ha_cases/test_addindex.py @@ -71,9 +71,10 @@ def test_addindex(self, storage_mode, snapshot): key2 = "key2" + str(i) self.cursor.execute(f"insert into {table_name} values (\'{key1}\', \'{key2}\', {ts})"); + time.sleep(2) result = self.cursor.execute(f"create index index2 on {table_name} (col2) options (ts=col3)") time.sleep(1) - assert self.executor.WaitingTableOP(database, table_name, partition_num).OK() + assert self.executor.WaitingOP(database, table_name, 0).OK() status, indexs = self.executor.GetIndexs(database, table_name) assert status.OK() and len(indexs) == 2 assert indexs[1].GetName() == "index2" and indexs[1].GetTsCol() == "col3"