diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index 9291af86e..11394824f 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -740,7 +740,7 @@ if (NOT WIN32) SET(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "server") SET(DIST_INCLUDE_FILES "tools/errors.h" "tools/serializer.h" "tools/varint.h" "tools/stringstools.h" "tools/customhash.h" "tools/assertrx.h" "tools/jsonstring.h" - "tools/verifying_updater.h" + "tools/verifying_updater.h" "tools/customlocal.h" "core/reindexer.h" "core/type_consts.h" "core/item.h" "core/payload/payloadvalue.h" "core/payload/payloadiface.h" "core/indexopts.h" "core/namespacedef.h" "core/keyvalue/variant.h" "core/keyvalue/geometry.h" "core/sortingprioritiestable.h" "core/rdxcontext.h" "core/activity_context.h" "core/activity.h" "core/activitylog.h" "core/type_consts_helpers.h" "core/payload/fieldsset.h" "core/payload/payloadtype.h" diff --git a/cpp_src/client/coroqueryresults.cc b/cpp_src/client/coroqueryresults.cc index 008654841..982ab6c54 100644 --- a/cpp_src/client/coroqueryresults.cc +++ b/cpp_src/client/coroqueryresults.cc @@ -87,15 +87,21 @@ void CoroQueryResults::Bind(std::string_view rawResult, RPCQrId id, const Query PayloadType("tmp").clone()->deserialize(ser); }, opts, i_.parsingData_); + + const auto copyStart = i_.lazyMode_ ? rawResult.begin() : (rawResult.begin() + ser.Pos()); + const auto rawResLen = std::distance(copyStart, rawResult.end()); + if rx_unlikely (rawResLen > QrRawBuffer::max_size()) { + throw Error( + errLogic, + "client::QueryResults::Bind: rawResult buffer overflow. Max size if %d bytes, but %d bytes requested. Try to reduce " + "fetch limit (current limit is %d)", + QrRawBuffer::max_size(), rawResLen, i_.fetchAmount_); + } + + i_.rawResult_.assign(copyStart, rawResult.end()); } catch (const Error &err) { i_.status_ = err; } - - if (i_.lazyMode_) { - i_.rawResult_.assign(rawResult.begin(), rawResult.end()); - } else { - i_.rawResult_.assign(rawResult.begin() + ser.Pos(), rawResult.end()); - } } void CoroQueryResults::fetchNextResults() { @@ -120,11 +126,15 @@ void CoroQueryResults::handleFetchedBuf(net::cproto::CoroRPCAnswer &ans) { ResultSerializer ser(rawResult); ser.GetRawQueryParams(i_.queryParams_, nullptr, ResultSerializer::Options{}, i_.parsingData_); - if (i_.lazyMode_) { - i_.rawResult_.assign(rawResult.begin(), rawResult.end()); - } else { - i_.rawResult_.assign(rawResult.begin() + ser.Pos(), rawResult.end()); - } + const auto copyStart = i_.lazyMode_ ? rawResult.begin() : (rawResult.begin() + ser.Pos()); + const auto rawResLen = std::distance(copyStart, rawResult.end()); + if rx_unlikely (rawResLen > QrRawBuffer::max_size()) { + throw Error(errLogic, + "client::QueryResults::fetchNextResults: rawResult buffer overflow. Max size if %d bytes, but %d bytes requested. Try " + "to reduce fetch limit (current limit is %d)", + QrRawBuffer::max_size(), rawResLen, i_.fetchAmount_); + } + i_.rawResult_.assign(copyStart, rawResult.end()); i_.status_ = Error(); } diff --git a/cpp_src/client/itemimplbase.cc b/cpp_src/client/itemimplbase.cc index c33f92004..3ceb336c0 100644 --- a/cpp_src/client/itemimplbase.cc +++ b/cpp_src/client/itemimplbase.cc @@ -59,7 +59,7 @@ void ItemImplBase::FromCJSON(std::string_view slice) { const auto tupleSize = ser_.Len(); tupleHolder_ = ser_.DetachBuf(); tupleData_ = std::string_view(reinterpret_cast(tupleHolder_.get()), tupleSize); - pl.Set(0, Variant(p_string(&tupleData_))); + pl.Set(0, Variant(p_string(&tupleData_), Variant::no_hold_t{})); } Error ItemImplBase::FromJSON(std::string_view slice, char **endp, bool /*pkOnly*/) { @@ -95,7 +95,7 @@ Error ItemImplBase::FromJSON(std::string_view slice, char **endp, bool /*pkOnly* const auto tupleSize = ser_.Len(); tupleHolder_ = ser_.DetachBuf(); tupleData_ = std::string_view(reinterpret_cast(tupleHolder_.get()), tupleSize); - pl.Set(0, Variant(p_string(&tupleData_))); + pl.Set(0, Variant(p_string(&tupleData_), Variant::no_hold_t{})); } return err; } @@ -116,7 +116,7 @@ Error ItemImplBase::FromMsgPack(std::string_view buf, size_t &offset) { const auto tupleSize = ser_.Len(); tupleHolder_ = ser_.DetachBuf(); tupleData_ = std::string_view(reinterpret_cast(tupleHolder_.get()), tupleSize); - pl.Set(0, Variant(p_string(&tupleData_))); + pl.Set(0, Variant(p_string(&tupleData_), Variant::no_hold_t{})); } return err; } diff --git a/cpp_src/client/namespace.h b/cpp_src/client/namespace.h index e26c2e6b6..398834e67 100644 --- a/cpp_src/client/namespace.h +++ b/cpp_src/client/namespace.h @@ -19,11 +19,11 @@ class Namespace { template Item NewItem(ClientT& client, std::chrono::milliseconds execTimeout); void TryReplaceTagsMatcher(TagsMatcher&& tm, bool checkVersion = true); - TagsMatcher GetTagsMatcher() const { + TagsMatcher GetTagsMatcher() const noexcept { shared_lock lk(lck_); return tagsMatcher_; } - int GetStateToken() const { + int GetStateToken() const noexcept { shared_lock lk(lck_); return int(tagsMatcher_.stateToken()); } diff --git a/cpp_src/client/queryresults.cc b/cpp_src/client/queryresults.cc index 27febd26f..5b0c03315 100644 --- a/cpp_src/client/queryresults.cc +++ b/cpp_src/client/queryresults.cc @@ -61,8 +61,8 @@ QueryResults::~QueryResults() { results_.setClosed(); } -TagsMatcher QueryResults::GetTagsMatcher(int nsid) const { return results_.GetTagsMatcher(nsid); } -TagsMatcher QueryResults::GetTagsMatcher(std::string_view ns) const { return results_.GetTagsMatcher(ns); } +TagsMatcher QueryResults::GetTagsMatcher(int nsid) const noexcept { return results_.GetTagsMatcher(nsid); } +TagsMatcher QueryResults::GetTagsMatcher(std::string_view ns) const noexcept { return results_.GetTagsMatcher(ns); } } // namespace client } // namespace reindexer diff --git a/cpp_src/client/queryresults.h b/cpp_src/client/queryresults.h index 24b49f9dd..8924b0690 100644 --- a/cpp_src/client/queryresults.h +++ b/cpp_src/client/queryresults.h @@ -81,10 +81,10 @@ class QueryResults { bool IsCacheEnabled() const noexcept { return results_.IsCacheEnabled(); } int GetMergedNSCount() const noexcept { return results_.GetMergedNSCount(); } - TagsMatcher GetTagsMatcher(int nsid) const; - TagsMatcher GetTagsMatcher(std::string_view ns) const; - PayloadType GetPayloadType(int nsid) const { return results_.GetPayloadType(nsid); } - PayloadType GetPayloadType(std::string_view ns) const { return results_.GetPayloadType(ns); } + TagsMatcher GetTagsMatcher(int nsid) const noexcept; + TagsMatcher GetTagsMatcher(std::string_view ns) const noexcept; + PayloadType GetPayloadType(int nsid) const noexcept { return results_.GetPayloadType(nsid); } + PayloadType GetPayloadType(std::string_view ns) const noexcept { return results_.GetPayloadType(ns); } int GetFormat() const noexcept { return results_.GetFormat(); } int GetFlags() const noexcept { return results_.GetFlags(); } diff --git a/cpp_src/cluster/config.cc b/cpp_src/cluster/config.cc index 24bba7824..b47879aa8 100644 --- a/cpp_src/cluster/config.cc +++ b/cpp_src/cluster/config.cc @@ -866,6 +866,7 @@ Error ShardingConfig::FromYAML(const std::string &yaml) { thisShardId = root["this_shard_id"].as(); reconnectTimeout = std::chrono::milliseconds(root["reconnect_timeout_msec"].as(reconnectTimeout.count())); shardsAwaitingTimeout = std::chrono::seconds(root["shards_awaiting_timeout_sec"].as(shardsAwaitingTimeout.count())); + configRollbackTimeout = std::chrono::seconds(root["config_rollback_timeout_sec"].as(configRollbackTimeout.count())); proxyConnCount = root["proxy_conn_count"].as(proxyConnCount); proxyConnConcurrency = root["proxy_conn_concurrency"].as(proxyConnConcurrency); proxyConnThreads = root["proxy_conn_threads"].as(proxyConnThreads); @@ -922,6 +923,7 @@ Error ShardingConfig::FromJSON(const gason::JsonNode &root) { thisShardId = root["this_shard_id"].As(); reconnectTimeout = std::chrono::milliseconds(root["reconnect_timeout_msec"].As(reconnectTimeout.count())); shardsAwaitingTimeout = std::chrono::seconds(root["shards_awaiting_timeout_sec"].As(shardsAwaitingTimeout.count())); + configRollbackTimeout = std::chrono::seconds(root["config_rollback_timeout_sec"].As(configRollbackTimeout.count())); proxyConnCount = root["proxy_conn_count"].As(proxyConnCount); proxyConnConcurrency = root["proxy_conn_concurrency"].As(proxyConnConcurrency); proxyConnThreads = root["proxy_conn_threads"].As(proxyConnThreads); @@ -937,8 +939,9 @@ Error ShardingConfig::FromJSON(const gason::JsonNode &root) { bool operator==(const ShardingConfig &lhs, const ShardingConfig &rhs) { return lhs.namespaces == rhs.namespaces && lhs.thisShardId == rhs.thisShardId && lhs.shards == rhs.shards && lhs.reconnectTimeout == rhs.reconnectTimeout && lhs.shardsAwaitingTimeout == rhs.shardsAwaitingTimeout && - lhs.proxyConnCount == rhs.proxyConnCount && lhs.proxyConnConcurrency == rhs.proxyConnConcurrency && - rhs.proxyConnThreads == lhs.proxyConnThreads && rhs.sourceId == lhs.sourceId; + lhs.configRollbackTimeout == rhs.configRollbackTimeout && lhs.proxyConnCount == rhs.proxyConnCount && + lhs.proxyConnConcurrency == rhs.proxyConnConcurrency && rhs.proxyConnThreads == lhs.proxyConnThreads && + rhs.sourceId == lhs.sourceId; } bool operator==(const ShardingConfig::Key &lhs, const ShardingConfig::Key &rhs) { return lhs.shardId == rhs.shardId && lhs.algorithmType == rhs.algorithmType && lhs.RelaxCompare(rhs.values) == 0; @@ -972,6 +975,7 @@ YAML::Node ShardingConfig::GetYAMLObj() const { yaml["this_shard_id"] = thisShardId; yaml["reconnect_timeout_msec"] = reconnectTimeout.count(); yaml["shards_awaiting_timeout_sec"] = shardsAwaitingTimeout.count(); + yaml["config_rollback_timeout_sec"] = configRollbackTimeout.count(); yaml["proxy_conn_count"] = proxyConnCount; yaml["proxy_conn_concurrency"] = proxyConnConcurrency; yaml["proxy_conn_threads"] = proxyConnThreads; @@ -1015,6 +1019,7 @@ void ShardingConfig::GetJSON(JsonBuilder &jb) const { jb.Put("this_shard_id", thisShardId); jb.Put("reconnect_timeout_msec", reconnectTimeout.count()); jb.Put("shards_awaiting_timeout_sec", shardsAwaitingTimeout.count()); + jb.Put("config_rollback_timeout_sec", configRollbackTimeout.count()); jb.Put("proxy_conn_count", proxyConnCount); jb.Put("proxy_conn_concurrency", proxyConnConcurrency); jb.Put("proxy_conn_threads", proxyConnThreads); diff --git a/cpp_src/cluster/config.h b/cpp_src/cluster/config.h index 43ca32365..29bfebdbe 100644 --- a/cpp_src/cluster/config.h +++ b/cpp_src/cluster/config.h @@ -236,6 +236,7 @@ struct ShardingConfig { int thisShardId = ShardingKeyType::ProxyOff; std::chrono::milliseconds reconnectTimeout = std::chrono::milliseconds(3000); std::chrono::seconds shardsAwaitingTimeout = std::chrono::seconds(30); + std::chrono::seconds configRollbackTimeout = std::chrono::seconds(30); int proxyConnCount = kDefaultShardingProxyConnCount; int proxyConnConcurrency = kDefaultShardingProxyCoroPerConn; int proxyConnThreads = kDefaultShardingProxyConnThreads; diff --git a/cpp_src/cluster/raftmanager.cc b/cpp_src/cluster/raftmanager.cc index bb31f3c77..8f3d51c79 100644 --- a/cpp_src/cluster/raftmanager.cc +++ b/cpp_src/cluster/raftmanager.cc @@ -34,29 +34,32 @@ void RaftManager::Configure(const ReplicationConfigData& baseConfig, const Clust } Error RaftManager::SendDesiredLeaderId(int nextServerId) { - logTrace("%d SendDesiredLeaderId nextLeaderId = %d", serverId_, nextServerId); + std::vector tmpClients; // Using separated clients sete here to avoid intersection with Leader's pings + std::shared_ptr CloseConnection(nullptr, [this, &tmpClients](void*) { + coroutine::wait_group wgStop; + for (auto& client : tmpClients) { + loop_.spawn(wgStop, [&client]() { client.Stop(); }); + } + wgStop.wait(); + }); + + client::ReindexerConfig rpcCfg; + rpcCfg.AppName = "raft_manager_tmp"; + rpcCfg.NetTimeout = kRaftTimeout; + rpcCfg.EnableCompression = false; + rpcCfg.RequestDedicatedThread = true; + tmpClients.reserve(nodes_.size()); size_t nextServerNodeIndex = nodes_.size(); - for (size_t i = 0; i < nodes_.size(); i++) { + for (size_t i = 0; i < nodes_.size(); ++i) { + auto& client = tmpClients.emplace_back(rpcCfg); + auto err = client.Connect(nodes_[i].dsn, loop_); + (void)err; // Ignore connection errors. Handle them on the status phase if (nodes_[i].serverId == nextServerId) { nextServerNodeIndex = i; - break; - } - } - - std::shared_ptr CloseConnection(nullptr, [this](void*) { - if (GetLeaderId() != serverId_) { // leader node clients, used for pinging - coroutine::wait_group wgStop; - for (auto& node : nodes_) { - loop_.spawn(wgStop, [&node]() { node.client.Stop(); }); + err = client.WithTimeout(kDesiredLeaderTimeout).Status(true); + if (!err.ok()) { + return Error(err.code(), "Target node %s is not available.", nodes_[i].dsn); } - wgStop.wait(); - } - }); - - if (nextServerNodeIndex != nodes_.size()) { - Error err = clientStatus(nextServerNodeIndex, kDesiredLeaderTimeout); - if (!err.ok()) { - return Error(err.code(), "Target node %s is not available.", nodes_[nextServerNodeIndex].dsn); } } @@ -64,15 +67,20 @@ Error RaftManager::SendDesiredLeaderId(int nextServerId) { coroutine::wait_group wg; std::string errString; - for (size_t nodeId = 0; nodeId < nodes_.size(); ++nodeId) { + auto sendDesiredServerIdToNode = [](client::RaftClient& client, int nextServerId) { + auto err = client.WithTimeout(kDesiredLeaderTimeout).Status(true); + return !err.ok() ? err : client.WithTimeout(kDesiredLeaderTimeout).SetDesiredLeaderId(nextServerId); + }; + + for (size_t nodeId = 0; nodeId < tmpClients.size(); ++nodeId) { if (nodeId == nextServerNodeIndex) { continue; } - loop_.spawn(wg, [this, nodeId, nextServerId, &errString, &okCount] { + loop_.spawn(wg, [this, nodeId, nextServerId, &errString, &okCount, &tmpClients, &sendDesiredServerIdToNode] { try { - const auto err = sendDesiredServerIdToNode(nodeId, nextServerId); - if (err.ok()) { + logTrace("%d Sending desired server ID (%d) to node with server ID %d", serverId_, nextServerId, nodes_[nodeId].serverId); + if (auto err = sendDesiredServerIdToNode(tmpClients[nodeId], nextServerId); err.ok()) { ++okCount; } else { errString += "[" + err.what() + "]"; @@ -84,33 +92,45 @@ Error RaftManager::SendDesiredLeaderId(int nextServerId) { } wg.wait(); if (nextServerNodeIndex != nodes_.size()) { - Error err = sendDesiredServerIdToNode(nextServerNodeIndex, nextServerId); - if (!err.ok()) { + logTrace("%d Sending desired server ID (%d) to node with server ID %d", serverId_, nextServerId, nextServerId); + if (auto err = sendDesiredServerIdToNode(tmpClients[nextServerNodeIndex], nextServerId); !err.ok()) { return err; } okCount++; } if (okCount >= GetConsensusForN(nodes_.size() + 1)) { - return errOK; + return Error(); } return Error(errNetwork, "Can't send nextLeaderId to servers okCount %d err: %s", okCount, errString); } -RaftInfo::Role RaftManager::Elections() { +void RaftManager::SetDesiredLeaderId(int serverId) { + logInfo("%d Set (%d) as a desired leader", serverId_, serverId); + nextServerId_.SetNextServerId(serverId); + lastLeaderPingTs_ = {ClockT::time_point()}; +} + +RaftInfo::Role RaftManager::Elections(bool forceElectionsRestart) { std::vector succeedRoutines; succeedRoutines.reserve(nodes_.size()); while (!terminate_.load()) { const int nextServerId = nextServerId_.GetNextServerId(); + const bool isDesiredLeader = (nextServerId == serverId_); + if (!isDesiredLeader) { + if (nextServerId != -1) { + endElections(GetTerm(), RaftInfo::Role::Follower); + logInfo("Skipping elections (desired leader id is %d)", serverId_, nextServerId); + return RaftInfo::Role::Follower; + } + if (!forceElectionsRestart && hasRecentLeadersPing(ClockT::now()) && endElections(GetTerm(), RaftInfo::Role::Follower)) { + logInfo("Skipping elections (node is already connected to the active leader (%d))", serverId_, nextServerId); + return RaftInfo::Role::Follower; + } + } int32_t term = beginElectionsTerm(nextServerId); logInfo("Starting new elections term for %d. Term number: %d", serverId_, term); - if (nextServerId != -1 && nextServerId != serverId_) { - endElections(term, RaftInfo::Role::Follower); - logInfo("Skipping elections (desired leader id is %d)", serverId_, nextServerId); - return RaftInfo::Role::Follower; - } - const bool isDesiredLeader = (nextServerId == serverId_); coroutine::wait_group wg; succeedRoutines.resize(0); struct { @@ -122,7 +142,7 @@ RaftInfo::Role RaftManager::Elections() { loop_.spawn(wg, [this, &electionsStat, nodeId, term, &succeedRoutines, isDesiredLeader] { auto& node = nodes_[nodeId]; if (!node.client.Status().ok()) { - node.client.Connect(node.dsn, loop_, client::ConnectOpts().WithExpectedClusterID(clusterID_)); + node.client.Connect(node.dsn, loop_, createConnectionOpts()); } NodeData suggestion, result; suggestion.serverId = serverId_; @@ -201,8 +221,8 @@ RaftInfo::Role RaftManager::Elections() { return RaftInfo::Role::Follower; } -bool RaftManager::LeaderIsAvailable(ClockT::time_point now) { - return ((now - lastLeaderPingTs_.load()) < kMinLeaderAwaitInterval) || (GetRole() == RaftInfo::Role::Leader); +bool RaftManager::LeaderIsAvailable(ClockT::time_point now) const noexcept { + return hasRecentLeadersPing(now) || (GetRole() == RaftInfo::Role::Leader); } bool RaftManager::FollowersAreAvailable() { @@ -392,24 +412,8 @@ bool RaftManager::endElections(int32_t term, RaftInfo::Role result) { bool RaftManager::isConsensus(size_t num) const noexcept { return num >= GetConsensusForN(nodes_.size() + 1); } -Error RaftManager::clientStatus(size_t index, std::chrono::seconds timeout) { - Error err; - if (!nodes_[index].client.WithTimeout(timeout).Status(true).ok()) { - err = nodes_[index].client.Connect(nodes_[index].dsn, loop_, client::ConnectOpts().WithExpectedClusterID(clusterID_)); - if (err.ok()) { - err = nodes_[index].client.WithTimeout(timeout).Status(true); - } - } - return err; -} - -Error RaftManager::sendDesiredServerIdToNode(size_t index, int nextServerId) { - Error err = clientStatus(index, kDesiredLeaderTimeout); - if (!err.ok()) { - return err; - } - logTrace("%d Sending desired server ID (%d) to node with server ID %d", serverId_, nextServerId, nodes_[index].serverId); - return nodes_[index].client.WithTimeout(kDesiredLeaderTimeout).SetDesiredLeaderId(nextServerId); +bool RaftManager::hasRecentLeadersPing(RaftManager::ClockT::time_point now) const noexcept { + return (now - lastLeaderPingTs_.load()) < kMinLeaderAwaitInterval; } } // namespace cluster diff --git a/cpp_src/cluster/raftmanager.h b/cpp_src/cluster/raftmanager.h index f0132958d..00077cf00 100644 --- a/cpp_src/cluster/raftmanager.h +++ b/cpp_src/cluster/raftmanager.h @@ -23,18 +23,15 @@ class RaftManager { void SetTerminateFlag(bool val) noexcept { terminate_ = val; } void Configure(const ReplicationConfigData &, const ClusterConfigData &); - RaftInfo::Role Elections(); - bool LeaderIsAvailable(ClockT::time_point now); + RaftInfo::Role Elections(bool forceElectionsRestart); + bool LeaderIsAvailable(ClockT::time_point now) const noexcept; bool FollowersAreAvailable(); int32_t GetLeaderId() const { return getLeaderId(voteData_.load()); } RaftInfo::Role GetRole() const { return getRole(voteData_.load()); } int32_t GetTerm() const { return getTerm(voteData_.load()); } Error SuggestLeader(const cluster::NodeData &suggestion, cluster::NodeData &response); Error SendDesiredLeaderId(int serverId); - void SetDesiredLeaderId(int serverId) { - nextServerId_.SetNextServerId(serverId); - lastLeaderPingTs_ = {ClockT::time_point()}; - } + void SetDesiredLeaderId(int serverId); int GetDesiredLeaderId() { return nextServerId_.GetNextServerId(); } Error LeadersPing(const cluster::NodeData &); void AwaitTermination(); @@ -86,8 +83,8 @@ class RaftManager { int32_t beginElectionsTerm(int presetLeader); bool endElections(int32_t term, RaftInfo::Role result); bool isConsensus(size_t num) const noexcept; - Error sendDesiredServerIdToNode(size_t index, int nextServerId); - Error clientStatus(size_t index, std::chrono::seconds timeout); + bool hasRecentLeadersPing(ClockT::time_point now) const noexcept; + client::ConnectOpts createConnectionOpts() const { return client::ConnectOpts().WithExpectedClusterID(clusterID_); } net::ev::dynamic_loop &loop_; ReplicationStatsCollector statsCollector_; diff --git a/cpp_src/cluster/replication/clusterdatareplicator.cc b/cpp_src/cluster/replication/clusterdatareplicator.cc index e4d6873fe..4f5ac1586 100644 --- a/cpp_src/cluster/replication/clusterdatareplicator.cc +++ b/cpp_src/cluster/replication/clusterdatareplicator.cc @@ -100,7 +100,11 @@ void ClusterDataReplicator::Run() { syncList_.Init(std::vector(nodes.size(), SynchronizationList::kUnsynchronizedID)); updatesQueue_.ReinitSyncQueue(statsCollector_, std::optional(config_->namespaces), log_); - sharedSyncState_.Reset(config_->namespaces, 1, config_->nodes.size() - 1); + const bool isSyncCluster = config_->nodes.size() > 1; + sharedSyncState_.Reset(config_->namespaces, 1, isSyncCluster); + if (isSyncCluster) { + onRoleChanged(RaftInfo::Role::Candidate, -1); + } assert(replThreads_.empty()); ReplThreadConfig threadsConfig(baseConfig_.value(), config_.value()); @@ -249,10 +253,11 @@ void ClusterDataReplicator::clusterControlRoutine(int serverId) { while (!terminate_) { onRoleChanged(RaftInfo::Role::Candidate, raftInfo.role == RaftInfo::Role::Leader ? serverId : raftInfo.leaderId); raftInfo.role = RaftInfo::Role::Candidate; + const bool forceElectionsRestart = restartElections_; restartElections_ = false; RaftInfo newRaftInfo; - newRaftInfo.role = raftManager_.Elections(); + newRaftInfo.role = raftManager_.Elections(forceElectionsRestart); const int desiredLeaderId = raftManager_.GetDesiredLeaderId(); if (desiredLeaderId == serverId && newRaftInfo.role != RaftInfo::Role::Leader) { continue; @@ -290,7 +295,9 @@ void ClusterDataReplicator::clusterControlRoutine(int serverId) { if (c.id == kCmdSetDesiredLeader) { Error err; if (c.send) { + logInfo("%d Sending desired leader(%d) to other nodes", serverId, c.serverId); err = raftManager_.SendDesiredLeaderId(c.serverId); + logInfo("%d Desired leader(%d) send result: %s", serverId, c.serverId, err.ok() ? "OK" : err.what()); } if (err.ok()) { raftManager_.SetDesiredLeaderId(c.serverId); diff --git a/cpp_src/cluster/sharding/sharding.conf b/cpp_src/cluster/sharding/sharding.conf index 9aa4f3390..2ff1a42e7 100644 --- a/cpp_src/cluster/sharding/sharding.conf +++ b/cpp_src/cluster/sharding/sharding.conf @@ -74,6 +74,9 @@ reconnect_timeout_msec: 3000 # 0 - disables awaiting mechanism and -1 means "unlimited timeout" shards_awaiting_timeout_sec: 30 +# Idle timeout for the sharding config candidate auto rollback (in cases, when source node did not approved/rolled back this config candidate by itself) +config_rollback_timeout_sec: 30 + # Count of proxying connections to each other shard (1-64) proxy_conn_count: 8 diff --git a/cpp_src/cmd/reindexer_server/contrib/Dockerfile b/cpp_src/cmd/reindexer_server/contrib/Dockerfile index 94681c300..0da240cb0 100644 --- a/cpp_src/cmd/reindexer_server/contrib/Dockerfile +++ b/cpp_src/cmd/reindexer_server/contrib/Dockerfile @@ -1,12 +1,11 @@ -FROM alpine:3.14 AS build +FROM alpine:3.19 AS build RUN cd /tmp && apk update && \ - apk add git curl autoconf automake libtool linux-headers g++ make libunwind-dev grpc-dev grpc protobuf-dev c-ares-dev && \ + apk add git curl autoconf automake libtool linux-headers g++ make libunwind-dev grpc-dev protobuf-dev c-ares-dev patch && \ git clone https://github.com/gperftools/gperftools.git && \ - cd gperftools && git checkout gperftools-2.13 && \ - echo "noinst_PROGRAMS =" >> Makefile.am && \ + cd gperftools && \ sed -i s/_sigev_un\._tid/sigev_notify_thread_id/ src/profile-handler.cc && \ - ./autogen.sh && ./configure --disable-dependency-tracking && make -j8 && make install + ./autogen.sh && ./configure --disable-dependency-tracking && make -j8 && make install ADD . /src @@ -19,17 +18,16 @@ RUN ./dependencies.sh && \ make -j8 reindexer_server reindexer_tool && \ make install -C cpp_src/cmd/reindexer_server && \ make install -C cpp_src/cmd/reindexer_tool && \ + make install -C cpp_src/server/grpc && \ cp ../cpp_src/cmd/reindexer_server/contrib/entrypoint.sh /entrypoint.sh && \ rm -rf /usr/local/lib/*.a /usr/local/include /usr/local/lib/libtcmalloc_debug* /usr/local/lib/libtcmalloc_minimal* \ - /usr/local/lib/libprofiler* /usr/local/lib/libtcmalloc.* /usr/local/share/doc /usr/local/share/man /usr/local/share/perl5 /usr/local/bin/pprof* + /usr/local/lib/libprofiler* /usr/local/lib/libtcmalloc.* /usr/local/share/doc /usr/local/share/man /usr/local/share/perl5 /usr/local/bin/pprof* -RUN cd build && make install -C cpp_src/server/grpc - -FROM alpine:3.14 +FROM alpine:3.19 COPY --from=build /usr/local /usr/local COPY --from=build /entrypoint.sh /entrypoint.sh -RUN apk update && apk add libstdc++ libunwind snappy libexecinfo leveldb c-ares libprotobuf xz-libs && rm -rf /var/cache/apk/* +RUN apk update && apk add libstdc++ libunwind snappy leveldb c-ares libprotobuf xz-libs grpc-cpp && rm -rf /var/cache/apk/* ENV RX_DATABASE /db ENV RX_CORELOG stdout diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100755 index 000000000..d189d3841 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +echo "## choose latest release rpm file" +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/core/cjson/cjsondecoder.cc b/cpp_src/core/cjson/cjsondecoder.cc index b6e61adb4..3973d0e79 100644 --- a/cpp_src/core/cjson/cjsondecoder.cc +++ b/cpp_src/core/cjson/cjsondecoder.cc @@ -127,7 +127,7 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs [[nodiscard]] Variant CJsonDecoder::cjsonValueToVariant(TagType tagType, Serializer &rdser, KeyValueType fieldType) { if (fieldType.Is() && tagType != TagType::TAG_STRING) { storage_.emplace_back(rdser.GetRawVariant(KeyValueType{tagType}).As()); - return Variant(p_string(&storage_.back()), false); + return Variant(p_string(&storage_.back()), Variant::no_hold_t{}); } else { return reindexer::cjsonValueToVariant(tagType, rdser, fieldType); } diff --git a/cpp_src/core/clusterproxy.cc b/cpp_src/core/clusterproxy.cc index 04d6fccb0..2ec8e6222 100644 --- a/cpp_src/core/clusterproxy.cc +++ b/cpp_src/core/clusterproxy.cc @@ -142,9 +142,12 @@ Error ClusterProxy::Connect(const std::string &dsn, ConnectOpts opts) { if (!err.ok()) { return err; } - if (impl_.clusterConfig_) { - clusterConns_.SetParams(impl_.clusterConfig_->proxyConnThreads, impl_.clusterConfig_->proxyConnCount, - impl_.clusterConfig_->proxyConnConcurrency); + if (!connected_.load(std::memory_order_relaxed)) { + if (impl_.clusterConfig_) { + clusterConns_.SetParams(impl_.clusterConfig_->proxyConnThreads, impl_.clusterConfig_->proxyConnCount, + impl_.clusterConfig_->proxyConnConcurrency); + } + connected_.store(err.ok(), std::memory_order_release); } return err; } diff --git a/cpp_src/core/clusterproxy.h b/cpp_src/core/clusterproxy.h index 927893181..e61bd6731 100644 --- a/cpp_src/core/clusterproxy.h +++ b/cpp_src/core/clusterproxy.h @@ -210,7 +210,16 @@ class ClusterProxy { Error GetSqlSuggestions(std::string_view sqlQuery, int pos, std::vector &suggestions, const RdxContext &ctx) { return impl_.GetSqlSuggestions(sqlQuery, pos, suggestions, ctx); } - Error Status() { return impl_.Status(); } + Error Status() noexcept { + if (connected_.load(std::memory_order_acquire)) { + return {}; + } + auto st = impl_.Status(); + if (st.ok()) { + return Error(errNotValid, "Reindexer's cluster proxy layer was not initialized properly"); + } + return st; + } Error GetProtobufSchema(WrSerializer &ser, std::vector &namespaces) { return impl_.GetProtobufSchema(ser, namespaces); } Error GetReplState(std::string_view nsName, ReplicationStateV2 &state, const RdxContext &ctx) { return impl_.GetReplState(nsName, state, ctx); @@ -308,7 +317,6 @@ class ClusterProxy { : cluster::kDefaultClusterProxyCoroPerConn; } std::shared_ptr Get(const std::string &dsn) { - std::shared_ptr res; { shared_lock lck(mtx_); if (shutdown_) { @@ -316,25 +324,25 @@ class ClusterProxy { } auto found = conns_.find(dsn); if (found != conns_.end()) { - res = found->second; + return found->second; } } client::ReindexerConfig cfg; cfg.AppName = "cluster_proxy"; - cfg.SyncRxCoroCount = clientConnConcurrency_; cfg.EnableCompression = true; cfg.RequestDedicatedThread = true; std::lock_guard lck(mtx_); + cfg.SyncRxCoroCount = clientConnConcurrency_; if (shutdown_) { throw Error(errTerminated, "Proxy is already shut down"); } auto found = conns_.find(dsn); if (found != conns_.end()) { - return res; + return found->second; } - res = std::make_shared(cfg, clientConns_, clientThreads_); + auto res = std::make_shared(cfg, clientConns_, clientThreads_); auto err = res->Connect(dsn); if (!err.ok()) { throw err; @@ -370,6 +378,7 @@ class ClusterProxy { std::condition_variable processPingEvent_; std::mutex processPingEventMutex_; int lastPingLeaderId_ = -1; + std::atomic_bool connected_ = false; int getServerIDRel() const noexcept { return sId_.load(std::memory_order_relaxed); } std::shared_ptr getLeader(const cluster::RaftInfo &info); diff --git a/cpp_src/core/ft/config/baseftconfig.cc b/cpp_src/core/ft/config/baseftconfig.cc index 2cb4ad7f1..5df2d0198 100644 --- a/cpp_src/core/ft/config/baseftconfig.cc +++ b/cpp_src/core/ft/config/baseftconfig.cc @@ -1,16 +1,14 @@ #include "baseftconfig.h" -#include #include "core/cjson/jsonbuilder.h" #include "core/ft/stopwords/stop.h" #include "tools/errors.h" -#include "tools/jsontools.h" namespace reindexer { BaseFTConfig::BaseFTConfig() { - for (const char **p = stop_words_en; *p != nullptr; p++) stopWords.insert(*p); - for (const char **p = stop_words_ru; *p != nullptr; p++) stopWords.insert(*p); + for (const char **p = stop_words_en; *p != nullptr; p++) stopWords.insert({*p, StopWord::Type::Morpheme}); + for (const char **p = stop_words_ru; *p != nullptr; p++) stopWords.insert({*p, StopWord::Type::Morpheme}); } void BaseFTConfig::parseBase(const gason::JsonNode &root) { @@ -25,7 +23,25 @@ void BaseFTConfig::parseBase(const gason::JsonNode &root) { auto &stopWordsNode = root["stop_words"]; if (!stopWordsNode.empty()) { stopWords.clear(); - for (auto &sw : stopWordsNode) stopWords.insert(sw.As()); + for (auto &sw : stopWordsNode) { + std::string word; + StopWord::Type type = StopWord::Type::Stop; + if (sw.value.getTag() == gason::JsonTag::JSON_STRING) { + word = sw.As(); + } else if (sw.value.getTag() == gason::JsonTag::JSON_OBJECT) { + word = sw["word"].As(); + type = sw["is_morpheme"].As() ? StopWord::Type::Morpheme : StopWord::Type::Stop; + } + + if (std::find_if(word.begin(), word.end(), [](const auto &symbol) { return std::isspace(symbol); }) != word.end()) { + throw Error(errParams, "Stop words can't contain spaces: %s", word); + } + + auto [it, inserted] = stopWords.emplace(std::move(word), type); + if (!inserted && it->type != type) { + throw Error(errParams, "Duplicate stop-word with different morpheme attribute: %s", *it); + } + } } auto &stemmersNode = root["stemmers"]; @@ -80,7 +96,9 @@ void BaseFTConfig::getJson(JsonBuilder &jsonBuilder) const { { auto stopWordsNode = jsonBuilder.Array("stop_words"); for (const auto &sw : stopWords) { - stopWordsNode.Put(nullptr, sw); + auto wordNode = stopWordsNode.Object(nullptr); + wordNode.Put("word", sw); + wordNode.Put("is_morpheme", sw.type == StopWord::Type::Morpheme); } } { diff --git a/cpp_src/core/ft/config/baseftconfig.h b/cpp_src/core/ft/config/baseftconfig.h index 532d94f2f..33f68e964 100644 --- a/cpp_src/core/ft/config/baseftconfig.h +++ b/cpp_src/core/ft/config/baseftconfig.h @@ -19,6 +19,12 @@ static constexpr int kMinMergeLimitValue = 0; class JsonBuilder; +struct StopWord : std::string { + enum class Type { Stop, Morpheme }; + StopWord(std::string base, Type type = Type::Stop) noexcept : std::string(std::move(base)), type(type) {} + Type type; +}; + class BaseFTConfig { public: struct Synonym { @@ -39,7 +45,8 @@ class BaseFTConfig { bool enableKbLayout = true; bool enableNumbersSearch = false; bool enableWarmupOnNsCopy = false; - fast_hash_set stopWords; + + fast_hash_set stopWords; std::vector synonyms; int logLevel = 0; std::string extraWordSymbols = "-/+"; // word contains symbols (IsAlpa | IsDigit) {IsAlpa | IsDigit | IsExtra} diff --git a/cpp_src/core/ft/ftdsl.cc b/cpp_src/core/ft/ftdsl.cc index e153c2f87..98e78623e 100644 --- a/cpp_src/core/ft/ftdsl.cc +++ b/cpp_src/core/ft/ftdsl.cc @@ -1,6 +1,7 @@ #include "core/ft/ftdsl.h" #include #include +#include "core/ft/config/baseftconfig.h" #include "tools/customlocal.h" #include "tools/errors.h" #include "tools/stringstools.h" @@ -30,7 +31,7 @@ void FtDSLQuery::parse(std::wstring &utf16str) { bool inGroup = false; bool hasAnythingExceptNot = false; int groupCounter = 0; - int maxPatternLen = 1; + size_t maxPatternLen = 1; h_vector fieldsOpts; std::string utf8str; fieldsOpts.insert(fieldsOpts.end(), std::max(int(fields_.size()), 1), {1.0, false}); @@ -49,27 +50,26 @@ void FtDSLQuery::parse(std::wstring &utf16str) { ++it; } else { if (*it == '@') { - it++; + ++it; parseFields(utf16str, it, fieldsOpts); continue; } if (*it == '-') { fte.opts.op = OpNot; - it++; + ++it; } else if (*it == '+') { fte.opts.op = OpAnd; - it++; + ++it; } if (it != utf16str.end() && (*it == '\'' || *it == '\"')) { inGroup = !inGroup; - it++; + ++it; // closing group if (!inGroup) { int distance = 1; if (it != utf16str.end() && *it == '~') { - ++it; - if (it == utf16str.end()) { + if (++it == utf16str.end()) { throw Error(errParseDSL, "Expected digit after '~' operator in phrase, but found nothing"); } if (!std::isdigit(*it)) { @@ -96,17 +96,17 @@ void FtDSLQuery::parse(std::wstring &utf16str) { fteIt->opts.groupNum = groupCounter; } groupTermCounter = 0; - groupCounter++; + ++groupCounter; } } } if (it != utf16str.end() && *it == '=') { fte.opts.exact = true; - it++; + ++it; } if (it != utf16str.end() && *it == '*') { fte.opts.suff = true; - it++; + ++it; } } auto begIt = it; @@ -120,21 +120,21 @@ void FtDSLQuery::parse(std::wstring &utf16str) { } } auto endIt = it; - for (; it != utf16str.end(); it++) { + for (; it != utf16str.end(); ++it) { if (*it == '*') { fte.opts.pref = true; } else if (*it == '~') { fte.opts.typos = true; } else if (*it == '^') { - ++it; - if (it == utf16str.end()) { + if (++it == utf16str.end()) { throw Error(errParseDSL, "Expected digit after '^' operator in search query DSL, but found nothing"); } wchar_t *end = nullptr, *start = &*it; fte.opts.boost = wcstod(start, &end); - it += end - start - 1; - if (end == start) + if (end == start) { throw Error(errParseDSL, "Expected digit after '^' operator in search query DSL, but found '%c' ", char(*start)); + } + it += end - start - 1; } else { break; } @@ -143,18 +143,14 @@ void FtDSLQuery::parse(std::wstring &utf16str) { if (endIt != begIt) { fte.pattern.assign(begIt, endIt); utf16_to_utf8(fte.pattern, utf8str); - if (is_number(utf8str)) fte.opts.number = true; - if (fte.opts.op != OpNot && groupTermCounter == 0) { - // Setting up this flag before stopWords check, to prevent error on DSL with stop word + NOT - hasAnythingExceptNot = true; - } - if (stopWords_.find(utf8str) != stopWords_.end()) { + fte.opts.number = is_number(utf8str); + // Setting up this flag before stopWords check, to prevent error on DSL with stop word + NOT + hasAnythingExceptNot = hasAnythingExceptNot || (fte.opts.op != OpNot && groupTermCounter == 0); + if (auto it = stopWords_.find(utf8str); it != stopWords_.end() && it->type == StopWord::Type::Stop) { continue; } - if (int(fte.pattern.length()) > maxPatternLen) { - maxPatternLen = fte.pattern.length(); - } + maxPatternLen = (fte.pattern.length() > maxPatternLen) ? fte.pattern.length() : maxPatternLen; emplace_back(std::move(fte)); if (inGroup) ++groupTermCounter; } diff --git a/cpp_src/core/ft/ftdsl.h b/cpp_src/core/ft/ftdsl.h index 28e36fcae..a01dbac76 100644 --- a/cpp_src/core/ft/ftdsl.h +++ b/cpp_src/core/ft/ftdsl.h @@ -49,9 +49,11 @@ struct FtDSLVariant { int proc = 0; }; +struct StopWord; + class FtDSLQuery : public RVector { public: - FtDSLQuery(const RHashMap &fields, const fast_hash_set &stopWords, + FtDSLQuery(const RHashMap &fields, const fast_hash_set &stopWords, const std::string &extraWordSymbols) noexcept : fields_(fields), stopWords_(stopWords), extraWordSymbols_(extraWordSymbols) {} void parse(std::wstring &utf16str); @@ -64,7 +66,7 @@ class FtDSLQuery : public RVector { std::function resolver_; const RHashMap &fields_; - const fast_hash_set &stopWords_; + const fast_hash_set &stopWords_; const std::string &extraWordSymbols_; }; diff --git a/cpp_src/core/ft/stopwords/stop_ru.cc b/cpp_src/core/ft/stopwords/stop_ru.cc index 6a63c1a15..7e5472da0 100644 --- a/cpp_src/core/ft/stopwords/stop_ru.cc +++ b/cpp_src/core/ft/stopwords/stop_ru.cc @@ -268,7 +268,7 @@ const char *stop_words_ru[] = { "кроме", "куда", "кругом", - "с т", + "с", "у", "я", "та", diff --git a/cpp_src/core/idsetcache.h b/cpp_src/core/idsetcache.h index 185073c22..6acdb92b5 100644 --- a/cpp_src/core/idsetcache.h +++ b/cpp_src/core/idsetcache.h @@ -75,12 +75,12 @@ T &operator<<(T &os, const IdSetCacheVal &v) { } struct equal_idset_cache_key { - bool operator()(const IdSetCacheKey &lhs, const IdSetCacheKey &rhs) const { + bool operator()(const IdSetCacheKey &lhs, const IdSetCacheKey &rhs) const noexcept { return lhs.cond == rhs.cond && lhs.sort == rhs.sort && *lhs.keys == *rhs.keys; } }; struct hash_idset_cache_key { - size_t operator()(const IdSetCacheKey &s) const { return (s.cond << 8) ^ (s.sort << 16) ^ s.keys->Hash(); } + size_t operator()(const IdSetCacheKey &s) const noexcept { return (size_t(s.cond) << 8) ^ (size_t(s.sort) << 16) ^ s.keys->Hash(); } }; using IdSetCacheBase = LRUCache; diff --git a/cpp_src/core/index/indexordered.cc b/cpp_src/core/index/indexordered.cc index 643562e35..b138ef643 100644 --- a/cpp_src/core/index/indexordered.cc +++ b/cpp_src/core/index/indexordered.cc @@ -60,34 +60,43 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c auto startIt = this->idx_map.begin(); auto endIt = this->idx_map.end(); auto key1 = *keys.begin(); + std::array cacheKeyIts; + unsigned cacheKeyItsCnt = 0; switch (condition) { case CondLt: - endIt = this->idx_map.lower_bound(static_cast(key1)); + cacheKeyIts[0] = endIt = this->idx_map.lower_bound(static_cast(key1)); + cacheKeyItsCnt = 1; break; case CondLe: - endIt = this->idx_map.lower_bound(static_cast(key1)); + cacheKeyIts[0] = endIt = this->idx_map.lower_bound(static_cast(key1)); + cacheKeyItsCnt = 1; if (endIt != this->idx_map.end() && !this->idx_map.key_comp()(static_cast(key1), endIt->first)) endIt++; break; case CondGt: - startIt = this->idx_map.upper_bound(static_cast(key1)); + cacheKeyIts[0] = startIt = this->idx_map.upper_bound(static_cast(key1)); + cacheKeyItsCnt = 1; break; case CondGe: startIt = this->idx_map.find(static_cast(key1)); if (startIt == this->idx_map.end()) startIt = this->idx_map.upper_bound(static_cast(key1)); + cacheKeyIts[0] = startIt; + cacheKeyItsCnt = 1; break; case CondRange: { const auto &key2 = keys[1]; startIt = this->idx_map.find(static_cast(key1)); if (startIt == this->idx_map.end()) startIt = this->idx_map.upper_bound(static_cast(key1)); + cacheKeyIts[0] = startIt; - endIt = this->idx_map.lower_bound(static_cast(key2)); - if (endIt != this->idx_map.end() && !this->idx_map.key_comp()(static_cast(key2), endIt->first)) endIt++; + cacheKeyIts[1] = endIt = this->idx_map.lower_bound(static_cast(key2)); + if (endIt != this->idx_map.end() && !this->idx_map.key_comp()(static_cast(key2), endIt->first)) ++endIt; if (endIt != this->idx_map.end() && this->idx_map.key_comp()(endIt->first, static_cast(key1))) { return SelectKeyResults(std::move(res)); } + cacheKeyItsCnt = 2; } break; case CondAny: @@ -145,7 +154,16 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c }; if (count > 1 && !opts.distinct && !opts.disableIdSetCache) { - this->tryIdsetCache(keys, condition, sortId, std::move(selector), res); + assertrx_throw(cacheKeyItsCnt == 1 || cacheKeyItsCnt == 2); + // Using btree node pointers instead of the real values from the condition + VariantArray cacheKeys; + for (unsigned i = 0; i < cacheKeyItsCnt; ++i) { + auto &cacheKeyIt = cacheKeyIts[i]; + const int64_t ptrVal = + (cacheKeyIt == this->idx_map.end()) ? std::numeric_limits::max() : int64_t(&(*cacheKeyIt)); + cacheKeys.emplace_back(ptrVal); + } + this->tryIdsetCache(cacheKeys, condition, sortId, std::move(selector), res); } else { size_t idsCount; selector(res, idsCount); diff --git a/cpp_src/core/index/indextext/ftkeyentry.h b/cpp_src/core/index/indextext/ftkeyentry.h index 5ac1ce72f..5cf257f50 100644 --- a/cpp_src/core/index/indextext/ftkeyentry.h +++ b/cpp_src/core/index/indextext/ftkeyentry.h @@ -39,7 +39,7 @@ class FtKeyEntry { IdSetPlain& Unsorted() noexcept { return impl_->Unsorted(); } const IdSetPlain& Unsorted() const noexcept { return impl_->Unsorted(); } - IdSetRef Sorted(unsigned sortId) const { return impl_->Sorted(sortId); } + IdSetRef Sorted(unsigned sortId) const noexcept { return impl_->Sorted(sortId); } void UpdateSortedIds(const UpdateSortedContext& ctx) { impl_->UpdateSortedIds(ctx); } void SetVDocID(int vdoc_id) noexcept { impl_->SetVDocID(vdoc_id); } const int& VDocID() const { return impl_->vdoc_id_; } diff --git a/cpp_src/core/index/indexunordered.cc b/cpp_src/core/index/indexunordered.cc index 06af94780..1d24cca2b 100644 --- a/cpp_src/core/index/indexunordered.cc +++ b/cpp_src/core/index/indexunordered.cc @@ -252,6 +252,8 @@ void IndexUnordered::Delete(const Variant &key, IdType id, StringsHolder &str } } +// WARNING: 'keys' is a key for LRUCache and in some cases (for ordered indexes, for example) can contain values, +// which are not correspond to the initial values from queries conditions template bool IndexUnordered::tryIdsetCache(const VariantArray &keys, CondType condition, SortType sortId, const std::function &selector, SelectKeyResult &res) { @@ -271,7 +273,7 @@ bool IndexUnordered::tryIdsetCache(const VariantArray &keys, CondType conditi cache_->Put(ckey, res.MergeIdsets(res.deferedExplicitSort, idsCount)); } } else { - res.push_back(SingleSelectKeyResult(cached.val.ids)); + res.emplace_back(std::move(cached.val.ids)); } } else { scanWin = selector(res, idsCount); diff --git a/cpp_src/core/item.cc b/cpp_src/core/item.cc index c74904379..f7a7f555e 100644 --- a/cpp_src/core/item.cc +++ b/cpp_src/core/item.cc @@ -102,7 +102,7 @@ Item::FieldRef &Item::FieldRef::operator=(span arr) { } else { itemImpl_->holder_->push_back(elem); } - pl.Set(field_, pos++, Variant(p_string{&itemImpl_->holder_->back()})); + pl.Set(field_, pos++, Variant(p_string{&itemImpl_->holder_->back()}, Variant::no_hold_t{})); } } } else { diff --git a/cpp_src/core/itemimpl.cc b/cpp_src/core/itemimpl.cc index 9e0cd204d..43c2dce20 100644 --- a/cpp_src/core/itemimpl.cc +++ b/cpp_src/core/itemimpl.cc @@ -73,7 +73,7 @@ void ItemImpl::ModifyField(const IndexedTagsPath &tagsPath, const VariantArray & } tupleData_ = ser_.DetachLStr(); - pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())))); + pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())), Variant::no_hold_t{})); } void ItemImpl::SetField(std::string_view jsonPath, const VariantArray &keys, const IndexExpressionEvaluator &ev) { @@ -94,7 +94,7 @@ Error ItemImpl::FromMsgPack(std::string_view buf, size_t &offset) { Error err = msgPackDecoder_->Decode(buf, pl, ser_, offset); if (err.ok()) { tupleData_ = ser_.DetachLStr(); - pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())))); + pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())), Variant::no_hold_t{})); } return err; } @@ -109,7 +109,7 @@ Error ItemImpl::FromProtobuf(std::string_view buf) { Error err = decoder.Decode(buf, pl, ser_); if (err.ok()) { tupleData_ = ser_.DetachLStr(); - pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())))); + pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())), Variant::no_hold_t{})); } return err; } @@ -179,7 +179,7 @@ void ItemImpl::FromCJSON(std::string_view slice, bool pkOnly, Recoder *recoder) if (!rdser.Eof()) throw Error(errParseJson, "Internal error - left unparsed data %d", rdser.Pos()); tupleData_ = ser_.DetachLStr(); - pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())))); + pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())), Variant::no_hold_t{})); } Error ItemImpl::FromJSON(std::string_view slice, char **endp, bool pkOnly) { @@ -230,7 +230,7 @@ Error ItemImpl::FromJSON(std::string_view slice, char **endp, bool pkOnly) { // Put tuple to field[0] tupleData_ = ser_.DetachLStr(); - pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())))); + pl.Set(0, Variant(p_string(reinterpret_cast(tupleData_.get())), Variant::no_hold_t{})); return err; } diff --git a/cpp_src/core/itemmodifier.cc b/cpp_src/core/itemmodifier.cc index 48a62e6c5..d87bc068b 100644 --- a/cpp_src/core/itemmodifier.cc +++ b/cpp_src/core/itemmodifier.cc @@ -365,8 +365,10 @@ void ItemModifier::modifyCJSON(IdType id, FieldData &field, VariantArray &values } catch (const Error &) { ns_.krefs.resize(0); } + } else if (index.Opts().IsArray()) { + pl.Get(fieldIdx, ns_.krefs, Variant::hold_t{}); } else { - pl.Get(fieldIdx, ns_.krefs, index.Opts().IsArray()); + pl.Get(fieldIdx, ns_.krefs); } if (ns_.krefs == ns_.skrefs) continue; bool needClearCache{false}; @@ -581,7 +583,7 @@ void ItemModifier::modifyIndexValues(IdType itemId, const FieldData &field, Vari if (index.Opts().IsSparse()) { pl.GetByJsonPath(field.tagspathWithLastIndex(), ns_.skrefs, index.KeyType()); } else { - pl.Get(field.index(), ns_.skrefs, true); + pl.Get(field.index(), ns_.skrefs, Variant::hold_t{}); } // Required when updating index array field with several tagpaths diff --git a/cpp_src/core/keyvalue/variant.cc b/cpp_src/core/keyvalue/variant.cc index 2e6eb87f4..6c05a5ef4 100644 --- a/cpp_src/core/keyvalue/variant.cc +++ b/cpp_src/core/keyvalue/variant.cc @@ -17,9 +17,9 @@ namespace reindexer { -Variant::Variant(const PayloadValue &v) : variant_{0, 1, KeyValueType::Composite{}} { new (cast()) PayloadValue(v); } +Variant::Variant(const PayloadValue &v) noexcept : variant_{0, 1, KeyValueType::Composite{}} { new (cast()) PayloadValue(v); } -Variant::Variant(PayloadValue &&v) : variant_{0, 1, KeyValueType::Composite{}} { new (cast()) PayloadValue(std::move(v)); } +Variant::Variant(PayloadValue &&v) noexcept : variant_{0, 1, KeyValueType::Composite{}} { new (cast()) PayloadValue(std::move(v)); } Variant::Variant(const std::string &v) : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(make_key_string(v)); } @@ -27,20 +27,23 @@ Variant::Variant(std::string &&v) : variant_{0, 1, KeyValueType::String{}} { new Variant::Variant(std::string_view v) : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(make_key_string(v)); } -Variant::Variant(const key_string &v) : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(v); } +Variant::Variant(const key_string &v) noexcept : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(v); } -Variant::Variant(key_string &&v) : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(std::move(v)); } +Variant::Variant(key_string &&v) noexcept : variant_{0, 1, KeyValueType::String{}} { new (cast()) key_string(std::move(v)); } -Variant::Variant(const char *v) : Variant(p_string(v)) {} +Variant::Variant(const char *v) noexcept : Variant(p_string(v), Variant::no_hold_t{}) {} -Variant::Variant(p_string v, bool enableHold) : variant_{0, 0, KeyValueType::String{}} { - if (v.type() == p_string::tagKeyString && enableHold) { +Variant::Variant(p_string v, no_hold_t) noexcept : variant_{0, 0, KeyValueType::String{}} { *cast() = v; } + +Variant::Variant(p_string v, hold_t) : variant_{0, 0, KeyValueType::String{}} { + if (v.type() == p_string::tagKeyString) { variant_.hold = 1; new (cast()) key_string(v.getKeyString()); } else { *cast() = v; } } +Variant::Variant(p_string v) noexcept : Variant(v, no_hold_t{}) {} Variant::Variant(const VariantArray &values) : variant_{0, 1, KeyValueType::Tuple{}} { WrSerializer ser; @@ -51,7 +54,7 @@ Variant::Variant(const VariantArray &values) : variant_{0, 1, KeyValueType::Tupl new (cast()) key_string(make_key_string(ser.Slice())); } -Variant::Variant(Point p) : Variant{VariantArray(p)} {} +Variant::Variant(Point p) noexcept : Variant{VariantArray{p}} {} Variant::Variant(Uuid uuid) noexcept : uuid_() { if (uuid.data_[0] == 0 && uuid.data_[1] == 0) { @@ -476,7 +479,7 @@ int Variant::RelaxCompare(const Variant &other, const CollateOpts &collateOpts) return Uuid{*this}.Compare(*otherUuid); } else { Uuid{*this}.PutToStr(uuidStrBuf); - return -other.Compare(Variant{uuidStrBufPString, false}); + return -other.Compare(Variant{uuidStrBufPString}); } } else if constexpr (withString == WithString::Yes) { Uuid{*this}.PutToStr(uuidStrBuf); @@ -491,7 +494,7 @@ int Variant::RelaxCompare(const Variant &other, const CollateOpts &collateOpts) return uuid->Compare(Uuid{other}); } else { Uuid{other}.PutToStr(uuidStrBuf); - return Compare(Variant{uuidStrBufPString, false}); + return Compare(Variant{uuidStrBufPString}); } } else if constexpr (withString == WithString::Yes) { Uuid{other}.PutToStr(uuidStrBuf); diff --git a/cpp_src/core/keyvalue/variant.h b/cpp_src/core/keyvalue/variant.h index aee77e581..ade3d3e6b 100644 --- a/cpp_src/core/keyvalue/variant.h +++ b/cpp_src/core/keyvalue/variant.h @@ -24,22 +24,27 @@ class Variant { friend Uuid; public: + struct no_hold_t {}; + struct hold_t {}; + Variant() noexcept : variant_{0, 0, KeyValueType::Null{}, uint64_t{}} {} explicit Variant(int v) noexcept : variant_{0, 0, KeyValueType::Int{}, v} {} explicit Variant(bool v) noexcept : variant_{0, 0, KeyValueType::Bool{}, v} {} explicit Variant(int64_t v) noexcept : variant_{0, 0, KeyValueType::Int64{}, v} {} explicit Variant(double v) noexcept : variant_{0, 0, KeyValueType::Double{}, v} {} - explicit Variant(const char *v); - explicit Variant(p_string v, bool enableHold = true); + explicit Variant(const char *v) noexcept; + Variant(p_string v, no_hold_t) noexcept; + Variant(p_string v, hold_t); + explicit Variant(p_string v) noexcept; explicit Variant(const std::string &v); explicit Variant(std::string &&v); explicit Variant(std::string_view v); - explicit Variant(const key_string &v); - explicit Variant(key_string &&v); - explicit Variant(const PayloadValue &v); - explicit Variant(PayloadValue &&v); + explicit Variant(const key_string &v) noexcept; + explicit Variant(key_string &&v) noexcept; + explicit Variant(const PayloadValue &v) noexcept; + explicit Variant(PayloadValue &&v) noexcept; explicit Variant(const VariantArray &values); - explicit Variant(Point); + explicit Variant(Point) noexcept; explicit Variant(Uuid) noexcept; Variant(const Variant &other) : uuid_{other.uuid_} { if (!isUuid()) { diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index e3e12b030..c6c199556 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -575,7 +575,7 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con for (auto fieldIdx : changedFields) { auto& index = *indexes_[fieldIdx]; if ((fieldIdx == 0) || deltaFields <= 0) { - oldValue.Get(fieldIdx, skrefsDel, true); + oldValue.Get(fieldIdx, skrefsDel, Variant::hold_t{}); bool needClearCache{false}; index.Delete(skrefsDel, rowId, *strHolder_, needClearCache); if (needClearCache && index.IsOrdered()) indexesCacheCleaner.Add(index.SortId()); @@ -1408,9 +1408,12 @@ void NamespaceImpl::doDelete(IdType id) { if (index.Opts().IsSparse()) { assertrx(index.Fields().getTagsPathsLength() > 0); pl.GetByJsonPath(index.Fields().getTagsPath(0), skrefs, index.KeyType()); + } else if (index.Opts().IsArray()) { + pl.Get(field, skrefs, Variant::hold_t{}); } else { - pl.Get(field, skrefs, index.Opts().IsArray()); + pl.Get(field, skrefs); } + // Delete value from index bool needClearCache{false}; index.Delete(skrefs, id, *strHolder_, needClearCache); @@ -1733,8 +1736,10 @@ void NamespaceImpl::doUpsert(ItemImpl* ritem, IdType id, bool doUpdate) { } catch (const Error&) { krefs.resize(0); } + } else if (index.Opts().IsArray()) { + pl.Get(field, krefs, Variant::hold_t{}); } else { - pl.Get(field, krefs, index.Opts().IsArray()); + pl.Get(field, krefs); } if ((krefs.ArrayType().Is() && skrefs.ArrayType().Is()) || krefs == skrefs) continue; bool needClearCache{false}; diff --git a/cpp_src/core/nsselecter/btreeindexiterator.h b/cpp_src/core/nsselecter/btreeindexiterator.h index a805955a8..7a0b6bc25 100644 --- a/cpp_src/core/nsselecter/btreeindexiterator.h +++ b/cpp_src/core/nsselecter/btreeindexiterator.h @@ -10,8 +10,8 @@ namespace reindexer { template class BtreeIndexIterator final : public IndexIterator { public: - explicit BtreeIndexIterator(const T& idxMap) : idxMap_(idxMap), first_(idxMap.begin()), last_(idxMap.end()) {} - BtreeIndexIterator(const T& idxMap, const typename T::iterator& first, const typename T::iterator& last) + explicit BtreeIndexIterator(const T& idxMap) noexcept : idxMap_(idxMap), first_(idxMap.begin()), last_(idxMap.end()) {} + BtreeIndexIterator(const T& idxMap, const typename T::iterator& first, const typename T::iterator& last) noexcept : idxMap_(idxMap), first_(first), last_(last) {} ~BtreeIndexIterator() override final = default; diff --git a/cpp_src/core/nsselecter/explaincalc.cc b/cpp_src/core/nsselecter/explaincalc.cc index 642cade9c..4dfff967e 100644 --- a/cpp_src/core/nsselecter/explaincalc.cc +++ b/cpp_src/core/nsselecter/explaincalc.cc @@ -202,6 +202,16 @@ std::string ExplainCalc::GetJSON() { json.Put("postprocess_us"sv, To_us(postprocess_)); json.Put("loop_us"sv, To_us(loop_)); json.Put("general_sort_us"sv, To_us(sort_)); + if (!subqueries_.empty()) { + auto subQuries = json.Array("subqueries"); + for (const auto &sq : subqueries_) { + auto s = subQuries.Object(); + s.Put("namespace", sq.NsName()); + s.Raw("explain", sq.Explain()); + std::visit(overloaded{[&](size_t k) { s.Put("keys", k); }, [&](const std::string &f) { s.Put("field", f); }}, + sq.FieldOrKeys()); + } + } } json.Put("sort_index"sv, sortIndex_); json.Put("sort_by_uncommitted_index"sv, sortOptimization_); @@ -305,45 +315,6 @@ std::string SelectIteratorContainer::explainJSON(const_iterator begin, const_ite return name.str(); } -ExplainCalc::Duration ExplainCalc::lap() noexcept { - auto now = Clock::now(); - Duration d = now - last_point_; - last_point_ = now; - return d; -} - int ExplainCalc::To_us(const ExplainCalc::Duration &d) noexcept { return duration_cast(d).count(); } -void ExplainCalc::StartTiming() noexcept { - if (enabled_) lap(); -} - -void ExplainCalc::StopTiming() noexcept { - if (enabled_) total_ = preselect_ + prepare_ + select_ + postprocess_ + loop_; -} - -void ExplainCalc::AddPrepareTime() noexcept { - if (enabled_) prepare_ += lap(); -} - -void ExplainCalc::AddSelectTime() noexcept { - if (enabled_) select_ += lap(); -} - -void ExplainCalc::AddPostprocessTime() noexcept { - if (enabled_) postprocess_ += lap(); -} - -void ExplainCalc::AddLoopTime() noexcept { - if (enabled_) loop_ += lap(); -} - -void ExplainCalc::StartSort() noexcept { - if (enabled_) sort_start_point_ = Clock::now(); -} - -void ExplainCalc::StopSort() noexcept { - if (enabled_) sort_ = Clock::now() - sort_start_point_; -} - } // namespace reindexer diff --git a/cpp_src/core/nsselecter/explaincalc.h b/cpp_src/core/nsselecter/explaincalc.h index 837dfafde..ab23d8290 100644 --- a/cpp_src/core/nsselecter/explaincalc.h +++ b/cpp_src/core/nsselecter/explaincalc.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "core/type_consts.h" @@ -17,6 +18,24 @@ struct ConditionInjection; typedef std::vector JoinedSelectors; typedef std::vector OnConditionInjections; +class SubQueryExplain { +public: + SubQueryExplain(const std::string& ns, std::string&& exp) : explain_{std::move(exp)}, namespace_{ns} {} + [[nodiscard]] const std::string& NsName() const& noexcept { return namespace_; } + [[nodiscard]] const auto& FieldOrKeys() const& noexcept { return fieldOrKeys_; } + [[nodiscard]] const std::string& Explain() const& noexcept { return explain_; } + void SetFieldOrKeys(std::variant&& fok) noexcept { fieldOrKeys_ = std::move(fok); } + + auto NsName() const&& = delete; + auto FieldOrKeys() const&& = delete; + auto Explain() const&& = delete; + +private: + std::string explain_; + std::string namespace_; + std::variant fieldOrKeys_{size_t(0)}; +}; + class ExplainCalc { public: typedef std::chrono::high_resolution_clock Clock; @@ -29,24 +48,40 @@ class ExplainCalc { ExplainCalc() = default; ExplainCalc(bool enable) noexcept : enabled_(enable) {} - void StartTiming() noexcept; - void StopTiming() noexcept; - - void AddPrepareTime() noexcept; - void AddSelectTime() noexcept; - void AddPostprocessTime() noexcept; - void AddLoopTime() noexcept; + void StartTiming() noexcept { + if (enabled_) lap(); + } + void StopTiming() noexcept { + if (enabled_) total_ = preselect_ + prepare_ + select_ + postprocess_ + loop_; + } + void AddPrepareTime() noexcept { + if (enabled_) prepare_ += lap(); + } + void AddSelectTime() noexcept { + if (enabled_) select_ += lap(); + } + void AddPostprocessTime() noexcept { + if (enabled_) postprocess_ += lap(); + } + void AddLoopTime() noexcept { + if (enabled_) loop_ += lap(); + } void AddIterations(int iters) noexcept { iters_ += iters; } - void StartSort() noexcept; - void StopSort() noexcept; + void StartSort() noexcept { + if (enabled_) sort_start_point_ = Clock::now(); + } + void StopSort() noexcept { + if (enabled_) sort_ = Clock::now() - sort_start_point_; + } void PutCount(int cnt) noexcept { count_ = cnt; } void PutSortIndex(std::string_view index) noexcept { sortIndex_ = index; } - void PutSelectors(const SelectIteratorContainer *qres) noexcept { selectors_ = qres; } - void PutJoinedSelectors(const JoinedSelectors *jselectors) noexcept { jselectors_ = jselectors; } + void PutSelectors(const SelectIteratorContainer* qres) noexcept { selectors_ = qres; } + void PutJoinedSelectors(const JoinedSelectors* jselectors) noexcept { jselectors_ = jselectors; } void SetPreselectTime(Duration preselectTime) noexcept { preselect_ = preselectTime; } - void PutOnConditionInjections(const OnConditionInjections *onCondInjections) noexcept { onInjections_ = onCondInjections; } + void PutOnConditionInjections(const OnConditionInjections* onCondInjections) noexcept { onInjections_ = onCondInjections; } void SetSortOptimization(bool enable) noexcept { sortOptimization_ = enable; } + void SetSubQueriesExplains(std::vector&& subQueriesExpl) noexcept { subqueries_ = std::move(subQueriesExpl); } void LogDump(int logLevel); std::string GetJSON(); @@ -59,14 +94,21 @@ class ExplainCalc { Duration Sort() const noexcept { return sort_; } size_t Iterations() const noexcept { return iters_; } - static int To_us(const Duration &d) noexcept; bool IsEnabled() const noexcept { return enabled_; } + static int To_us(const Duration &d) noexcept; + private: - Duration lap() noexcept; + Duration lap() noexcept { + const auto now = Clock::now(); + Duration d = now - last_point_; + last_point_ = now; + return d; + } time_point last_point_, sort_start_point_; - Duration total_, prepare_ = Duration::zero(); + Duration total_ = Duration::zero(); + Duration prepare_ = Duration::zero(); Duration preselect_ = Duration::zero(); Duration select_ = Duration::zero(); Duration postprocess_ = Duration::zero(); @@ -74,9 +116,10 @@ class ExplainCalc { Duration sort_ = Duration::zero(); std::string_view sortIndex_; - const SelectIteratorContainer *selectors_ = nullptr; - const JoinedSelectors *jselectors_ = nullptr; - const OnConditionInjections *onInjections_ = nullptr; ///< Optional + const SelectIteratorContainer* selectors_ = nullptr; + const JoinedSelectors* jselectors_ = nullptr; + const OnConditionInjections* onInjections_ = nullptr; ///< Optional + std::vector subqueries_; int iters_ = 0; int count_ = 0; diff --git a/cpp_src/core/nsselecter/fieldscomparator.cc b/cpp_src/core/nsselecter/fieldscomparator.cc index 2005861c8..1734a944c 100644 --- a/cpp_src/core/nsselecter/fieldscomparator.cc +++ b/cpp_src/core/nsselecter/fieldscomparator.cc @@ -41,8 +41,8 @@ class ArrayAdapter { [&](reindexer::KeyValueType::Double) noexcept { return reindexer::Variant{*reinterpret_cast(ptr_ + sizeof_ * i)}; }, - [&](reindexer::KeyValueType::String) { - return reindexer::Variant{*reinterpret_cast(ptr_ + sizeof_ * i), false}; + [&](reindexer::KeyValueType::String) noexcept { + return reindexer::Variant{*reinterpret_cast(ptr_ + sizeof_ * i)}; }, [&](reindexer::KeyValueType::Bool) noexcept { return reindexer::Variant{*reinterpret_cast(ptr_ + sizeof_ * i)}; }, [&](reindexer::KeyValueType::Int) noexcept { return reindexer::Variant{*reinterpret_cast(ptr_ + sizeof_ * i)}; }, diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 84004986b..22366c585 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -27,6 +27,7 @@ void NsSelecter::operator()(LocalQueryResults &result, SelectCtx &ctx, const Rdx auto &explain = ctx.explain; explain = ExplainCalc(ctx.query.GetExplain() || logLevel >= LogInfo); + explain.SetSubQueriesExplains(std::move(ctx.subQueriesExplains)); ActiveQueryScope queryScope(ctx, ns_->optimizationState_, explain, ns_->locker_.IsReadOnly(), ns_->strHolder_.get()); explain.SetPreselectTime(ctx.preResultTimeTotal); diff --git a/cpp_src/core/nsselecter/nsselecter.h b/cpp_src/core/nsselecter/nsselecter.h index 50fe4c77e..9f8cc0f2b 100644 --- a/cpp_src/core/nsselecter/nsselecter.h +++ b/cpp_src/core/nsselecter/nsselecter.h @@ -33,6 +33,7 @@ struct SelectCtx { const Query *parentQuery = nullptr; ExplainCalc explain; bool requiresCrashTracking = false; + std::vector subQueriesExplains; RX_ALWAYS_INLINE bool isMergeQuerySubQuery() const noexcept { return isMergeQuery == IsMergeQuery::Yes && parentQuery; } }; @@ -47,7 +48,7 @@ class NsSelecter { class JoinedNsValueGetter; public: - NsSelecter(NamespaceImpl *parent) : ns_(parent) {} + NsSelecter(NamespaceImpl *parent) noexcept : ns_(parent) {} void operator()(LocalQueryResults &result, SelectCtx &ctx, const RdxContext &); diff --git a/cpp_src/core/nsselecter/selectiterator.cc b/cpp_src/core/nsselecter/selectiterator.cc index 6a5cd3ba3..d9af8b35b 100644 --- a/cpp_src/core/nsselecter/selectiterator.cc +++ b/cpp_src/core/nsselecter/selectiterator.cc @@ -7,9 +7,6 @@ namespace reindexer { -SelectIterator::SelectIterator(SelectKeyResult res, bool dist, std::string n, IteratorFieldKind fKind, bool forcedFirst) - : SelectKeyResult(std::move(res)), distinct(dist), name(std::move(n)), fieldKind(fKind), forcedFirst_(forcedFirst), type_(Forward) {} - void SelectIterator::Bind(const PayloadType &type, int field) { for (Comparator &cmp : comparators_) cmp.Bind(type, field); } @@ -18,62 +15,69 @@ void SelectIterator::Start(bool reverse, int maxIterations) { const bool explicitSort = applyDeferedSort(maxIterations); isReverse_ = reverse; - lastIt_ = begin(); + const auto begIt = begin(); + lastIt_ = begIt; - for (auto it = begin(); it != end(); it++) { + for (auto it = begIt, endIt = end(); it != endIt; ++it) { if (it->isRange_) { if (isReverse_) { - auto rrBegin = it->rEnd_ - 1; + const auto rrBegin = it->rEnd_ - 1; it->rrEnd_ = it->rBegin_ - 1; it->rrBegin_ = rrBegin; - it->rrIt_ = it->rrBegin_; + it->rrIt_ = rrBegin; } else { it->rIt_ = it->rBegin_; } } else { if (it->useBtree_) { - assertrx(it->set_); + assertrx_dbg(it->set_); if (reverse) { - it->setrbegin_ = it->set_->rbegin(); + const auto setRBegin = it->set_->rbegin(); + it->ritset_ = setRBegin; + it->setrbegin_ = setRBegin; it->setrend_ = it->set_->rend(); - it->ritset_ = it->set_->rbegin(); } else { - it->setbegin_ = it->set_->begin(); + const auto setBegin = it->set_->begin(); + it->itset_ = setBegin; + it->setbegin_ = setBegin; it->setend_ = it->set_->end(); - it->itset_ = it->setbegin_; } } else { if (isReverse_) { - it->rbegin_ = it->ids_.rbegin(); + const auto idsRBegin = it->ids_.rbegin(); it->rend_ = it->ids_.rend(); - it->rit_ = it->ids_.rbegin(); + it->rit_ = idsRBegin; + it->rbegin_ = idsRBegin; } else { - it->begin_ = it->ids_.begin(); + const auto idsBegin = it->ids_.begin(); it->end_ = it->ids_.end(); - it->it_ = it->ids_.begin(); + it->it_ = idsBegin; + it->begin_ = idsBegin; } } } } lastVal_ = isReverse_ ? INT_MAX : INT_MIN; - type_ = isReverse_ ? Reverse : Forward; - if (size() == 1 && begin()->indexForwardIter_) { + + if (size() == 0) { + type_ = OnlyComparator; + lastVal_ = isReverse_ ? INT_MIN : INT_MAX; + } else if (size() == 1 && begIt->indexForwardIter_) { type_ = UnbuiltSortOrdersIndex; - begin()->indexForwardIter_->Start(reverse); + begIt->indexForwardIter_->Start(reverse); } else if (isUnsorted) { type_ = Unsorted; } else if (size() == 1) { if (!isReverse_) { - type_ = begin()->isRange_ ? SingleRange : (explicitSort ? SingleIdSetWithDeferedSort : SingleIdset); + type_ = begIt->isRange_ ? SingleRange : (explicitSort ? SingleIdSetWithDeferedSort : SingleIdset); } else { - type_ = begin()->isRange_ ? RevSingleRange : (explicitSort ? RevSingleIdSetWithDeferedSort : RevSingleIdset); + type_ = begIt->isRange_ ? RevSingleRange : (explicitSort ? RevSingleIdSetWithDeferedSort : RevSingleIdset); } + } else { + type_ = isReverse_ ? Reverse : Forward; } - if (size() == 0) { - type_ = OnlyComparator; - lastVal_ = isReverse_ ? INT_MIN : INT_MAX; - } + ClearDistinct(); } @@ -81,7 +85,7 @@ void SelectIterator::Start(bool reverse, int maxIterations) { bool SelectIterator::nextFwd(IdType minHint) noexcept { if (minHint > lastVal_) lastVal_ = minHint - 1; int minVal = INT_MAX; - for (auto it = begin(); it != end(); it++) { + for (auto it = begin(), endIt = end(); it != endIt; ++it) { if (it->useBtree_) { if (it->itset_ != it->setend_) { it->itset_ = it->set_->upper_bound(lastVal_); @@ -100,7 +104,7 @@ bool SelectIterator::nextFwd(IdType minHint) noexcept { } } else if (!it->isRange_ && it->it_ != it->end_) { - for (; it->it_ != it->end_ && *it->it_ <= lastVal_; it->it_++) { + for (; it->it_ != it->end_ && *it->it_ <= lastVal_; ++it->it_) { } if (it->it_ != it->end_ && *it->it_ < minVal) { minVal = *it->it_; @@ -117,7 +121,7 @@ bool SelectIterator::nextRev(IdType maxHint) noexcept { if (maxHint < lastVal_) lastVal_ = maxHint + 1; int maxVal = INT_MIN; - for (auto it = begin(); it != end(); it++) { + for (auto it = begin(), endIt = end(); it != endIt; ++it) { if (it->useBtree_ && it->ritset_ != it->setrend_) { for (; it->ritset_ != it->setrend_ && *it->ritset_ >= lastVal_; ++it->ritset_) { } @@ -133,7 +137,7 @@ bool SelectIterator::nextRev(IdType maxHint) noexcept { lastIt_ = it; } } else if (!it->isRange_ && !it->useBtree_ && it->rit_ != it->rend_) { - for (; it->rit_ != it->rend_ && *it->rit_ >= lastVal_; it->rit_++) { + for (; it->rit_ != it->rend_ && *it->rit_ >= lastVal_; ++it->rit_) { } if (it->rit_ != it->rend_ && *it->rit_ > maxVal) { maxVal = *it->rit_; @@ -160,7 +164,7 @@ bool SelectIterator::nextFwdSingleIdset(IdType minHint) noexcept { it->it_ = std::upper_bound(it->it_, it->end_, lastVal_); } } else { - for (; it->it_ != it->end_ && *it->it_ <= lastVal_; it->it_++) { + for (; it->it_ != it->end_ && *it->it_ <= lastVal_; ++it->it_) { } } lastVal_ = (it->it_ != it->end_) ? *it->it_ : INT_MAX; @@ -174,11 +178,11 @@ bool SelectIterator::nextRevSingleIdset(IdType maxHint) noexcept { auto it = begin(); if (it->useBtree_) { - for (; it->ritset_ != it->setrend_ && *it->ritset_ >= lastVal_; it->ritset_++) { + for (; it->ritset_ != it->setrend_ && *it->ritset_ >= lastVal_; ++it->ritset_) { } lastVal_ = (it->ritset_ != it->setrend_) ? *it->ritset_ : INT_MIN; } else { - for (; it->rit_ != it->rend_ && *it->rit_ >= lastVal_; it->rit_++) { + for (; it->rit_ != it->rend_ && *it->rit_ >= lastVal_; ++it->rit_) { } lastVal_ = (it->rit_ != it->rend_) ? *it->rit_ : INT_MIN; } @@ -192,41 +196,44 @@ bool SelectIterator::nextUnbuiltSortOrders() noexcept { return begin()->indexFor bool SelectIterator::nextFwdSingleRange(IdType minHint) noexcept { if (minHint > lastVal_) lastVal_ = minHint - 1; - if (lastVal_ < begin()->rBegin_) lastVal_ = begin()->rBegin_ - 1; + const auto begIt = begin(); + if (lastVal_ < begIt->rBegin_) lastVal_ = begIt->rBegin_ - 1; - lastVal_ = (lastVal_ < begin()->rEnd_) ? lastVal_ + 1 : begin()->rEnd_; - if (lastVal_ == begin()->rEnd_) lastVal_ = INT_MAX; + lastVal_ = (lastVal_ < begIt->rEnd_) ? lastVal_ + 1 : begIt->rEnd_; + if (lastVal_ == begIt->rEnd_) lastVal_ = INT_MAX; return (lastVal_ != INT_MAX); } bool SelectIterator::nextRevSingleRange(IdType maxHint) noexcept { if (maxHint < lastVal_) lastVal_ = maxHint + 1; - if (lastVal_ > begin()->rrBegin_) lastVal_ = begin()->rrBegin_ + 1; + const auto begIt = begin(); + if (lastVal_ > begIt->rrBegin_) lastVal_ = begIt->rrBegin_ + 1; - lastVal_ = (lastVal_ > begin()->rrEnd_) ? lastVal_ - 1 : begin()->rrEnd_; - if (lastVal_ == begin()->rrEnd_) lastVal_ = INT_MIN; + lastVal_ = (lastVal_ > begIt->rrEnd_) ? lastVal_ - 1 : begIt->rrEnd_; + if (lastVal_ == begIt->rrEnd_) lastVal_ = INT_MIN; return (lastVal_ != INT_MIN); } // Unsorted next implementation bool SelectIterator::nextUnsorted() noexcept { - if (lastIt_ == end()) { + const auto endIt = end(); + if (lastIt_ == endIt) { return false; } else if (lastIt_->it_ == lastIt_->end_) { ++lastIt_; - while (lastIt_ != end()) { + while (lastIt_ != endIt) { if (lastIt_->it_ != lastIt_->end_) { lastVal_ = *lastIt_->it_; - lastIt_->it_++; + ++lastIt_->it_; return true; } ++lastIt_; } } else { lastVal_ = *lastIt_->it_; - lastIt_->it_++; + ++lastIt_->it_; return true; } @@ -236,8 +243,9 @@ bool SelectIterator::nextUnsorted() noexcept { void SelectIterator::ExcludeLastSet(const PayloadValue &value, IdType rowId, IdType properRowId) { for (auto &comp : comparators_) comp.ExcludeDistinct(value, properRowId); if (type_ == UnbuiltSortOrdersIndex) { - if (begin()->indexForwardIter_->Value() == rowId) { - begin()->indexForwardIter_->ExcludeLastSet(); + const auto begIt = begin(); + if (begIt->indexForwardIter_->Value() == rowId) { + begIt->indexForwardIter_->ExcludeLastSet(); } } else if (!End() && lastIt_ != end() && lastVal_ == rowId) { assertrx(!lastIt_->isRange_); @@ -285,12 +293,13 @@ double SelectIterator::Cost(int expectedIterations) const noexcept { result = jsonPathComparators ? (kNonIdxFieldComparatorCostMultiplier * double(expectedIterations) + jsonPathComparators + 1) : (double(expectedIterations) + 1); } + const auto sz = size(); if (distinct) { - result += size(); + result += sz; } else if (type_ != SingleIdSetWithDeferedSort && type_ != RevSingleIdSetWithDeferedSort && !deferedExplicitSort) { - result += static_cast(GetMaxIterations()) * size(); + result += static_cast(GetMaxIterations()) * sz; } else { - result += static_cast(CostWithDefferedSort(size(), GetMaxIterations(), expectedIterations)); + result += static_cast(CostWithDefferedSort(sz, GetMaxIterations(), expectedIterations)); } return isNotOperation_ ? expectedIterations + result : result; } diff --git a/cpp_src/core/nsselecter/selectiterator.h b/cpp_src/core/nsselecter/selectiterator.h index 55286e183..ef4606b1e 100644 --- a/cpp_src/core/nsselecter/selectiterator.h +++ b/cpp_src/core/nsselecter/selectiterator.h @@ -24,7 +24,13 @@ class SelectIterator : public SelectKeyResult { }; SelectIterator() = default; - SelectIterator(SelectKeyResult res, bool distinct, std::string name, IteratorFieldKind fieldKind, bool forcedFirst = false); + SelectIterator(SelectKeyResult res, bool dist, std::string n, IteratorFieldKind fKind, bool forcedFirst = false) noexcept + : SelectKeyResult(std::move(res)), + distinct(dist), + name(std::move(n)), + fieldKind(fKind), + forcedFirst_(forcedFirst), + type_(Forward) {} /// Starts iteration process: prepares /// object for further work. diff --git a/cpp_src/core/nsselecter/sortingcontext.cc b/cpp_src/core/nsselecter/sortingcontext.cc deleted file mode 100644 index 65376dbd9..000000000 --- a/cpp_src/core/nsselecter/sortingcontext.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include "sortingcontext.h" -#include "core/index/index.h" -#include "core/query/query.h" - -namespace reindexer { - -Index *SortingContext::sortIndex() const noexcept { - if (entries.empty()) return nullptr; - return std::visit(overloaded{[](const OneOf &) noexcept -> Index * { return nullptr; }, - [](const FieldEntry &e) noexcept { return e.index; }}, - entries[0]); -} - -const Index *SortingContext::sortIndexIfOrdered() const noexcept { - if (entries.empty() || !isIndexOrdered() || !enableSortOrders) return nullptr; - return std::visit(overloaded{[](const OneOf &) noexcept -> Index * { return nullptr; }, - [](const FieldEntry &e) noexcept { return e.index; }}, - entries[0]); -} - -int SortingContext::sortId() const noexcept { - if (!enableSortOrders) return 0; - Index *sortIdx = sortIndex(); - return sortIdx ? sortIdx->SortId() : 0; -} - -bool SortingContext::isIndexOrdered() const noexcept { - if (entries.empty()) return false; - return std::visit(overloaded{[](const OneOf &) noexcept { return false; }, - [](const FieldEntry &e) noexcept { return e.index && e.index->IsOrdered(); }}, - entries[0]); -} - -bool SortingContext::isOptimizationEnabled() const noexcept { return (uncommitedIndex >= 0) && sortIndex(); } - -const SortingContext::Entry &SortingContext::getFirstColumnEntry() const noexcept { - assertrx(!entries.empty()); - return entries[0]; -} - -void SortingContext::resetOptimization() noexcept { - uncommitedIndex = -1; - if (!entries.empty()) { - std::visit( - overloaded{[](const OneOf &) noexcept {}, [](FieldEntry &e) noexcept { e.index = nullptr; }}, - entries[0]); - } -} - -SortingOptions::SortingOptions(const SortingContext &sortingContext) noexcept - : forcedMode{sortingContext.forcedMode}, - multiColumn{sortingContext.entries.size() > 1}, - haveExpression{!sortingContext.expressions.empty()} { - if (sortingContext.entries.empty()) { - usingGeneralAlgorithm = false; - byBtreeIndex = false; - } else { - std::visit(overloaded{[](const OneOf &) noexcept {}, - [&](const SortingContext::FieldEntry &sortEntry) noexcept { - if (sortEntry.index && sortEntry.index->IsOrdered()) { - byBtreeIndex = (sortingContext.isOptimizationEnabled() || sortingContext.enableSortOrders); - multiColumnByBtreeIndex = (byBtreeIndex && multiColumn); - } - usingGeneralAlgorithm = !byBtreeIndex; - }}, - sortingContext.entries[0]); - } -} - -bool SortingOptions::postLoopSortingRequired() const noexcept { - return multiColumn || usingGeneralAlgorithm || forcedMode || haveExpression; -} - -} // namespace reindexer diff --git a/cpp_src/core/nsselecter/sortingcontext.h b/cpp_src/core/nsselecter/sortingcontext.h index cf2f63662..9db7d589b 100644 --- a/cpp_src/core/nsselecter/sortingcontext.h +++ b/cpp_src/core/nsselecter/sortingcontext.h @@ -1,5 +1,6 @@ #pragma once +#include "core/index/index.h" #include "core/indexopts.h" #include "core/sorting/sortexpression.h" #include "estl/h_vector.h" @@ -33,13 +34,49 @@ struct SortingContext { }; using Entry = std::variant; - [[nodiscard]] int sortId() const noexcept; - [[nodiscard]] Index *sortIndex() const noexcept; - [[nodiscard]] const Index *sortIndexIfOrdered() const noexcept; - [[nodiscard]] bool isOptimizationEnabled() const noexcept; - [[nodiscard]] bool isIndexOrdered() const noexcept; - [[nodiscard]] const Entry &getFirstColumnEntry() const noexcept; - void resetOptimization() noexcept; + [[nodiscard]] int sortId() const noexcept { + if (!enableSortOrders) return 0; + const Index *sortIdx = sortIndex(); + return sortIdx ? int(sortIdx->SortId()) : 0; + } + [[nodiscard]] Index *sortIndex() const noexcept { + if (entries.empty()) return nullptr; + // get_if is truly noexcept, so using it instead of std::visit + if (const auto *fe = std::get_if(&entries[0]); fe) { + return fe->index; + } + return nullptr; + } + [[nodiscard]] const Index *sortIndexIfOrdered() const noexcept { + if (entries.empty() || !isIndexOrdered() || !enableSortOrders) return nullptr; + // get_if is truly noexcept, so using it instead of std::visit + if (const auto *fe = std::get_if(&entries[0]); fe) { + return fe->index; + } + return nullptr; + } + [[nodiscard]] bool isOptimizationEnabled() const noexcept { return (uncommitedIndex >= 0) && sortIndex(); } + [[nodiscard]] bool isIndexOrdered() const noexcept { + if (entries.empty()) return false; + // get_if is truly noexcept, so using it instead of std::visit + if (const auto *fe = std::get_if(&entries[0]); fe) { + return fe->index && fe->index->IsOrdered(); + } + return false; + } + [[nodiscard]] const Entry &getFirstColumnEntry() const noexcept { + assertrx(!entries.empty()); + return entries[0]; + } + void resetOptimization() noexcept { + uncommitedIndex = -1; + if (!entries.empty()) { + // get_if is truly noexcept, so using it instead of std::visit + if (auto *fe = std::get_if(&entries[0]); fe) { + fe->index = nullptr; + } + } + } bool enableSortOrders = false; h_vector entries; @@ -50,8 +87,27 @@ struct SortingContext { }; struct SortingOptions { - SortingOptions(const SortingContext &sortingContext) noexcept; - [[nodiscard]] bool postLoopSortingRequired() const noexcept; + SortingOptions(const SortingContext &sortingContext) noexcept + : forcedMode{sortingContext.forcedMode}, + multiColumn{sortingContext.entries.size() > 1}, + haveExpression{!sortingContext.expressions.empty()} { + if (sortingContext.entries.empty()) { + usingGeneralAlgorithm = false; + byBtreeIndex = false; + } else { + // get_if is truly noexcept, so using it instead of std::visit + if (auto *sortEntry = std::get_if(&sortingContext.entries[0]); sortEntry) { + if (sortEntry->index && sortEntry->index->IsOrdered()) { + byBtreeIndex = (sortingContext.isOptimizationEnabled() || sortingContext.enableSortOrders); + multiColumnByBtreeIndex = (byBtreeIndex && multiColumn); + } + usingGeneralAlgorithm = !byBtreeIndex; + } + } + } + [[nodiscard]] bool postLoopSortingRequired() const noexcept { + return multiColumn || usingGeneralAlgorithm || forcedMode || haveExpression; + } bool byBtreeIndex = false; bool usingGeneralAlgorithm = true; diff --git a/cpp_src/core/payload/payloadfieldvalue.h b/cpp_src/core/payload/payloadfieldvalue.h index 011b01ee8..b9dfcf94a 100644 --- a/cpp_src/core/payload/payloadfieldvalue.h +++ b/cpp_src/core/payload/payloadfieldvalue.h @@ -47,19 +47,36 @@ class PayloadFieldValue { abort(); }); } - Variant Get(bool enableHold = false) const { + Variant Get() noexcept { return Get(Variant::no_hold_t{}); } + template + Variant Get(HoldT h) const noexcept(noexcept(Variant(std::declval(), h))) { return t_.Type().EvaluateOneOf( [&](KeyValueType::Bool) noexcept { return Variant(*reinterpret_cast(p_)); }, [&](KeyValueType::Int) noexcept { return Variant(*reinterpret_cast(p_)); }, [&](KeyValueType::Int64) noexcept { return Variant(*reinterpret_cast(p_)); }, [&](KeyValueType::Double) noexcept { return Variant(*reinterpret_cast(p_)); }, - [&](KeyValueType::String) { return Variant(*reinterpret_cast(p_), enableHold); }, + [&](KeyValueType::String) noexcept(noexcept(Variant(std::declval(), h))) { + return Variant(*reinterpret_cast(p_), h); + }, [&](KeyValueType::Uuid) noexcept { return Variant(*reinterpret_cast(p_)); }, [](OneOf) noexcept -> Variant { assertrx(0); abort(); }); } + // Variant Get(Variant::hold_t) const noexcept { + // return t_.Type().EvaluateOneOf( + // [&](KeyValueType::Bool) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [&](KeyValueType::Int) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [&](KeyValueType::Int64) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [&](KeyValueType::Double) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [&](KeyValueType::String) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [&](KeyValueType::Uuid) noexcept { return Variant(*reinterpret_cast(p_)); }, + // [](OneOf) noexcept -> Variant { + // assertrx(0); + // abort(); + // }); + // } size_t Hash() const noexcept { return t_.Type().EvaluateOneOf( [&](KeyValueType::Bool) noexcept { return std::hash()(*reinterpret_cast(p_)); }, diff --git a/cpp_src/core/payload/payloadiface.cc b/cpp_src/core/payload/payloadiface.cc index b7becd91d..a22a11fa0 100644 --- a/cpp_src/core/payload/payloadiface.cc +++ b/cpp_src/core/payload/payloadiface.cc @@ -10,8 +10,39 @@ namespace reindexer { +// Get element(s) by field index +template +void PayloadIface::Get(int field, VariantArray &keys, Variant::hold_t h) const { + get(field, keys, h); +} +template +void PayloadIface::Get(int field, VariantArray &keys) const { + get(field, keys, Variant::no_hold_t{}); +} + +// Get element by field and array index +template +Variant PayloadIface::Get(int field, int idx, Variant::hold_t h) const { + return get(field, idx, h); +} +template +Variant PayloadIface::Get(int field, int idx) const { + return get(field, idx, Variant::no_hold_t{}); +} + +// Get element(s) by field name template -void PayloadIface::Get(int field, VariantArray &keys, bool enableHold) const { +void PayloadIface::Get(std::string_view field, VariantArray &kvs, Variant::hold_t h) const { + get(t_.FieldByName(field), kvs, h); +} +template +void PayloadIface::Get(std::string_view field, VariantArray &kvs) const { + get(t_.FieldByName(field), kvs, Variant::no_hold_t{}); +} + +template +template +void PayloadIface::get(int field, VariantArray &keys, HoldT h) const { assertrx(field < NumFields()); keys.clear(); if (t_.Field(field).IsArray()) { @@ -20,15 +51,16 @@ void PayloadIface::Get(int field, VariantArray &keys, bool enableHold) const for (int i = 0; i < arr->len; i++) { PayloadFieldValue pv(t_.Field(field), v_->Ptr() + arr->offset + i * t_.Field(field).ElemSizeof()); - keys.push_back(pv.Get(enableHold)); + keys.push_back(pv.Get(h)); } } else { - keys.push_back(Field(field).Get(enableHold)); + keys.push_back(Field(field).Get(h)); } } template -Variant PayloadIface::Get(int field, int idx, bool enableHold) const { +template +Variant PayloadIface::get(int field, int idx, HoldT h) const { assertrx(field < NumFields()); if (t_.Field(field).IsArray()) { @@ -36,20 +68,13 @@ Variant PayloadIface::Get(int field, int idx, bool enableHold) const { assertf(idx < arr->len, "Field '%s.%s' bound exceed idx %d > len %d", Type().Name(), Type().Field(field).Name(), idx, arr->len); PayloadFieldValue pv(t_.Field(field), v_->Ptr() + arr->offset + idx * t_.Field(field).ElemSizeof()); - return pv.Get(enableHold); - + return pv.Get(h); } else { assertf(idx == 0, "Field '%s.%s' is not array, can't get idx %d", Type().Name(), Type().Field(field).Name(), idx); - return Field(field).Get(enableHold); + return Field(field).Get(h); } } -// Get element(s) by field index -template -void PayloadIface::Get(std::string_view field, VariantArray &kvs, bool enableHold) const { - Get(t_.FieldByName(field), kvs, enableHold); -} - template void PayloadIface::GetByJsonPath(std::string_view jsonPath, TagsMatcher &tagsMatcher, VariantArray &kvs, KeyValueType expectedType) const { diff --git a/cpp_src/core/payload/payloadiface.h b/cpp_src/core/payload/payloadiface.h index 4ff6e6b62..178f9ddac 100644 --- a/cpp_src/core/payload/payloadiface.h +++ b/cpp_src/core/payload/payloadiface.h @@ -27,9 +27,11 @@ class PayloadIface { void Reset() noexcept { memset(v_->Ptr(), 0, t_.TotalSize()); } // Get element(s) by field index - void Get(int field, VariantArray &, bool enableHold = false) const; + void Get(int field, VariantArray &, Variant::hold_t) const; + void Get(int field, VariantArray &) const; // Get element by field and array index - [[nodiscard]] Variant Get(int field, int idx, bool enableHold = false) const; + [[nodiscard]] Variant Get(int field, int idx, Variant::hold_t) const; + [[nodiscard]] Variant Get(int field, int idx) const; // Get array as span of typed elements template @@ -105,8 +107,9 @@ class PayloadIface { template ::value>::type * = nullptr> T CopyTo(PayloadType t, bool newFields = true); - // Get element(s) by field index - void Get(std::string_view field, VariantArray &, bool enableHold = false) const; + // Get element(s) by field name + void Get(std::string_view field, VariantArray &, Variant::hold_t) const; + void Get(std::string_view field, VariantArray &) const; // Get element(s) by json path void GetByJsonPath(std::string_view jsonPath, TagsMatcher &tagsMatcher, VariantArray &, KeyValueType expectedType) const; @@ -166,6 +169,7 @@ class PayloadIface { void GetJSON(const TagsMatcher &tm, WrSerializer &ser); private: + enum class HoldPolicy : bool { Hold, NoHold }; template ::value>::type * = nullptr> T CopyWithNewOrUpdatedFields(PayloadType t); @@ -177,6 +181,12 @@ class PayloadIface { void getByJsonPath(const P &path, VariantArray &, KeyValueType expectedType) const; template ::value>::type * = nullptr> void setArray(int field, const VariantArray &keys, bool append); + template + void get(int field, VariantArray &, HoldT h) const; + template + [[nodiscard]] Variant get(int field, int idx, HoldT h) const; + template + void get(std::string_view field, VariantArray &, HoldT h) const; // Array of elements types , not owning const PayloadTypeImpl &t_; diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index d78ffa5db..fe81c63c7 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -293,7 +293,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { Debug(ser.GetVarUint()); break; case QueryStrictMode: - strictMode_ = StrictMode(ser.GetVarUint()); + Strict(StrictMode(ser.GetVarUint())); break; case QueryLimit: count_ = ser.GetVarUint(); @@ -315,7 +315,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { break; } case QueryExplain: - explain_ = true; + Explain(true); break; case QueryLocal: local_ = true; diff --git a/cpp_src/core/querycache.h b/cpp_src/core/querycache.h index a9c0c4c14..2c50e4546 100644 --- a/cpp_src/core/querycache.h +++ b/cpp_src/core/querycache.h @@ -18,6 +18,8 @@ struct QueryCountCacheVal { }; struct QueryCacheKey { + using BufT = h_vector; + QueryCacheKey() = default; QueryCacheKey(QueryCacheKey&& other) = default; QueryCacheKey(const QueryCacheKey& other) = default; @@ -26,13 +28,15 @@ struct QueryCacheKey { QueryCacheKey(const Query& q) { WrSerializer ser; q.Serialize(ser, (SkipJoinQueries | SkipMergeQueries | SkipLimitOffset)); - buf.reserve(ser.Len()); + if rx_unlikely (ser.Len() > BufT::max_size()) { + throw Error(errLogic, "QueryCacheKey: buffer overflow"); + } buf.assign(ser.Buf(), ser.Buf() + ser.Len()); } size_t Size() const noexcept { return sizeof(QueryCacheKey) + (buf.is_hdata() ? 0 : buf.size()); } QueryCacheKey(WrSerializer& ser) : buf(ser.Buf(), ser.Buf() + ser.Len()) {} - h_vector buf; + BufT buf; }; struct EqQueryCacheKey { diff --git a/cpp_src/core/queryresults/joinresults.cc b/cpp_src/core/queryresults/joinresults.cc index c3706f118..b17936674 100644 --- a/cpp_src/core/queryresults/joinresults.cc +++ b/cpp_src/core/queryresults/joinresults.cc @@ -7,11 +7,6 @@ namespace reindexer { namespace joins { -JoinedFieldIterator::JoinedFieldIterator(const NamespaceResults* parent, const ItemOffsets& offsets, uint8_t joinedFieldOrder) - : joinRes_(parent), offsets_(&offsets), order_(joinedFieldOrder) { - if (offsets_->size() > 0) updateOffset(); -} - bool JoinedFieldIterator::operator==(const JoinedFieldIterator& other) const { if (joinRes_ != other.joinRes_) throw Error(errLogic, "Comparising joined fields of different namespaces!"); if (offsets_ != other.offsets_) throw Error(errLogic, "Comparising joined fields of different items!"); @@ -19,25 +14,7 @@ bool JoinedFieldIterator::operator==(const JoinedFieldIterator& other) const { return true; } -bool JoinedFieldIterator::operator!=(const JoinedFieldIterator& other) const { return !operator==(other); } - -JoinedFieldIterator::const_reference JoinedFieldIterator::operator[](size_t idx) const { - assertrx(currOffset_ + idx < joinRes_->items_.size()); - return joinRes_->items_[currOffset_ + idx]; -} - -JoinedFieldIterator::reference JoinedFieldIterator::operator[](size_t idx) { - assertrx(currOffset_ + idx < joinRes_->items_.size()); - return const_cast(joinRes_->items_[currOffset_ + idx]); -} - -JoinedFieldIterator& JoinedFieldIterator::operator++() { - ++order_; - updateOffset(); - return *this; -} - -void JoinedFieldIterator::updateOffset() { +void JoinedFieldIterator::updateOffset() noexcept { currField_ = -1; if (order_ == joinRes_->GetJoinedSelectorsCount()) return; @@ -65,7 +42,7 @@ LocalQueryResults JoinedFieldIterator::ToQueryResults() const { return LocalQueryResults(begin, end); } -int JoinedFieldIterator::ItemsCount() const { +int JoinedFieldIterator::ItemsCount() const noexcept { assertrx(order_ < joinRes_->GetJoinedSelectorsCount()); if ((currField_ != -1) && (currField_ < uint8_t(offsets_->size()))) { @@ -75,33 +52,32 @@ int JoinedFieldIterator::ItemsCount() const { return 0; } -const JoinedFieldIterator noJoinedDataIt(nullptr, {}, 0); +static const ItemOffsets kEmptyOffsets; +static const JoinedFieldIterator kNoJoinedDataIt(nullptr, kEmptyOffsets, 0); -JoinedFieldIterator ItemIterator::begin() const { +JoinedFieldIterator ItemIterator::begin() const noexcept { auto it = joinRes_->offsets_.find(rowid_); - if (it == joinRes_->offsets_.end()) return noJoinedDataIt; - if (it->second.empty()) return noJoinedDataIt; + if (it == joinRes_->offsets_.end()) return kNoJoinedDataIt; + if (it->second.empty()) return kNoJoinedDataIt; return JoinedFieldIterator(joinRes_, it->second, 0); } JoinedFieldIterator ItemIterator::at(uint8_t joinedField) const { auto it = joinRes_->offsets_.find(rowid_); - if (it == joinRes_->offsets_.end()) return noJoinedDataIt; - if (it->second.empty()) return noJoinedDataIt; + if (it == joinRes_->offsets_.end()) return kNoJoinedDataIt; + if (it->second.empty()) return kNoJoinedDataIt; assertrx(joinedField < joinRes_->GetJoinedSelectorsCount()); return JoinedFieldIterator(joinRes_, it->second, joinedField); } -JoinedFieldIterator ItemIterator::end() const { +JoinedFieldIterator ItemIterator::end() const noexcept { auto it = joinRes_->offsets_.find(rowid_); - if (it == joinRes_->offsets_.end()) return noJoinedDataIt; - if (it->second.empty()) return noJoinedDataIt; + if (it == joinRes_->offsets_.end()) return kNoJoinedDataIt; + if (it->second.empty()) return kNoJoinedDataIt; return JoinedFieldIterator(joinRes_, it->second, joinRes_->GetJoinedSelectorsCount()); } -int ItemIterator::getJoinedFieldsCount() const { return joinRes_->GetJoinedSelectorsCount(); } - -int ItemIterator::getJoinedItemsCount() const { +int ItemIterator::getJoinedItemsCount() const noexcept { if (joinedItemsCount_ == -1) { joinedItemsCount_ = 0; auto it = joinRes_->offsets_.find(rowid_); @@ -113,14 +89,14 @@ int ItemIterator::getJoinedItemsCount() const { return joinedItemsCount_; } -ItemIterator ItemIterator::CreateFrom(const LocalQueryResults::Iterator& it) { +ItemIterator ItemIterator::CreateFrom(const LocalQueryResults::Iterator& it) noexcept { auto ret = ItemIterator::CreateEmpty(); auto& itemRef = it.qr_->Items()[it.idx_]; if ((itemRef.Nsid() >= it.qr_->joined_.size())) return ret; return ItemIterator(&(it.qr_->joined_[itemRef.Nsid()]), itemRef.Id()); } -ItemIterator ItemIterator::CreateEmpty() { +ItemIterator ItemIterator::CreateEmpty() noexcept { static NamespaceResults empty; static ItemIterator ret(&empty, 0); return ret; diff --git a/cpp_src/core/queryresults/joinresults.h b/cpp_src/core/queryresults/joinresults.h index 3b2d1ee47..6ee0947eb 100644 --- a/cpp_src/core/queryresults/joinresults.h +++ b/cpp_src/core/queryresults/joinresults.h @@ -81,22 +81,35 @@ class JoinedFieldIterator { using reference = ItemRef&; using const_reference = const ItemRef&; - JoinedFieldIterator(const NamespaceResults* parent, const ItemOffsets& offsets, uint8_t joinedFieldOrder); + JoinedFieldIterator(const NamespaceResults* parent, const ItemOffsets& offsets, uint8_t joinedFieldOrder) noexcept + : joinRes_(parent), offsets_(&offsets), order_(joinedFieldOrder) { + if (offsets_->size() > 0) updateOffset(); + } bool operator==(const JoinedFieldIterator& other) const; - bool operator!=(const JoinedFieldIterator& other) const; + bool operator!=(const JoinedFieldIterator& other) const { return !operator==(other); } - const_reference operator[](size_t idx) const; - reference operator[](size_t idx); - JoinedFieldIterator& operator++(); + const_reference operator[](size_t idx) const noexcept { + assertrx(currOffset_ + idx < joinRes_->items_.size()); + return joinRes_->items_[currOffset_ + idx]; + } + reference operator[](size_t idx) noexcept { + assertrx(currOffset_ + idx < joinRes_->items_.size()); + return const_cast(joinRes_->items_[currOffset_ + idx]); + } + JoinedFieldIterator& operator++() noexcept { + ++order_; + updateOffset(); + return *this; + } ItemImpl GetItem(int itemIdx, const PayloadType& pt, const TagsMatcher& tm) const; LocalQueryResults ToQueryResults() const; - int ItemsCount() const; + int ItemsCount() const noexcept; private: - void updateOffset(); + void updateOffset() noexcept; const NamespaceResults* joinRes_ = nullptr; const ItemOffsets* offsets_ = nullptr; uint8_t order_ = 0; @@ -111,14 +124,14 @@ class ItemIterator { ItemIterator(const NamespaceResults* parent, IdType rowid) noexcept : joinRes_(parent), rowid_(rowid) {} JoinedFieldIterator at(uint8_t joinedField) const; - JoinedFieldIterator begin() const; - JoinedFieldIterator end() const; + JoinedFieldIterator begin() const noexcept; + JoinedFieldIterator end() const noexcept; - int getJoinedFieldsCount() const; - int getJoinedItemsCount() const; + int getJoinedFieldsCount() const noexcept { return joinRes_->GetJoinedSelectorsCount(); } + int getJoinedItemsCount() const noexcept; - static ItemIterator CreateFrom(const LocalQueryResults::Iterator& it); - static ItemIterator CreateEmpty(); + static ItemIterator CreateFrom(const LocalQueryResults::Iterator& it) noexcept; + static ItemIterator CreateEmpty() noexcept; private: const NamespaceResults* joinRes_; diff --git a/cpp_src/core/queryresults/localqueryresults.cc b/cpp_src/core/queryresults/localqueryresults.cc index 23e5754b6..1ff9f171c 100644 --- a/cpp_src/core/queryresults/localqueryresults.cc +++ b/cpp_src/core/queryresults/localqueryresults.cc @@ -364,16 +364,6 @@ Error LocalQueryResults::Iterator::GetCJSON(WrSerializer &ser, bool withHdrLen) return errOK; } -bool LocalQueryResults::Iterator::IsRaw() const noexcept { - auto &itemRef = qr_->items_[idx_]; - return itemRef.Raw(); -} -std::string_view LocalQueryResults::Iterator::GetRaw() const noexcept { - auto &itemRef = qr_->items_[idx_]; - assertrx(itemRef.Raw()); - return std::string_view(reinterpret_cast(itemRef.Value().Ptr()), itemRef.Value().GetCapacity()); -} - Item LocalQueryResults::Iterator::GetItem(bool enableHold) { auto &itemRef = qr_->items_[idx_]; diff --git a/cpp_src/core/queryresults/localqueryresults.h b/cpp_src/core/queryresults/localqueryresults.h index 9b6171cf7..e900242d5 100644 --- a/cpp_src/core/queryresults/localqueryresults.h +++ b/cpp_src/core/queryresults/localqueryresults.h @@ -55,12 +55,13 @@ class LocalQueryResults { size_t TotalCount() const noexcept { return totalCount; } const std::string &GetExplainResults() const & noexcept { return explainResults; } const std::string &GetExplainResults() const &&; + std::string &&MoveExplainResults() & noexcept { return std::move(explainResults); } const std::vector &GetAggregationResults() const & noexcept { return aggregationResults; } const std::vector &GetAggregationResults() const && = delete; void Clear(); h_vector GetNamespaces() const; NsShardsIncarnationTags GetIncarnationTags() const; - bool IsCacheEnabled() const { return !nonCacheableData; } + bool IsCacheEnabled() const noexcept { return !nonCacheableData; } void SetOutputShardId(int shardId) noexcept { outputShardId = shardId; } CsvOrdering MakeCSVTagOrdering(unsigned limit, unsigned offset) const; @@ -77,8 +78,12 @@ class LocalQueryResults { joins::ItemIterator GetJoined(); const ItemRef &GetItemRef() const noexcept { return qr_->items_[idx_]; } lsn_t GetLSN() const noexcept { return qr_->items_[idx_].Value().GetLSN(); } - bool IsRaw() const noexcept; - std::string_view GetRaw() const noexcept; + bool IsRaw() const noexcept { return qr_->items_[idx_].Raw(); } + std::string_view GetRaw() const noexcept { + auto &itemRef = qr_->items_[idx_]; + assertrx(itemRef.Raw()); + return std::string_view(reinterpret_cast(itemRef.Value().Ptr()), itemRef.Value().GetCapacity()); + } Iterator &operator++() noexcept { idx_++; return *this; @@ -97,9 +102,9 @@ class LocalQueryResults { Error err_; }; - Iterator begin() const { return Iterator{this, 0, errOK}; } - Iterator end() const { return Iterator{this, int(items_.size()), errOK}; } - Iterator operator[](int idx) const { return Iterator{this, idx, errOK}; } + Iterator begin() const noexcept { return Iterator{this, 0, Error()}; } + Iterator end() const noexcept { return Iterator{this, int(items_.size()), Error()}; } + Iterator operator[](int idx) const noexcept { return Iterator{this, idx, Error()}; } std::vector joined_; std::vector aggregationResults; diff --git a/cpp_src/core/reindexer.cc b/cpp_src/core/reindexer.cc index 7e262ba8f..1603c5c0f 100644 --- a/cpp_src/core/reindexer.cc +++ b/cpp_src/core/reindexer.cc @@ -251,7 +251,7 @@ Error Reindexer::SetTagsMatcher(std::string_view nsName, TagsMatcher&& tm) { return impl_->SetTagsMatcher(nsName, std::move(tm), rdxCtx); } void Reindexer::ShutdownCluster() { impl_->ShutdownCluster(); } -Error Reindexer::Status() { return impl_->Status(); } +Error Reindexer::Status() noexcept { return impl_->Status(); } Error Reindexer::DumpIndex(std::ostream& os, std::string_view nsName, std::string_view index) { const auto rdxCtx = impl_->CreateRdxContext(ctx_, [&](WrSerializer& s) { s << "DUMP INDEX " << index << " ON " << nsName; }); return impl_->DumpIndex(os, nsName, index, rdxCtx); diff --git a/cpp_src/core/reindexer.h b/cpp_src/core/reindexer.h index 2d22a37c6..db3858d34 100644 --- a/cpp_src/core/reindexer.h +++ b/cpp_src/core/reindexer.h @@ -243,7 +243,7 @@ class Reindexer { /// @param suggestions - all the suggestions for 'pos' position in query. Error GetSqlSuggestions(std::string_view sqlQuery, int pos, std::vector &suggestions); /// Get curret connection status - Error Status(); + Error Status() noexcept; /// Init system namepaces, and load config from config namespace /// Cancelation context doesn't affect this call diff --git a/cpp_src/core/reindexer_impl/reindexerimpl.cc b/cpp_src/core/reindexer_impl/reindexerimpl.cc index 83106906f..c81d4ecc1 100644 --- a/cpp_src/core/reindexer_impl/reindexerimpl.cc +++ b/cpp_src/core/reindexer_impl/reindexerimpl.cc @@ -74,7 +74,6 @@ ReindexerImpl::ReindexerImpl(ReindexerConfig cfg, ActivityContainer& activities, nsLock_(*clusterizator_, *this), activities_(activities), storageType_(StorageType::LevelDB), - connected_(false), config_(std::move(cfg)), proxyCallbacks_(std::move(proxyCallbacks)) { configProvider_.setHandler(ProfilingConf, std::bind(&ReindexerImpl::onProfiligConfigLoad, this)); @@ -355,9 +354,7 @@ Error ReindexerImpl::Connect(const std::string& dsn, ConnectOpts opts) { } } - if (err.ok()) { - connected_.store(true, std::memory_order_release); - } + connected_.store(err.ok(), std::memory_order_release); return err; } @@ -2110,13 +2107,6 @@ bool ReindexerImpl::isSystemNamespaceNameStrict(std::string_view name) noexcept [name](const NamespaceDef& nsDef) { return iequals(nsDef.name, name); }) != kSystemNsDefs.end(); } -Error ReindexerImpl::Status() { - if (connected_.load(std::memory_order_acquire)) { - return errOK; - } - return Error(errNotValid, "DB is not connected"sv); -} - Error ReindexerImpl::SuggestLeader(const cluster::NodeData& suggestion, cluster::NodeData& response) { return clusterizator_->SuggestLeader(suggestion, response); } diff --git a/cpp_src/core/reindexer_impl/reindexerimpl.h b/cpp_src/core/reindexer_impl/reindexerimpl.h index 121999567..8273f49fb 100644 --- a/cpp_src/core/reindexer_impl/reindexerimpl.h +++ b/cpp_src/core/reindexer_impl/reindexerimpl.h @@ -126,7 +126,12 @@ class ReindexerImpl { Error SetClusterizationStatus(std::string_view nsName, const ClusterizationStatus &status, const RdxContext &ctx); Error GetSnapshot(std::string_view nsName, const SnapshotOpts &opts, Snapshot &snapshot, const RdxContext &ctx); Error ApplySnapshotChunk(std::string_view nsName, const SnapshotChunk &ch, const RdxContext &ctx); - Error Status(); + Error Status() noexcept { + if rx_likely (connected_.load(std::memory_order_acquire)) { + return {}; + } + return Error(errNotValid, "DB is not connected"); + } Error SuggestLeader(const cluster::NodeData &suggestion, cluster::NodeData &response); Error LeadersPing(const cluster::NodeData &); Error GetRaftInfo(bool allowTransitState, cluster::RaftInfo &, const RdxContext &ctx); diff --git a/cpp_src/core/reindexer_impl/rx_selector.cc b/cpp_src/core/reindexer_impl/rx_selector.cc index 420c2ce8e..cff716175 100644 --- a/cpp_src/core/reindexer_impl/rx_selector.cc +++ b/cpp_src/core/reindexer_impl/rx_selector.cc @@ -44,12 +44,16 @@ void RxSelector::DoSelect(const Query& q, LocalQueryResults& result, NsLocker } std::vector queryResultsHolder; std::optional queryCopy; + ExplainCalc::Duration preselectTimeTotal{0}; + std::vector subQueryExplains; if (!q.GetSubQueries().empty()) { if (q.GetDebugLevel() >= LogInfo || ns->config_.logLevel >= LogInfo) { logPrintf(LogInfo, "Query before subqueries substitution: %s", q.GetSQL()); } queryCopy.emplace(q); - preselectSubQueries(*queryCopy, queryResultsHolder, locks, func, ctx); + const auto preselectStartTime = ExplainCalc::Clock::now(); + subQueryExplains = preselectSubQueries(*queryCopy, queryResultsHolder, locks, func, ctx); + preselectTimeTotal += ExplainCalc::Clock::now() - preselectStartTime; } const Query& query = queryCopy ? *queryCopy : q; std::vector joinQueryResultsContexts; @@ -64,12 +68,11 @@ void RxSelector::DoSelect(const Query& q, LocalQueryResults& result, NsLocker } JoinedSelectors mainJoinedSelectors; - ExplainCalc::Duration preselectTimeTotal{0}; if (thereAreJoins) { const auto preselectStartTime = ExplainCalc::Clock::now(); mainJoinedSelectors = prepareJoinedSelectors(query, result, locks, func, joinQueryResultsContexts, ctx); result.joined_.resize(1 + query.GetMergeQueries().size()); - preselectTimeTotal = ExplainCalc::Clock::now() - preselectStartTime; + preselectTimeTotal += ExplainCalc::Clock::now() - preselectStartTime; } IsFTQuery isFtQuery{IsFTQuery::NotSet}; { @@ -79,6 +82,7 @@ void RxSelector::DoSelect(const Query& q, LocalQueryResults& result, NsLocker selCtx.contextCollectingMode = true; selCtx.functions = &func; selCtx.nsid = 0; + selCtx.subQueriesExplains = std::move(subQueryExplains); if (!query.GetMergeQueries().empty()) { selCtx.isMergeQuery = IsMergeQuery::Yes; if rx_unlikely (!query.sortingEntries_.empty()) { @@ -148,13 +152,16 @@ void RxSelector::DoSelect(const Query& q, LocalQueryResults& result, NsLocker std::optional mQueryCopy; if (!mq.GetSubQueries().empty()) { mQueryCopy.emplace(mq); - preselectSubQueries(*mQueryCopy, queryResultsHolder, locks, func, ctx); } const JoinedQuery& mQuery = mQueryCopy ? *mQueryCopy : mq; + SelectCtx mctx(mQuery, &query); + if (!mq.GetSubQueries().empty()) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + mctx.subQueriesExplains = preselectSubQueries(*mQueryCopy, queryResultsHolder, locks, func, ctx); + } auto mns = locks.Get(mQuery.NsName()); assertrx_throw(mns); - SelectCtx mctx(mQuery, &query); mctx.nsid = ++counter; mctx.isMergeQuery = IsMergeQuery::Yes; mctx.isFtQuery = isFtQuery; @@ -253,7 +260,7 @@ bool RxSelector::isPreResultValuesModeOptimizationAvailable(const Query& jItemQ, template bool RxSelector::selectSubQuery(const Query& subQuery, const Query& mainQuery, NsLocker& locks, SelectFunctionsHolder& func, - const RdxContext& rdxCtx) { + std::vector& explain, const RdxContext& rdxCtx) { auto ns = locks.Get(subQuery.NsName()); assertrx_throw(ns); @@ -267,12 +274,16 @@ bool RxSelector::selectSubQuery(const Query& subQuery, const Query& mainQuery, N LocalQueryResults result; ns->Select(result, sctx, rdxCtx); locks.Delete(ns); + if (!result.GetExplainResults().empty()) { + explain.emplace_back(subQuery.NsName(), result.MoveExplainResults()); + } return sctx.matchedAtLeastOnce; } template VariantArray RxSelector::selectSubQuery(const Query& subQuery, const Query& mainQuery, NsLocker& locks, LocalQueryResults& qr, - SelectFunctionsHolder& func, const RdxContext& rdxCtx) { + SelectFunctionsHolder& func, std::variant fieldOrKeys, + std::vector& explain, const RdxContext& rdxCtx) { NamespaceImpl::Ptr ns = locks.Get(subQuery.NsName()); assertrx_throw(ns); @@ -361,6 +372,10 @@ VariantArray RxSelector::selectSubQuery(const Query& subQuery, const Query& main } } locks.Delete(ns); + if (!qr.GetExplainResults().empty()) { + explain.emplace_back(subQuery.NsName(), std::move(qr.MoveExplainResults())); + explain.back().SetFieldOrKeys(std::move(fieldOrKeys)); + } return result; } @@ -459,8 +474,12 @@ JoinedSelectors RxSelector::prepareJoinedSelectors(const Query& q, LocalQueryRes } template -void RxSelector::preselectSubQueries(Query& mainQuery, std::vector& queryResultsHolder, NsLocker& locks, - SelectFunctionsHolder& func, const RdxContext& ctx) { +std::vector RxSelector::preselectSubQueries(Query& mainQuery, std::vector& queryResultsHolder, + NsLocker& locks, SelectFunctionsHolder& func, const RdxContext& ctx) { + std::vector explains; + if (mainQuery.GetExplain() || mainQuery.GetDebugLevel() >= LogInfo) { + explains.reserve(mainQuery.GetSubQueries().size()); + } for (size_t i = 0, s = mainQuery.Entries().Size(); i < s; ++i) { mainQuery.Entries().InvokeAppropriate( i, Skip{}, @@ -468,14 +487,16 @@ void RxSelector::preselectSubQueries(Query& mainQuery, std::vector(i); } else { mainQuery.SetEntry(i); } } else { LocalQueryResults qr; - const auto values = selectSubQuery(mainQuery.GetSubQuery(sqe.QueryIndex()), mainQuery, locks, qr, func, ctx); + const auto values = selectSubQuery(mainQuery.GetSubQuery(sqe.QueryIndex()), mainQuery, locks, qr, func, + sqe.Values().size(), explains, ctx); if (QueryEntries::CheckIfSatisfyCondition(values, sqe.Condition(), sqe.Values())) { mainQuery.SetEntry(i); } else { @@ -490,15 +511,17 @@ void RxSelector::preselectSubQueries(Query& mainQuery, std::vector( - i, std::move(mainQuery.GetUpdatableEntry(i)).FieldName(), sqe.Condition(), - selectSubQuery(mainQuery.GetSubQuery(sqe.QueryIndex()), mainQuery, locks, queryResultsHolder.back(), func, ctx)); + mainQuery.SetEntry(i, std::move(mainQuery.GetUpdatableEntry(i)).FieldName(), + sqe.Condition(), + selectSubQuery(mainQuery.GetSubQuery(sqe.QueryIndex()), mainQuery, locks, + queryResultsHolder.back(), func, sqe.FieldName(), explains, ctx)); } catch (const Error& err) { throw Error(err.code(), "Error during preprocessing of subquery '" + mainQuery.GetSubQuery(sqe.QueryIndex()).GetSQL() + "': " + err.what()); } }); } + return explains; } template void RxSelector::DoSelect>( diff --git a/cpp_src/core/reindexer_impl/rx_selector.h b/cpp_src/core/reindexer_impl/rx_selector.h index 5a2923d97..10f850ae0 100644 --- a/cpp_src/core/reindexer_impl/rx_selector.h +++ b/cpp_src/core/reindexer_impl/rx_selector.h @@ -85,14 +85,15 @@ class RxSelector { static JoinedSelectors prepareJoinedSelectors(const Query &q, LocalQueryResults &result, NsLocker &locks, SelectFunctionsHolder &func, std::vector &, const RdxContext &ctx); template - static void preselectSubQueries(Query &mainQuery, std::vector &queryResultsHolder, NsLocker &, - SelectFunctionsHolder &, const RdxContext &); + [[nodiscard]] static std::vector preselectSubQueries(Query &mainQuery, std::vector &queryResultsHolder, + NsLocker &, SelectFunctionsHolder &, const RdxContext &); template [[nodiscard]] static bool selectSubQuery(const Query &subQuery, const Query &mainQuery, NsLocker &, SelectFunctionsHolder &, - const RdxContext &); + std::vector &, const RdxContext &); template [[nodiscard]] static VariantArray selectSubQuery(const Query &subQuery, const Query &mainQuery, NsLocker &, LocalQueryResults &, - SelectFunctionsHolder &, const RdxContext &); + SelectFunctionsHolder &, std::variant fieldOrKeys, + std::vector &, const RdxContext &); static bool isPreResultValuesModeOptimizationAvailable(const Query &jItemQ, const NamespaceImpl::Ptr &jns, const Query &mainQ); }; diff --git a/cpp_src/core/selectkeyresult.h b/cpp_src/core/selectkeyresult.h index 55f83498b..8018061bd 100644 --- a/cpp_src/core/selectkeyresult.h +++ b/cpp_src/core/selectkeyresult.h @@ -21,11 +21,11 @@ class SingleSelectKeyResult { public: SingleSelectKeyResult() noexcept {} - SingleSelectKeyResult(IndexIterator::Ptr indexForwardIter) : indexForwardIter_(std::move(indexForwardIter)) { + explicit SingleSelectKeyResult(IndexIterator::Ptr &&indexForwardIter) noexcept : indexForwardIter_(std::move(indexForwardIter)) { assertrx(indexForwardIter_ != nullptr); } template - explicit SingleSelectKeyResult(const KeyEntryT &ids, SortType sortId) { + explicit SingleSelectKeyResult(const KeyEntryT &ids, SortType sortId) noexcept { if (ids.Unsorted().IsCommited()) { ids_ = ids.Sorted(sortId); } else { @@ -35,7 +35,7 @@ class SingleSelectKeyResult { useBtree_ = true; } } - explicit SingleSelectKeyResult(IdSet::Ptr ids) noexcept : tempIds_(std::move(ids)), ids_(*tempIds_) {} + explicit SingleSelectKeyResult(IdSet::Ptr &&ids) noexcept : tempIds_(std::move(ids)), ids_(*tempIds_) {} explicit SingleSelectKeyResult(const IdSetRef &ids) noexcept : ids_(ids) {} explicit SingleSelectKeyResult(IdType rBegin, IdType rEnd) noexcept : rBegin_(rBegin), rEnd_(rEnd), isRange_(true) {} SingleSelectKeyResult(const SingleSelectKeyResult &other) noexcept @@ -253,7 +253,7 @@ class SelectKeyResult : public h_vector { } clear(); deferedExplicitSort = false; - emplace_back(mergedIds); + emplace_back(IdSet::Ptr(mergedIds)); return mergedIds; } }; diff --git a/cpp_src/core/shardingproxy.cc b/cpp_src/core/shardingproxy.cc index 21a201155..985012662 100644 --- a/cpp_src/core/shardingproxy.cc +++ b/cpp_src/core/shardingproxy.cc @@ -3,6 +3,7 @@ #include "cluster/sharding/shardingcontrolrequest.h" #include "cluster/stats/replicationstats.h" #include "core/defnsconfigs.h" +#include "estl/smart_lock.h" #include "parallelexecutor.h" #include "tools/catch_and_return.h" #include "tools/compiletimemap.h" @@ -114,22 +115,28 @@ ShardingProxy::ShardingProxy(ReindexerConfig cfg) Error ShardingProxy::Connect(const std::string &dsn, ConnectOpts opts) { try { + bool connected = connected_.load(std::memory_order_acquire); + // Expecting for the first time Connect is being called under exlusive lock. + // And all the subsequent calls will be perfomed under shared locks. + smart_lock lck(connectMtx_, !connected); + Error err = impl_.Connect(dsn, opts); if (!err.ok()) { return err; } - // Expecting for the first time Connect is being called under exlusive lock. - // And all the subsequent calls will be perfomed under shared locks. - auto configPtr = impl_.GetShardingConfig(); - if (auto lockedShardingRouter = shardingRouter_.UniqueLock(); !lockedShardingRouter && configPtr) { - lockedShardingRouter = std::make_shared(impl_, *configPtr); - err = lockedShardingRouter->Start(); - if (err.ok()) { - shardingInitialized_.store(true, std::memory_order_release); - } else { - lockedShardingRouter.Reset(); - shardingInitialized_.store(false, std::memory_order_release); + if (!connected && !connected_.load(std::memory_order_relaxed)) { + auto configPtr = impl_.GetShardingConfig(); + if (auto lockedShardingRouter = shardingRouter_.UniqueLock(); !lockedShardingRouter && configPtr) { + lockedShardingRouter = std::make_shared(impl_, *configPtr); + err = lockedShardingRouter->Start(); + if (err.ok()) { + shardingInitialized_.store(true, std::memory_order_release); + } else { + lockedShardingRouter.Reset(); + shardingInitialized_.store(false, std::memory_order_release); + } } + connected_.store(err.ok(), std::memory_order_release); } return err; } catch (Error &e) { @@ -255,7 +262,6 @@ Error ShardingProxy::handleNewShardingConfig(const gason::JsonNode &configJSON, auto errReset = execNodes.Exec( ctx, sharding::ConnectionsPtr(connections), &client::Reindexer::ResetShardingConfigCandidate, [](int64_t) { return Error(); }, sourceId); - return Error(err.code(), err.what() + (!errReset.ok() ? ".\n" + errReset.what() : "")); } @@ -452,19 +458,17 @@ void ShardingProxy::saveShardingCfgCandidateImpl(cluster::ShardingConfig config, sourceId, lockedConfigCandidate.SourceId()); } config.sourceId = sourceId; - - lockedConfigCandidate.EnableReseter(false); - if (lockedConfigCandidate.Reseter().joinable()) lockedConfigCandidate.Reseter().join(); - lockedConfigCandidate.EnableReseter(); + lockedConfigCandidate.ShutdownReseter(); lockedConfigCandidate.SourceId() = sourceId; + const auto cfgTimeout = config.configRollbackTimeout; lockedConfigCandidate.Config() = std::move(config); logPrintf(LogInfo, "New sharding config candidate saved. Source - %d", sourceId); - lockedConfigCandidate.Reseter() = std::thread([this]() { + lockedConfigCandidate.InitReseterThread([this, cfgTimeout]() { using std::this_thread::sleep_for; - const auto timeout = std::chrono::seconds(30); + const auto timeout = cfgTimeout.count() > 0 ? cfgTimeout : std::chrono::seconds(30); const auto period = std::chrono::milliseconds(100); auto iters = timeout / period; @@ -599,16 +603,18 @@ void ShardingProxy::resetConfigCandidate(const sharding::ResetConfigCommand &dat int64_t sourceId = data.sourceId; auto lockedConfigCandidate = configCandidate_.UniqueLock(ctx); - if (!lockedConfigCandidate.Config()) return; + if (!lockedConfigCandidate.Config()) { + logPrintf(LogInfo, "Sharding config candidate reset was skipped. Source - %d", sourceId); + lockedConfigCandidate.ShutdownReseter(); + return; + } - if (lockedConfigCandidate.SourceId() != sourceId) + if (lockedConfigCandidate.SourceId() != sourceId) { throw Error(errParams, "Attempt to reset candidate with a different sourceId - %d. Current sourceId - %d", lockedConfigCandidate.SourceId(), sourceId); + } - lockedConfigCandidate.EnableReseter(false); - if (lockedConfigCandidate.Reseter().joinable()) lockedConfigCandidate.Reseter().join(); - lockedConfigCandidate.EnableReseter(); - + lockedConfigCandidate.ShutdownReseter(); lockedConfigCandidate.Config() = std::nullopt; logPrintf(LogInfo, "Sharding config candidate was reseted. Source - %d", sourceId); } diff --git a/cpp_src/core/shardingproxy.h b/cpp_src/core/shardingproxy.h index 8314562ce..5d074f069 100644 --- a/cpp_src/core/shardingproxy.h +++ b/cpp_src/core/shardingproxy.h @@ -1,7 +1,7 @@ #pragma once -#include #include "clusterproxy.h" +#include "estl/shared_mutex.h" namespace reindexer { @@ -63,7 +63,16 @@ class ShardingProxy { Error GetSqlSuggestions(std::string_view sqlQuery, int pos, std::vector &suggestions, const RdxContext &ctx) { return impl_.GetSqlSuggestions(sqlQuery, pos, suggestions, ctx); } - Error Status() { return impl_.Status(); } + Error Status() noexcept { + if (connected_.load(std::memory_order_acquire)) { + return {}; + } + auto st = impl_.Status(); + if (st.ok()) { + return Error(errNotValid, "Reindexer's sharding proxy layer was not initialized properly"); + } + return st; + } Error GetProtobufSchema(WrSerializer &ser, std::vector &namespaces) { return impl_.GetProtobufSchema(ser, namespaces); } Error GetReplState(std::string_view nsName, ReplicationStateV2 &state, const RdxContext &ctx) { return impl_.GetReplState(nsName, state, ctx); @@ -257,13 +266,26 @@ class ShardingProxy { private: template struct ConfigCandidateTSWrapper { - ConfigCandidateTSWrapper(Locker lock, ConfigCandidateType &configCandidate) + ConfigCandidateTSWrapper(Locker &&lock, ConfigCandidateType &configCandidate) noexcept : lock_(std::move(lock)), configCandidate_(configCandidate) {} auto &SourceId() const { return configCandidate_.sourceId_; } auto &Config() const { return configCandidate_.config_; } - auto &Reseter() const { return configCandidate_.reseter_; } - void EnableReseter(bool enable = true) noexcept { configCandidate_.reseterEnabled_ = enable; } + template + void InitReseterThread(F &&f) const { + if (configCandidate_.reseter_.joinable()) { + throw Error(errLogic, "Sharding config candidate's reset thread is already running"); + } + + configCandidate_.reseter_ = std::thread(std::forward(f)); + } + void ShutdownReseter() noexcept { + if (configCandidate_.reseter_.joinable()) { + configCandidate_.reseterEnabled_ = false; + configCandidate_.reseter_.join(); + configCandidate_.reseterEnabled_ = true; + } + } private: Locker lock_; @@ -312,6 +334,8 @@ class ShardingProxy { ConfigCandidate configCandidate_; std::atomic_bool shardingInitialized_ = {false}; + std::atomic_bool connected_ = {false}; + mutable shared_timed_mutex connectMtx_; ActivityContainer activities_; // this is required because newConfig process methods are private diff --git a/cpp_src/core/sortingprioritiestable.cc b/cpp_src/core/sortingprioritiestable.cc index c98ee154d..8f86f59b1 100644 --- a/cpp_src/core/sortingprioritiestable.cc +++ b/cpp_src/core/sortingprioritiestable.cc @@ -1,14 +1,15 @@ #include "sortingprioritiestable.h" #include -#include "tools/assertrx.h" #include "tools/errors.h" #include "tools/stringstools.h" -using namespace reindexer; +namespace reindexer { SortingPrioritiesTable::SortingPrioritiesTable(const std::string& sortOrderUTF8) - : sortOrder_(std::make_shared()), sortOrderCharacters_(sortOrderUTF8) { - if (sortOrderCharacters_.empty()) throw Error(errLogic, "Custom sort format string cannot be empty!"); + : sortOrder_(make_intrusive()), sortOrderCharacters_(sortOrderUTF8) { + if (sortOrderCharacters_.empty()) { + throw Error(errLogic, "Custom sort format string cannot be empty!"); + } wchar_t prevCh = 0; uint16_t priority = 0; @@ -47,7 +48,7 @@ SortingPrioritiesTable::SortingPrioritiesTable(const std::string& sortOrderUTF8) if (!ranges.empty()) { auto rangeIt = ranges.begin(); uint16_t outOfRangePriority = maxPriority; - for (size_t i = 0; i < tableSize;) { + for (size_t i = 0; i < kTableSize;) { if ((rangeIt != ranges.end()) && (rangeIt->first == i)) { i += rangeIt->second; ++rangeIt; @@ -69,11 +70,4 @@ bool SortingPrioritiesTable::checkForRangeIntersection(std::map(c) < tableSize); - uint16_t ch(static_cast(c)); - return sortOrder_->operator[](ch); -} - -const std::string& SortingPrioritiesTable::GetSortOrderCharacters() const { return sortOrderCharacters_; } +} // namespace reindexer diff --git a/cpp_src/core/sortingprioritiestable.h b/cpp_src/core/sortingprioritiestable.h index 54331149a..0dc957eb3 100644 --- a/cpp_src/core/sortingprioritiestable.h +++ b/cpp_src/core/sortingprioritiestable.h @@ -2,8 +2,9 @@ #include #include -#include #include +#include "estl/intrusive_ptr.h" +#include "tools/assertrx.h" #include "type_consts.h" namespace reindexer { @@ -20,12 +21,17 @@ class SortingPrioritiesTable { explicit SortingPrioritiesTable(const std::string& sortOrderUTF8); /// Returns priority of a character. - /// @param ch - character. + /// @param c - character /// @returns int priority value - int GetPriority(wchar_t ch) const; + int GetPriority(wchar_t c) const noexcept { + assertrx(sortOrder_.get() != nullptr); + // assertrx(static_cast(c) < tableSize); + uint16_t ch(static_cast(c)); + return sortOrder_->operator[](ch); + } /// @returns string of sort order characters - const std::string& GetSortOrderCharacters() const; + const std::string& GetSortOrderCharacters() const noexcept { return sortOrderCharacters_; } private: /// Checks whether ch is in existing ranges ir not. @@ -34,10 +40,11 @@ class SortingPrioritiesTable { /// @returns true, if character is in one of existing ranges already. bool checkForRangeIntersection(std::map& ranges, wchar_t ch); - static const uint32_t tableSize = 0x10000; - using SortOrderTable = std::array; - using SortOrderTablePtr = std::shared_ptr; + constexpr static uint32_t kTableSize = 0x10000; + using SortOrderTable = intrusive_atomic_rc_wrapper>; + using SortOrderTablePtr = intrusive_ptr; SortOrderTablePtr sortOrder_; std::string sortOrderCharacters_; }; + } // namespace reindexer diff --git a/cpp_src/estl/h_vector.h b/cpp_src/estl/h_vector.h index 56333004d..96c11fa5b 100644 --- a/cpp_src/estl/h_vector.h +++ b/cpp_src/estl/h_vector.h @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include "debug_macros.h" +#include "estl/defines.h" #include "trivial_reverse_iterator.h" namespace reindexer { @@ -39,6 +41,9 @@ class h_vector { typedef trivial_reverse_iterator reverse_iterator; typedef unsigned size_type; typedef std::ptrdiff_t difference_type; + static_assert(std::is_trivial_v, "Expecting trivial reverse iterator"); + static_assert(std::is_trivial_v, "Expecting trivial const reverse iterator"); + h_vector() noexcept : e_{0, 0}, size_(0), is_hdata_(1) {} explicit h_vector(size_type size) : h_vector() { resize(size); } h_vector(size_type size, const T& v) : h_vector() { @@ -56,7 +61,8 @@ class h_vector { reserve(other.capacity()); const pointer p = ptr(); const_pointer op = other.ptr(); - for (size_type i = 0; i < other.size(); i++) { + const size_type osz = other.size(); + for (size_type i = 0; i < osz; i++) { new (p + i) T(op[i]); } size_ = other.size_; @@ -65,9 +71,10 @@ class h_vector { if (other.is_hdata()) { const pointer p = reinterpret_cast(hdata_); const pointer op = reinterpret_cast(other.hdata_); - for (size_type i = 0; i < other.size(); i++) { + const size_type osz = other.size(); + for (size_type i = 0; i < osz; i++) { new (p + i) T(std::move(op[i])); - if constexpr (!std::is_trivially_destructible::value) { + if constexpr (!std::is_trivially_destructible_v) { op[i].~T(); } } @@ -89,11 +96,13 @@ class h_vector { size_type i = mv; const pointer p = ptr(); const_pointer op = other.ptr(); - for (; i < other.size(); i++) { + const auto osz = other.size(); + for (; i < osz; i++) { new (p + i) T(op[i]); } - if constexpr (!std::is_trivially_destructible::value) { - for (; i < size(); i++) p[i].~T(); + if constexpr (!std::is_trivially_destructible_v) { + const auto old_sz = size(); + for (; i < old_sz; i++) p[i].~T(); } size_ = other.size_; } @@ -104,11 +113,12 @@ class h_vector { if (&other != this) { clear(); if (other.is_hdata()) { - for (size_type i = 0; i < other.size(); i++) { - const pointer p = ptr(); - const pointer op = other.ptr(); + const size_type osz = other.size(); + const pointer p = ptr(); + const pointer op = other.ptr(); + for (size_type i = 0; i < osz; i++) { new (p + i) T(std::move(op[i])); - if constexpr (!std::is_trivially_destructible::value) { + if constexpr (!std::is_trivially_destructible_v) { op[i].~T(); } } @@ -126,9 +136,10 @@ class h_vector { bool operator==(const h_vector& other) const noexcept(noexcept(std::declval() == std::declval())) { if (&other != this) { - if (size() != other.size()) return false; - for (size_t i = 0; i < size(); ++i) { - if (!(at(i) == other.at(i))) return false; + const size_type sz = size_; + if (sz != other.size()) return false; + for (size_t i = 0; i < sz; ++i) { + if (!(operator[](i) == other[i])) return false; } return true; } @@ -138,6 +149,8 @@ class h_vector { return !operator==(other); } + static constexpr size_type max_size() noexcept { return std::numeric_limits::max() >> 1; } + template void clear() noexcept { if constexpr (FreeHeapMemory) { @@ -145,7 +158,8 @@ class h_vector { is_hdata_ = 1; } else if constexpr (!std::is_trivially_destructible_v) { const pointer p = ptr(); - for (size_type i = 0; i < size_; ++i) p[i].~T(); + const size_type sz = size_; + for (size_type i = 0; i < sz; ++i) p[i].~T(); } size_ = 0; } @@ -156,26 +170,10 @@ class h_vector { const_iterator end() const noexcept { return ptr() + size_; } const_iterator cbegin() const noexcept { return ptr(); } const_iterator cend() const noexcept { return ptr() + size_; } - const_reverse_iterator rbegin() const noexcept { - const_reverse_iterator it; - it = end(); - return it; - } - const_reverse_iterator rend() const noexcept { - const_reverse_iterator it; - it = begin(); - return it; - } - reverse_iterator rbegin() noexcept { - reverse_iterator it; - it = end(); - return it; - } - reverse_iterator rend() noexcept { - reverse_iterator it; - it = begin(); - return it; - } + const_reverse_iterator rbegin() const noexcept { return end(); } + const_reverse_iterator rend() const noexcept { return begin(); } + reverse_iterator rbegin() noexcept { return end(); } + reverse_iterator rend() noexcept { return begin(); } size_type size() const noexcept { return size_; } size_type capacity() const noexcept { return is_hdata_ ? holdSize : e_.cap_; } bool empty() const noexcept { return size_ == 0; } @@ -188,13 +186,13 @@ class h_vector { return ptr()[pos]; } const_reference at(size_type pos) const { - if (pos >= size()) { + if rx_unlikely (pos >= size()) { throw std::logic_error("h_vector: Out of range (pos: " + std::to_string(pos) + ", size: " + std::to_string(size())); } return ptr()[pos]; } reference at(size_type pos) { - if (pos >= size()) { + if rx_unlikely (pos >= size()) { throw std::logic_error("h_vector: Out of range (pos: " + std::to_string(pos) + ", size: " + std::to_string(size())); } return ptr()[pos]; @@ -222,34 +220,49 @@ class h_vector { grow(sz); if constexpr (!reindexer::is_trivially_default_constructible::value) { const pointer p = ptr(); - for (size_type i = size_; i < sz; ++i) new (p + i) T(); + const size_type old_sz = size_; + for (size_type i = old_sz; i < sz; ++i) new (p + i) T(); } - if constexpr (!std::is_trivially_destructible::value) { + if constexpr (!std::is_trivially_destructible_v) { const pointer p = ptr(); - for (size_type i = sz; i < size_; ++i) p[i].~T(); + const size_type old_sz = size_; + for (size_type i = sz; i < old_sz; ++i) p[i].~T(); } size_ = sz; } void resize(size_type sz, const T& default_value) { grow(sz); - for (size_type i = size_; i < sz; i++) new (ptr() + i) T(default_value); - if constexpr (!std::is_trivially_destructible::value) { - for (size_type i = sz; i < size_; i++) ptr()[i].~T(); + const size_type old_sz = size_; + const pointer p = ptr(); + for (size_type i = old_sz; i < sz; ++i) { + new (p + i) T(default_value); + } + if constexpr (!std::is_trivially_destructible_v) { + for (size_type i = sz; i < old_sz; ++i) { + p[i].~T(); + } } size_ = sz; } void reserve(size_type sz) { if (sz > capacity()) { - if (sz <= holdSize) { - throw std::logic_error("Unexpected reserved size"); + if rx_unlikely (sz > max_size()) { + throw std::logic_error("h_vector: max capacity overflow (requested: " + std::to_string(sz) + + ", max_size: " + std::to_string(max_size()) + " )"); + } + if rx_unlikely (sz <= holdSize) { + throw std::logic_error("h_vector: unexpected reserved size"); } // NOLINTNEXTLINE(bugprone-sizeof-expression) pointer new_data = static_cast(operator new(sz * sizeof(T))); // ?? dynamic pointer oold_data = ptr(); pointer old_data = oold_data; - for (size_type i = 0; i < size_; i++) { + // Creating those explicit old_sz variable for better vectorization + for (size_type i = 0, old_sz = size_; i < old_sz; ++i) { new (new_data + i) T(std::move(*old_data)); - if (!std::is_trivially_destructible::value) old_data->~T(); + if constexpr (!std::is_trivially_destructible_v) { + old_data->~T(); + } ++old_data; } if (!is_hdata()) operator delete(oold_data); @@ -260,17 +273,19 @@ class h_vector { } void grow(size_type sz) { const auto cap = capacity(); - if (sz > cap) reserve(std::max(sz, cap * 2)); + if (sz > cap) { + reserve(std::max(sz, std::min(max_size(), cap * 2))); + } } void push_back(const T& v) { grow(size_ + 1); new (ptr() + size_) T(v); - size_++; + ++size_; } void push_back(T&& v) { grow(size_ + 1); new (ptr() + size_) T(std::move(v)); - size_++; + ++size_; } template reference emplace_back(Args&&... args) { @@ -282,7 +297,7 @@ class h_vector { } void pop_back() { rx_debug_check_nonempty(); - if constexpr (!std::is_trivially_destructible::value) { + if constexpr (!std::is_trivially_destructible_v) { ptr()[--size_].~T(); } else { --size_; @@ -294,10 +309,11 @@ class h_vector { push_back(v); } else { rx_debug_check_subscript(i); - grow(size_ + 1); + const size_type sz = size_; + grow(sz + 1); const pointer p = ptr(); - new (p + size_) T(std::move(p[size_ - 1])); - for (size_type j = size_ - 1; j > i; --j) { + new (p + sz) T(std::move(p[sz - 1])); + for (size_type j = sz - 1; j > i; --j) { p[j] = std::move(p[j - 1]); } p[i] = v; @@ -311,10 +327,11 @@ class h_vector { push_back(std::move(v)); } else { rx_debug_check_subscript(i); - grow(size_ + 1); + const size_type sz = size_; + grow(sz + 1); const pointer p = ptr(); - new (p + size_) T(std::move(p[size_ - 1])); - for (size_type j = size_ - 1; j > i; --j) { + new (p + sz) T(std::move(p[sz - 1])); + for (size_type j = sz - 1; j > i; --j) { p[j] = std::move(p[j - 1]); } p[i] = std::move(v); @@ -326,16 +343,17 @@ class h_vector { if (count == 0) return const_cast(pos); difference_type i = pos - begin(); rx_debug_check_subscript_le(i); - grow(size_ + count); + const size_type sz = size_; + grow(sz + count); const pointer p = ptr(); - difference_type j = size_ + count - 1; - for (; j >= static_cast(size_) && j >= count + i; --j) { + difference_type j = sz + count - 1; + for (; j >= static_cast(sz) && j >= count + i; --j) { new (p + j) T(std::move(p[j - count])); } for (; j >= count + i; --j) { p[j] = std::move(p[j - count]); } - for (; j >= size_; --j) { + for (; j >= sz; --j) { new (p + j) T(v); } for (; j >= i; --j) { @@ -347,17 +365,18 @@ class h_vector { template iterator emplace(const_iterator pos, Args&&... args) { const size_type i = pos - begin(); - if (i == size()) { + const size_type sz = size_; + if (i == sz) { emplace_back(std::forward(args)...); } else { rx_debug_check_subscript(i); - grow(size_ + 1); + grow(sz + 1); const pointer p = ptr(); - new (p + size_) T(std::move(p[size_ - 1])); - for (size_type j = size_ - 1; j > i; --j) { + new (p + sz) T(std::move(p[sz - 1])); + for (size_type j = sz - 1; j > i; --j) { p[j] = std::move(p[j - 1]); } - p[i] = {std::forward(args)...}; + p[i] = T(std::forward(args)...); ++size_; } return begin() + i; @@ -370,7 +389,7 @@ class h_vector { auto firstPtr = p + i; std::move(firstPtr + 1, p + size_, firstPtr); --size_; - if constexpr (!std::is_trivially_destructible::value) { + if constexpr (!std::is_trivially_destructible_v) { p[size_].~T(); } return firstPtr; @@ -382,16 +401,17 @@ class h_vector { if (cnt == 0) return const_cast(pos); const difference_type i = pos - begin(); rx_debug_check_subscript_le(i); - grow(size_ + cnt); + const size_type sz = size_; + grow(sz + cnt); const pointer p = ptr(); - difference_type j = size_ + cnt - 1; - for (; j >= static_cast(size_) && j >= cnt + i; --j) { + difference_type j = sz + cnt - 1; + for (; j >= static_cast(sz) && j >= cnt + i; --j) { new (p + j) T(std::move(p[j - cnt])); } for (; j >= cnt + i; --j) { p[j] = std::move(p[j - cnt]); } - for (; j >= static_cast(size_); --j) { + for (; j >= static_cast(sz); --j) { new (p + j) T(*--last); } for (; j >= i; --j) { @@ -402,8 +422,20 @@ class h_vector { } template void assign(InputIt first, InputIt last) { - clear(); - insert(begin(), first, last); + static_assert(std::is_same_v::iterator_category, std::random_access_iterator_tag>, + "Expecting random access iterators here"); + rx_debug_check_valid_range(first, last); + const auto cnt = std::distance(first, last); + const auto cap = capacity(); + if (cap >= cnt && cap - (cnt >> 2) <= cnt) { + // Allow up to 25% extra memory + clear(); + } else { + clear(); + grow(cnt); + } + std::uninitialized_copy(first, last, begin()); + size_ = cnt; } iterator erase(const_iterator first, const_iterator last) { rx_debug_check_valid_range(first, last); @@ -416,21 +448,24 @@ class h_vector { return firstPtr; } rx_debug_check_subscript(i); + const size_type sz = size_; - std::move(firstPtr + cnt, p + size_, firstPtr); - const auto newSize = size_ - cnt; - if constexpr (!std::is_trivially_destructible::value) { - for (size_type j = newSize; j < size_; ++j) p[j].~T(); + std::move(std::make_move_iterator(firstPtr + cnt), std::make_move_iterator(p + sz), firstPtr); + const auto newSize = sz - cnt; + if constexpr (!std::is_trivially_destructible_v) { + for (size_type j = newSize; j < sz; ++j) p[j].~T(); } size_ = newSize; return firstPtr; } void shrink_to_fit() { - if (is_hdata() || size_ == capacity()) return; + const auto sz = size(); + if (is_hdata() || sz == capacity()) return; h_vector tmp; - tmp.reserve(size()); - tmp.insert(tmp.begin(), std::make_move_iterator(begin()), std::make_move_iterator(end())); + tmp.reserve(sz); + std::move(std::make_move_iterator(begin()), std::make_move_iterator(end()), tmp.begin()); + tmp.size_ = sz; *this = std::move(tmp); } size_t heap_size() const noexcept { return is_hdata_ ? 0 : e_.cap_ * sizeof(T); } @@ -442,11 +477,13 @@ class h_vector { void destruct() noexcept { if (is_hdata()) { if constexpr (!std::is_trivially_destructible_v) { - for (size_type i = 0; i < size_; ++i) reinterpret_cast(hdata_)[i].~T(); + const size_type sz = size_; + for (size_type i = 0; i < sz; ++i) reinterpret_cast(hdata_)[i].~T(); } } else { if constexpr (!std::is_trivially_destructible_v) { - for (size_type i = 0; i < size_; ++i) e_.data_[i].~T(); + const size_type sz = size_; + for (size_type i = 0; i < sz; ++i) e_.data_[i].~T(); } operator delete(e_.data_); } diff --git a/cpp_src/estl/span.h b/cpp_src/estl/span.h index 1ad2bac9c..d16134e89 100644 --- a/cpp_src/estl/span.h +++ b/cpp_src/estl/span.h @@ -17,6 +17,8 @@ class span { typedef trivial_reverse_iterator const_reverse_iterator; typedef trivial_reverse_iterator reverse_iterator; typedef size_t size_type; + static_assert(std::is_trivial_v, "Expecting trivial reverse iterator"); + static_assert(std::is_trivial_v, "Expecting trivial const reverse iterator"); constexpr span() noexcept : data_(nullptr), size_(0) {} constexpr span(const span& other) noexcept : data_(other.data_), size_(other.size_) {} @@ -44,16 +46,8 @@ class span { constexpr span(T (&arr)[L]) noexcept : data_(arr), size_(L) {} constexpr iterator begin() const noexcept { return data_; } constexpr iterator end() const noexcept { return data_ + size_; } - /*constexpr*/ reverse_iterator rbegin() const noexcept { - reverse_iterator it; - it = end(); - return it; - } - /*constexpr*/ reverse_iterator rend() const noexcept { - reverse_iterator it; - it = begin(); - return it; - } + constexpr reverse_iterator rbegin() const noexcept { return end(); } + constexpr reverse_iterator rend() const noexcept { return begin(); } constexpr size_type size() const noexcept { return size_; } constexpr bool empty() const noexcept { return size_ == 0; } constexpr const T& operator[](size_type pos) const noexcept { return data_[pos]; } diff --git a/cpp_src/estl/trivial_reverse_iterator.h b/cpp_src/estl/trivial_reverse_iterator.h index 3919e6bbb..ccf1aa514 100644 --- a/cpp_src/estl/trivial_reverse_iterator.h +++ b/cpp_src/estl/trivial_reverse_iterator.h @@ -6,7 +6,7 @@ using std::iterator_traits; template class trivial_reverse_iterator { -public: +public: typedef trivial_reverse_iterator this_type; typedef Iterator iterator_type; typedef typename iterator_traits::iterator_category iterator_category; @@ -16,91 +16,83 @@ class trivial_reverse_iterator { typedef typename iterator_traits::pointer pointer; public: - // if CTOR is enabled std::is_trivial> return false; - // trivial_reverse_iterator() : current_(nullptr) {} + constexpr trivial_reverse_iterator() = default; + constexpr trivial_reverse_iterator(Iterator it) noexcept : current_(it) { + static_assert(std::is_trivial_v, "Expecting std::is_trivial_v"); + } template - trivial_reverse_iterator& operator=(const trivial_reverse_iterator& u) { + trivial_reverse_iterator& operator=(const trivial_reverse_iterator& u) noexcept { current_ = u.base(); return *this; } - Iterator base() const { return current_; } - reference operator*() const { + Iterator base() const noexcept { return current_; } + reference operator*() const noexcept { Iterator tmp = current_; return *--tmp; } - pointer operator->() const { return std::addressof(operator*()); } - trivial_reverse_iterator& operator++() { + pointer operator->() const noexcept { return std::addressof(operator*()); } + trivial_reverse_iterator& operator++() noexcept { --current_; return *this; } - trivial_reverse_iterator operator++(int) { + trivial_reverse_iterator operator++(int) noexcept { trivial_reverse_iterator tmp(*this); --current_; return tmp; } - trivial_reverse_iterator& operator--() { + trivial_reverse_iterator& operator--() noexcept { ++current_; return *this; } - trivial_reverse_iterator operator--(int) { + trivial_reverse_iterator operator--(int) noexcept { trivial_reverse_iterator tmp(*this); ++current_; return tmp; } - trivial_reverse_iterator operator+(difference_type n) const { - Iterator ptr = current_ - n; - trivial_reverse_iterator tmp; - tmp = ptr; - return tmp; - } - trivial_reverse_iterator& operator+=(difference_type n) { + trivial_reverse_iterator operator+(difference_type n) const noexcept { return current_ - n; } + trivial_reverse_iterator& operator+=(difference_type n) noexcept { current_ -= n; return *this; } - trivial_reverse_iterator operator-(difference_type n) const { - Iterator ptr = current_ + n; - trivial_reverse_iterator tmp; - tmp = ptr; - return tmp; - } - trivial_reverse_iterator& operator-=(difference_type n) { + trivial_reverse_iterator operator-(difference_type n) const noexcept { return current_ + n; } + trivial_reverse_iterator& operator-=(difference_type n) noexcept { current_ += n; return *this; } - reference operator[](difference_type n) const { return *(*this + n); } + reference operator[](difference_type n) const noexcept { return *(*this + n); } // Assign operator overloading from const std::reverse_iterator template - trivial_reverse_iterator& operator=(const std::reverse_iterator& u) { + trivial_reverse_iterator& operator=(const std::reverse_iterator& u) noexcept { if (current_ != u.base()) current_ = u.base(); return *this; } // Assign operator overloading from non-const std::reverse_iterator template - trivial_reverse_iterator& operator=(std::reverse_iterator& u) { + trivial_reverse_iterator& operator=(std::reverse_iterator& u) noexcept { if (current_ != u.base()) current_ = u.base(); return *this; } // Assign native pointer template - trivial_reverse_iterator& operator=(Upn ptr) { + trivial_reverse_iterator& operator=(Upn ptr) noexcept { static_assert(std::is_pointer::value, "attempting assign a non-trivial pointer"); /*if (current_ != ptr)*/ current_ = ptr; return *this; } - inline bool operator!=(const this_type& rhs) const { return !EQ(current_, rhs.current_); } - inline bool operator==(const this_type& rhs) const { return EQ(current_, rhs.current_); } + inline bool operator!=(const this_type& rhs) const noexcept { return !EQ(current_, rhs.current_); } + inline bool operator==(const this_type& rhs) const noexcept { return EQ(current_, rhs.current_); } protected: Iterator current_; private: - inline bool EQ(Iterator lhs, Iterator rhs) const { return lhs == rhs; } + inline bool EQ(Iterator lhs, Iterator rhs) const noexcept { return lhs == rhs; } }; } // namespace reindexer diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc index 7c8cec8b9..2fe2dd32a 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc @@ -111,6 +111,9 @@ void ApiTvSimple::RegisterAllCases() { Register("FromCJSONPKOnly", &ApiTvSimple::FromCJSONPKOnly, this); Register("GetCJSON", &ApiTvSimple::GetCJSON, this); Register("ExtractField", &ApiTvSimple::ExtractField, this); + Register("SubQueryEq", &ApiTvSimple::SubQueryEq, this); + Register("SubQuerySet", &ApiTvSimple::SubQuerySet, this); + Register("SubQueryAggregate", &ApiTvSimple::SubQueryAggregate, this); // Those benches should be last, because they are recreating indexes cache Register("Query4CondRangeDropCache", &ApiTvSimple::Query4CondRangeDropCache, this)->Iterations(1000); @@ -197,12 +200,14 @@ reindexer::Error ApiTvSimple::Initialize() { err = db_->Commit(stringSelectNs_); if (!err.ok()) return err; - NamespaceDef mainNsDef{innerJoinLowSelectivityMainNs_}; + NamespaceDef mainNsDef{mainNs_}; mainNsDef.AddIndex("id", "hash", "int", IndexOpts().PK()).AddIndex("field", "hash", "int", IndexOpts()); err = db_->AddNamespace(mainNsDef); if (!err.ok()) return err; - NamespaceDef rightNsDef{innerJoinLowSelectivityRightNs_}; - rightNsDef.AddIndex("id", "hash", "int", IndexOpts().PK()).AddIndex("field", "hash", "int", IndexOpts()); + NamespaceDef rightNsDef{rightNs_}; + rightNsDef.AddIndex("id", "hash", "int", IndexOpts().PK()) + .AddIndex("field", "hash", "int", IndexOpts()) + .AddIndex("id_tree", "tree", "int", IndexOpts()); err = db_->AddNamespace(rightNsDef); if (!err.ok()) return err; @@ -227,6 +232,7 @@ reindexer::Error ApiTvSimple::Initialize() { reindexer::JsonBuilder bld2(wrSer_); bld2.Put("id", i); bld2.Put("field", i); + bld2.Put("id_tree", i); bld2.End(); err = rItem.FromJSON(wrSer_.Slice()); if (!err.ok()) return err; @@ -805,9 +811,9 @@ void ApiTvSimple::Query0CondInnerJoinUnlimit(benchmark::State& state) { void ApiTvSimple::Query0CondInnerJoinUnlimitLowSelectivity(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - Query q4join(innerJoinLowSelectivityRightNs_); + Query q4join(rightNs_); q4join.Where("id", CondLe, 250); - Query q(innerJoinLowSelectivityMainNs_); + Query q(mainNs_); q.InnerJoin("id", "id", CondEq, std::move(q4join)).ReqTotal(); QueryResults qres; @@ -816,6 +822,43 @@ void ApiTvSimple::Query0CondInnerJoinUnlimitLowSelectivity(benchmark::State& sta } } +void ApiTvSimple::SubQueryEq(benchmark::State& state) { + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + Query q = Query(mainNs_).Where( + "id", CondEq, Query(rightNs_).Select({"field"}).Where("id", CondEq, VariantArray::Create(int(rand() % kTotalItemsMainJoinNs)))); + QueryResults qres; + auto err = db_->Select(q, qres); + if (!err.ok()) state.SkipWithError(err.what().c_str()); + } +} + +void ApiTvSimple::SubQuerySet(benchmark::State& state) { + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + const int rangeMin = rand() % (kTotalItemsMainJoinNs - 500); + Query q = Query(mainNs_).Where( + "id", CondSet, Query(rightNs_).Select({"id"}).Where("id_tree", CondRange, VariantArray::Create(rangeMin, rangeMin + 500))); + QueryResults qres; + auto err = db_->Select(q, qres); + if (!err.ok()) state.SkipWithError(err.what().c_str()); + } +} + +void ApiTvSimple::SubQueryAggregate(benchmark::State& state) { + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + Query q = Query(mainNs_).Where("id", CondEq, + Query(rightNs_) + .Aggregate(AggAvg, {"id"}) + .Where("id", CondLt, VariantArray::Create(int(rand() % kTotalItemsMainJoinNs))) + .Limit(500)); + QueryResults qres; + auto err = db_->Select(q, qres); + if (!err.ok()) state.SkipWithError(err.what().c_str()); + } +} + void ApiTvSimple::Query2CondInnerJoin(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) @@ -1202,7 +1245,7 @@ void ApiTvSimple::query2CondIdSet(benchmark::State& state, const std::vector>& idsets); reindexer::Error prepareCJsonBench(); @@ -147,8 +150,8 @@ class ApiTvSimple : private BaseFixture { std::unordered_map>> idsets_; reindexer::WrSerializer wrSer_; std::string stringSelectNs_{"string_select_ns"}; - std::string innerJoinLowSelectivityMainNs_{"inner_join_low_selectivity_main_ns"}; - std::string innerJoinLowSelectivityRightNs_{"inner_join_low_selectivity_right_ns"}; + std::string mainNs_{"main_ns"}; + std::string rightNs_{"right_ns"}; std::string cjsonNsName_{"cjson_ns_name"}; std::unique_ptr itemForCjsonBench_; std::vector fieldsToExtract_; diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc index 30846061c..4ffafedc9 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc @@ -4,7 +4,7 @@ #include "core/cjson/jsonbuilder.h" #include "core/nsselecter/joinedselector.h" #include "core/reindexer.h" -// #include "gtests/tools.h" +#include "gtests/tools.h" #include "tools/string_regexp_functions.h" #include "helpers.h" @@ -51,6 +51,7 @@ void ApiTvSimpleComparators::RegisterAllCases() { Register("GetEqArrayInt", &ApiTvSimpleComparators::GetEqArrayInt, this); Register("GetEqString", &ApiTvSimpleComparators::GetEqString, this); Register("GetByRangeIDAndSort", &ApiTvSimpleComparators::GetByRangeIDAndSort, this); + Register("GetUuidStr", &ApiTvSimpleComparators::GetUuidStr, this); Register("Query1Cond", &ApiTvSimpleComparators::Query1Cond, this); Register("Query1CondTotal", &ApiTvSimpleComparators::Query1CondTotal, this); @@ -93,6 +94,11 @@ reindexer::Error ApiTvSimpleComparators::Initialize() { locations_ = {"mos", "ct", "dv", "sth", "vlg", "sib", "ural"}; + uuids_.reserve(1000); + for (size_t i = 0; i < 1000; ++i) { + uuids_.emplace_back(randStrUuid()); + } + for (int i = 0; i < 10; i++) packages_.emplace_back(randomNumArray(20, 10000, 10)); for (int i = 0; i < 20; i++) priceIDs_.emplace_back(randomNumArray(10, 7000, 50)); @@ -159,6 +165,7 @@ reindexer::Item ApiTvSimpleComparators::MakeItem(benchmark::State&) { item["location"] = locations_.at(random(0, locations_.size() - 1)); item["start_time"] = start_times_.at(random(0, start_times_.size() - 1)); item["end_time"] = startTime + random(1, 5) * 1000; + item["uuid_str"] = uuids_[rand() % uuids_.size()]; return item; } @@ -237,6 +244,19 @@ void ApiTvSimpleComparators::GetByRangeIDAndSort(benchmark::State& state) { } } +void ApiTvSimpleComparators::GetUuidStr(benchmark::State& state) { + const auto& uuid = uuids_[rand() % uuids_.size()]; + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + Query q(nsdef_.name); + q.Where("uuid_str", CondEq, uuid); + QueryResults qres; + auto err = db_->Select(q, qres); + if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!qres.Count()) state.SkipWithError("Results does not contain any value"); + } +} + void ApiTvSimpleComparators::Query1Cond(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.h b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.h index 96e5167f8..91e769bce 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.h +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.h @@ -19,7 +19,8 @@ class ApiTvSimpleComparators : private BaseFixture { .AddIndex("price_id", "-", "int", IndexOpts().Array()) .AddIndex("location", "-", "string", IndexOpts()) .AddIndex("end_time", "-", "int", IndexOpts()) - .AddIndex("start_time", "-", "int", IndexOpts()); + .AddIndex("start_time", "-", "int", IndexOpts()) + .AddIndex("uuid_str", "-", "string", IndexOpts()); } void RegisterAllCases(); @@ -36,6 +37,7 @@ class ApiTvSimpleComparators : private BaseFixture { void GetEqArrayInt(State& state); void GetEqString(State& state); void GetByRangeIDAndSort(State& state); + void GetUuidStr(State& state); void Query1Cond(State& state); void Query1CondTotal(State& state); @@ -61,6 +63,7 @@ class ApiTvSimpleComparators : private BaseFixture { std::vector start_times_; std::vector> packages_; std::vector> priceIDs_; + std::vector uuids_; #if !defined(REINDEX_WITH_ASAN) && !defined(REINDEX_WITH_TSAN) && !defined(RX_WITH_STDLIB_DEBUG) constexpr static unsigned kTotalItemsStringSelectNs = 100'000; #else // !defined(REINDEX_WITH_ASAN) && !defined(REINDEX_WITH_TSAN) && !defined(RX_WITH_STDLIB_DEBUG) diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.cc b/cpp_src/gtests/bench/fixtures/ft_fixture.cc index b0f103cc9..279888e70 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.cc +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.cc @@ -368,16 +368,16 @@ void FullText::Fast3PhraseLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w3 = + const std::string& w3 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + w3.size() + 32); ftQuery.append("'").append(w1).append(" ").append(w2).append("' ").append(w3); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); QueryResults qres; auto err = db_->Select(q, qres); @@ -394,16 +394,16 @@ void FullText::Fast3WordsLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w3 = + const std::string& w3 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + w3.size() + 32); ftQuery.append("+").append(w1).append(" +").append(w2).append(" +").append(w3); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); QueryResults qres; auto err = db_->Select(q, qres); @@ -420,14 +420,14 @@ void FullText::Fast2PhraseLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + 32); ftQuery.append("'").append(w1).append(" ").append(w2).append("'~50"); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); QueryResults qres; auto err = db_->Select(q, qres); @@ -444,14 +444,14 @@ void FullText::Fast2AndWordLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + 32); ftQuery.append("+").append(w1).append(" +").append(w2); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); QueryResults qres; auto err = db_->Select(q, qres); @@ -468,16 +468,16 @@ void FullText::Fast3PhraseWithAreasLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w3 = + const std::string& w3 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + w3.size() + 32); ftQuery.append("'").append(w1).append(" ").append(w2).append("' ").append(w3); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); q.AddFunction("search = highlight(!,!)"); QueryResults qres; auto err = db_->Select(q, qres); @@ -492,7 +492,7 @@ void FullText::Fast1WordWithAreaHighDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - std::string& word = + const std::string& word = words_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words_.size() - 1)})); q.Where("searchfast", CondEq, word); q.AddFunction("search = highlight(!,!)"); @@ -509,16 +509,16 @@ void FullText::Fast3WordsWithAreasLowDiversity(State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(lowWordsDiversityNsDef_.name); - std::string& w1 = + const std::string& w1 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w2 = + const std::string& w2 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); - std::string& w3 = + const std::string& w3 = words2_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words2_.size() - 1)})); std::string ftQuery; ftQuery.reserve(w1.size() + w2.size() + w3.size() + 32); ftQuery.append(w1).append(" ").append(w2).append(" ").append(w3); - q.Where("search", CondEq, ftQuery); + q.Where("search", CondEq, std::move(ftQuery)); q.AddFunction("search = highlight(!,!)"); QueryResults qres; auto err = db_->Select(q, qres); @@ -603,7 +603,7 @@ void FullText::Fast2WordsMatch(benchmark::State& state) { words_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words_.size() - 1)})) + " " + words_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words_.size() - 1)})); - q.Where("searchfast", CondEq, words); + q.Where("searchfast", CondEq, std::move(words)); QueryResults qres; auto err = db_->Select(q, qres); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -639,7 +639,7 @@ void FullText::Fuzzy2WordsMatch(benchmark::State& state) { words_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words_.size() - 1)})) + " " + words_.at(randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(words_.size() - 1)})); - q.Where("searchfuzzy", CondEq, words); + q.Where("searchfuzzy", CondEq, std::move(words)); QueryResults qres; auto err = db_->Select(q, qres); @@ -656,9 +656,7 @@ void FullText::Fast1PrefixMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - - auto word = MakePrefixWord(); - q.Where("searchfast", CondEq, word); + q.Where("searchfast", CondEq, MakePrefixWord()); QueryResults qres; auto err = db_->Select(q, qres); @@ -675,9 +673,7 @@ void FullText::Fast2PrefixMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - - auto words = MakePrefixWord() + " " + MakePrefixWord(); - q.Where("searchfast", CondEq, words); + q.Where("searchfast", CondEq, MakePrefixWord().append(" ").append(MakePrefixWord())); QueryResults qres; auto err = db_->Select(q, qres); @@ -692,9 +688,7 @@ void FullText::Fuzzy1PrefixMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - auto word = MakePrefixWord(); - q.Where("searchfuzzy", CondEq, word); + q.Where("searchfuzzy", CondEq, MakePrefixWord()); QueryResults qres; auto err = db_->Select(q, qres); @@ -709,9 +703,7 @@ void FullText::Fuzzy2PrefixMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - std::string words = MakePrefixWord() + " " + MakePrefixWord(); - q.Where("searchfuzzy", CondEq, words); + q.Where("searchfuzzy", CondEq, MakePrefixWord().append(" ").append(MakePrefixWord())); QueryResults qres; auto err = db_->Select(q, qres); @@ -728,8 +720,7 @@ void FullText::Fast1SuffixMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - std::string word = MakeSuffixWord(); - q.Where("searchfast", CondEq, word); + q.Where("searchfast", CondEq, MakeSuffixWord()); QueryResults qres; auto err = db_->Select(q, qres); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -745,9 +736,7 @@ void FullText::Fast2SuffixMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - - std::string words = MakeSuffixWord() + " " + MakeSuffixWord(); - q.Where("searchfast", CondEq, words); + q.Where("searchfast", CondEq, MakeSuffixWord().append(" ").append(MakeSuffixWord())); QueryResults qres; auto err = db_->Select(q, qres); @@ -762,9 +751,7 @@ void FullText::Fuzzy1SuffixMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - std::string word = MakeSuffixWord(); - q.Where("searchfuzzy", CondEq, word); + q.Where("searchfuzzy", CondEq, MakeSuffixWord()); QueryResults qres; auto err = db_->Select(q, qres); @@ -779,9 +766,7 @@ void FullText::Fuzzy2SuffixMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - std::string words = MakeSuffixWord() + " " + MakeSuffixWord(); - q.Where("searchfuzzy", CondEq, words); + q.Where("searchfuzzy", CondEq, MakeSuffixWord().append(" ").append(MakeSuffixWord())); QueryResults qres; auto err = db_->Select(q, qres); @@ -798,9 +783,7 @@ void FullText::Fast1TypoWordMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - - std::string word = MakeTypoWord(); - q.Where("searchfast", CondEq, word); + q.Where("searchfast", CondEq, MakeTypoWord()); QueryResults qres; auto err = db_->Select(q, qres); @@ -817,9 +800,7 @@ void FullText::Fast2TypoWordMatch(benchmark::State& state) { for (auto _ : state) { // NOLINT(*deadcode.DeadStores) TIMEMEASURE(); Query q(nsdef_.name); - - std::string words = MakeTypoWord() + " " + MakeTypoWord(); - q.Where("searchfast", CondEq, words); + q.Where("searchfast", CondEq, MakeTypoWord().append(" ").append(MakeTypoWord())); QueryResults qres; auto err = db_->Select(q, qres); @@ -834,9 +815,7 @@ void FullText::Fuzzy1TypoWordMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - std::string word = MakeTypoWord(); - q.Where("searchfuzzy", CondEq, word); + q.Where("searchfuzzy", CondEq, MakeTypoWord()); QueryResults qres; auto err = db_->Select(q, qres); @@ -851,9 +830,7 @@ void FullText::Fuzzy2TypoWordMatch(benchmark::State& state) { size_t cnt = 0; for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - - std::string words = MakeTypoWord() + " " + MakeTypoWord(); - q.Where("searchfuzzy", CondEq, words); + q.Where("searchfuzzy", CondEq, MakeTypoWord().append(" ").append(MakeTypoWord())); QueryResults qres; auto err = db_->Select(q, qres); diff --git a/cpp_src/gtests/tests/fixtures/ft_api.h b/cpp_src/gtests/tests/fixtures/ft_api.h index 542630dec..4664eaea2 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.h +++ b/cpp_src/gtests/tests/fixtures/ft_api.h @@ -111,7 +111,7 @@ class FTApi : public ::testing::TestWithParam fields; - reindexer::fast_hash_set stopWords; + reindexer::fast_hash_set stopWords; std::string extraWordSymbols = "-/+"; }; int counter_ = 0; diff --git a/cpp_src/gtests/tests/fixtures/queries_verifier.h b/cpp_src/gtests/tests/fixtures/queries_verifier.h index dc68e83df..8705024ac 100644 --- a/cpp_src/gtests/tests/fixtures/queries_verifier.h +++ b/cpp_src/gtests/tests/fixtures/queries_verifier.h @@ -1,7 +1,16 @@ #pragma once #include + +#if defined(__GNUC__) && (__GNUC__ == 12) && defined(REINDEX_WITH_ASAN) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#include +#pragma GCC diagnostic pop +#else // REINDEX_WITH_ASAN #include +#endif // REINDEX_WITH_ASAN + #include #include "core/nsselecter/joinedselectormock.h" #include "core/query/query.h" diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.cc b/cpp_src/gtests/tests/fixtures/servercontrol.cc index 1fa9e0cb6..7175ff13e 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.cc +++ b/cpp_src/gtests/tests/fixtures/servercontrol.cc @@ -283,9 +283,6 @@ void ServerControl::Interface::Init() { } assertrx(res == EXIT_SUCCESS); })); - while (!srv.IsRunning() || !srv.IsReady()) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } } // init client diff --git a/cpp_src/gtests/tests/fixtures/sharding_api.h b/cpp_src/gtests/tests/fixtures/sharding_api.h index bb8f369bb..0c2447d09 100644 --- a/cpp_src/gtests/tests/fixtures/sharding_api.h +++ b/cpp_src/gtests/tests/fixtures/sharding_api.h @@ -36,13 +36,13 @@ struct InitShardingConfig { int nodeIdInThread = -1; // Allows to run one of the nodes in thread, instead fo process uint8_t strlen = 0; // Strings len in items, which will be created during Fill() // if it is necessary to explicitly set specific cluster nodes in shards - std::optional>> shardsMap = std::nullopt; + std::shared_ptr>> shardsMap; }; class ShardingApi : public ReindexerApi { public: using ShardingConfig = reindexer::cluster::ShardingConfig; - static const std::string configTemplate; + static const std::string_view kConfigTemplate; enum class ApplyType : bool { Shared, Local }; void Init(InitShardingConfig c = InitShardingConfig()) { @@ -62,7 +62,7 @@ class ShardingApi : public ReindexerApi { config_.namespaces.resize(namespaces.size()); for (size_t i = 0; i < namespaces.size(); ++i) { config_.namespaces[i].ns = namespaces[i].name; - config_.namespaces[i].index = namespaces[i].indexName; // kFieldLocation; + config_.namespaces[i].index = namespaces[i].indexName; config_.namespaces[i].defaultShard = 0; } config_.shards.clear(); @@ -88,13 +88,18 @@ class ShardingApi : public ReindexerApi { for (size_t shard = 0; shard < kShards; ++shard) { YAML::Node clusterConf; - clusterConf["app_name"] = "rx_node"; clusterConf["namespaces"] = YAML::Node(YAML::NodeType::Sequence); clusterConf["sync_threads"] = syncThreadsCount; clusterConf["enable_compression"] = true; clusterConf["online_updates_timeout_sec"] = 20; clusterConf["sync_timeout_sec"] = 60; clusterConf["syncs_per_thread"] = maxSyncCount; + clusterConf["leader_sync_threads"] = 2; +#ifdef REINDEX_WITH_TSAN + clusterConf["batching_routines_count"] = 20; +#else + clusterConf["batching_routines_count"] = 50; +#endif clusterConf["retry_sync_interval_msec"] = resyncTimeout.count(); clusterConf["nodes"] = YAML::Node(YAML::NodeType::Sequence); const size_t startId = shard * kNodesInCluster; @@ -115,6 +120,7 @@ class ShardingApi : public ReindexerApi { YAML::Node replConf; replConf["cluster_id"] = shard; replConf["server_id"] = idx; + clusterConf["app_name"] = fmt::sprintf("rx_node_%d", idx); std::string pathToDb = fs::JoinPath(GetDefaults().baseTestsetDbPath, "shard" + std::to_string(shard) + "/" + std::to_string(idx)); @@ -128,7 +134,7 @@ class ShardingApi : public ReindexerApi { } } - std::shared_ptr rx = getNode(0)->api.reindexer; + auto& rx = *getNode(0)->api.reindexer; for (auto& ns : namespaces) { if (ns.withData) { @@ -150,7 +156,7 @@ class ShardingApi : public ReindexerApi { } else { nsDef.AddIndex(kFieldId, "hash", "int", IndexOpts().PK()); } - Error err = rx->AddNamespace(nsDef); + Error err = rx.AddNamespace(nsDef); ASSERT_TRUE(err.ok()) << err.what() << "; ns: " << ns.name; } } @@ -159,6 +165,10 @@ class ShardingApi : public ReindexerApi { if (ns.withData) { size_t pos = 0; for (size_t i = 0; i < kShards; ++i) { + if (kNodesInCluster > 1) { + auto err = svc_[i][0].Get()->api.reindexer->OpenNamespace(ns.name); + ASSERT_TRUE(err.ok()) << err.what(); + } Fill(ns.name, i, pos, c.rowsInTableOnShard, c.insertedItemsById); pos += c.rowsInTableOnShard; } @@ -281,9 +291,9 @@ class ShardingApi : public ReindexerApi { return StopSC(svc_[i][j]); } - client::Item CreateItem(std::string_view nsName, std::shared_ptr rx, std::string_view key, int index, - WrSerializer& wrser, uint8_t strlen = 0) { - client::Item item = rx->NewItem(nsName); + client::Item CreateItem(std::string_view nsName, client::Reindexer& rx, std::string_view key, int index, WrSerializer& wrser, + uint8_t strlen = 0) { + client::Item item = rx.NewItem(nsName); if (!item.Status().ok()) return item; wrser.Reset(); reindexer::JsonBuilder jsonBuilder(wrser, ObjType::TypeObject); @@ -306,7 +316,7 @@ class ShardingApi : public ReindexerApi { return item; } - Error CreateAndUpsertItem(std::string_view nsName, std::shared_ptr rx, std::string_view key, int index, + Error CreateAndUpsertItem(std::string_view nsName, client::Reindexer& rx, std::string_view key, int index, fast_hash_map* insertedItemsById = nullptr, uint8_t strlen = 0) { WrSerializer wrser; client::Item item = CreateItem(nsName, rx, key, index, wrser, strlen); @@ -323,23 +333,21 @@ class ShardingApi : public ReindexerApi { if (!err.ok()) { return err; } - return rx->Upsert(nsName, item); + return rx.Upsert(nsName, item); } void Fill(std::string_view nsName, size_t shard, const size_t from, const size_t count, fast_hash_map* insertedItemsById = nullptr, uint8_t strlen = 0) { assert(shard < svc_.size()); - std::shared_ptr rx = svc_[shard][0].Get()->api.reindexer; - Error err = rx->OpenNamespace(nsName); - ASSERT_TRUE(err.ok()) << err.what(); + auto& rx = *svc_[shard][0].Get()->api.reindexer; for (size_t index = from; index < from + count; ++index) { - err = CreateAndUpsertItem(nsName, rx, std::string("key" + std::to_string((index % kShards) + 1)), index, insertedItemsById, - strlen); + auto err = CreateAndUpsertItem(nsName, rx, std::string("key" + std::to_string((index % kShards) + 1)), index, insertedItemsById, + strlen); ASSERT_TRUE(err.ok()) << err.what(); } } - void Fill(std::string_view nsName, std::shared_ptr rx, std::string_view key, const size_t from, const size_t count) { + void Fill(std::string_view nsName, client::Reindexer& rx, std::string_view key, const size_t from, const size_t count) { for (size_t index = from; index < from + count; ++index) { Error err = CreateAndUpsertItem(nsName, rx, key, index); ASSERT_TRUE(err.ok()) << err.what(); @@ -351,7 +359,7 @@ class ShardingApi : public ReindexerApi { if (!srv) { return Error(errNotValid, "Server is not running"); } - auto rx = srv->api.reindexer; + auto& rx = *srv->api.reindexer; return CreateAndUpsertItem(nsName, rx, std::string("key" + std::to_string((index % kShards) + 1)), index); } @@ -444,7 +452,7 @@ class ShardingApi : public ReindexerApi { ShardingConfig makeShardingConfigByDistrib(std::string_view nsName, const std::map>& shardDataDistrib, int shards = 3, int nodes = 3) const; - Error applyNewShardingConfig(const std::shared_ptr& rx, const ShardingConfig& config, ApplyType type, + Error applyNewShardingConfig(client::Reindexer& rx, const ShardingConfig& config, ApplyType type, std::optional sourceId = std::optional()) const; void checkConfig(const ServerControl::Interface::Ptr& server, const cluster::ShardingConfig& config); diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index 393d01895..84640f246 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -1328,6 +1328,87 @@ TEST_P(FTGenericApi, ExplainWithFtPreselect) { } } +TEST_P(FTGenericApi, StopWordsWithMorphemes) { + reindexer::FtFastConfig cfg = GetDefaultConfig(); + + Init(cfg); + Add("Шахматы из слоновой кости"sv); + Add("Мат в эфире "sv); + Add("Известняк"sv); + Add("Известия"sv); + Add("Изверг"sv); + + Add("Подобрал подосиновики, положил в лубочек"sv); + Add("Подопытный кролик"sv); + Add("Шла Саша по шоссе"sv); + + Add("Зайка серенький под елочкой скакал"sv); + Add("За Альянс! (с)"sv); + Add("Заноза в пальце"sv); + + Add("На западном фронте без перемен"sv); + Add("Наливные яблочки"sv); + Add("Нарком СССР"sv); + + CheckResults("*из*", {{"!Известняк!", ""}, {"!Известия!", ""}, {"!Изверг!", ""}}, false); + CheckResults("из", {}, false); + + CheckResults("*под*", {{"!Подобрал подосиновики!, положил в лубочек", ""}, {"!Подопытный! кролик", ""}}, false); + CheckResults("под", {}, false); + + CheckResults( + "*за*", {{"!Зайка! серенький под елочкой скакал", ""}, {"!Заноза! в пальце", ""}, {"На !западном! фронте без перемен", ""}}, false); + CheckResults("за", {}, false); + + CheckResults("*на*", + { + {"!Наливные! яблочки", ""}, + {"!Нарком! СССР", ""}, + }, + false); + CheckResults("на", {}, false); + + cfg.stopWords.clear(); + + cfg.stopWords.insert({"на"}); + cfg.stopWords.insert({"мат", reindexer::StopWord::Type::Morpheme}); + + SetFTConfig(cfg); + + CheckResults("*из*", {{"Шахматы !из! слоновой кости", ""}, {"!Известняк!", ""}, {"!Известия!", ""}, {"!Изверг!", ""}}, false); + CheckResults("из", {{"Шахматы !из! слоновой кости", ""}}, false); + + CheckResults( + "*под*", + {{"!Подобрал подосиновики!, положил в лубочек", ""}, {"!Подопытный! кролик", ""}, {"Зайка серенький !под! елочкой скакал", ""}}, + false); + CheckResults("под", {{"Зайка серенький !под! елочкой скакал", ""}}, false); + + CheckResults("*по*", + {{"Шла Саша !по! шоссе", ""}, + {"!Подобрал подосиновики, положил! в лубочек", ""}, + {"!Подопытный! кролик", ""}, + {"Зайка серенький !под! елочкой скакал", ""}}, + false); + CheckResults("по~", {{"Шла Саша !по! шоссе", ""}, {"Зайка серенький !под! елочкой скакал", ""}}, false); + CheckResults("по", {{"Шла Саша !по! шоссе", ""}}, false); + + CheckResults("*мат*", {{"!Шахматы! из слоновой кости", ""}}, false); + CheckResults("мат", {}, false); + + CheckResults("*за*", + {{"!Зайка! серенький под елочкой скакал", ""}, + {"!Заноза! в пальце", ""}, + {"!За! Альянс! (с)", ""}, + {"На !западном! фронте без перемен", ""}}, + false); + CheckResults("за", {{"!За! Альянс! (с)", ""}}, false); + + CheckResults("*на*", {}, false); + CheckResults("на~", {}, false); + CheckResults("на", {}, false); +} + INSTANTIATE_TEST_SUITE_P(, FTGenericApi, ::testing::Values(reindexer::FtFastConfig::Optimization::Memory, reindexer::FtFastConfig::Optimization::CPU), [](const auto& info) { diff --git a/cpp_src/gtests/tests/unit/queries_test.cc b/cpp_src/gtests/tests/unit/queries_test.cc index ca05b4d10..cc644d708 100644 --- a/cpp_src/gtests/tests/unit/queries_test.cc +++ b/cpp_src/gtests/tests/unit/queries_test.cc @@ -100,14 +100,12 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { ASSERT_TRUE(false); } } -#endif TEST_F(QueriesApi, QueriesConditions) { FillConditionsNs(); CheckConditions(); } -#if !defined(REINDEX_WITH_TSAN) TEST_F(QueriesApi, UuidQueries) { FillUUIDNs(); // hack to obtain not index not string uuid fields @@ -117,7 +115,7 @@ TEST_F(QueriesApi, UuidQueries) { ASSERT_TRUE(err.ok()) << err.what();*/ CheckUUIDQueries(); } -#endif +#endif // !defined(REINDEX_WITH_TSAN) TEST_F(QueriesApi, IndexCacheInvalidationTest) { std::vector> data{{0, 10}, {1, 9}, {2, 8}, {3, 7}, {4, 6}, {5, 5}, diff --git a/cpp_src/gtests/tests/unit/sharding_base_test.cc b/cpp_src/gtests/tests/unit/sharding_base_test.cc index eae626a64..8d3c895bc 100644 --- a/cpp_src/gtests/tests/unit/sharding_base_test.cc +++ b/cpp_src/gtests/tests/unit/sharding_base_test.cc @@ -839,12 +839,13 @@ ShardingApi::ShardingConfig ShardingApi::makeShardingConfigByDistrib(std::string cfg.proxyConnCount = 3; cfg.proxyConnThreads = 2; cfg.proxyConnConcurrency = 4; + cfg.configRollbackTimeout = std::chrono::seconds(10); return cfg; } -Error ShardingApi::applyNewShardingConfig(const std::shared_ptr &rx, const ShardingConfig &config, ApplyType type, +Error ShardingApi::applyNewShardingConfig(client::Reindexer &rx, const ShardingConfig &config, ApplyType type, std::optional sourceId) const { - reindexer::client::Item item = rx->NewItem("#config"); + reindexer::client::Item item = rx.NewItem("#config"); if (!item.Status().ok()) return item.Status(); { @@ -869,7 +870,7 @@ Error ShardingApi::applyNewShardingConfig(const std::shared_ptrUpsert("#config", item); + auto err = rx.Upsert("#config", item); if (!item.Status().ok()) return item.Status(); return err; @@ -883,8 +884,8 @@ TEST_F(ShardingApi, RuntimeShardingConfigTest) { cfg.nodesInCluster = 1; // Only one node in the cluster is needed for the test Init(std::move(cfg)); - auto rx = svc_[0][0].Get()->api.reindexer; - auto err = rx->DropNamespace(default_namespace); + auto &rx = *svc_[0][0].Get()->api.reindexer; + auto err = rx.DropNamespace(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); auto newConfig = makeShardingConfigByDistrib(kNsName, shardDataDistrib, 3, 1); @@ -923,16 +924,17 @@ void ShardingApi::MultyThreadApplyConfigTest(ShardingApi::ApplyType type) { } auto fillDataLocalNss = [this, &kNonShardedNs](int shardId) { - const auto &rx = svc_[shardId][0].Get()->api.reindexer; + auto &rx = *svc_[shardId][0].Get()->api.reindexer; NamespaceDef nsDef{kNonShardedNs}; nsDef.AddIndex(kFieldId, "hash", "int", IndexOpts().PK()); - Error err = rx->AddNamespace(nsDef); + Error err = rx.AddNamespace(nsDef); ASSERT_TRUE(err.ok()) << err.what(); + WrSerializer wrser; for (int i = 0; i < 10; ++i) { - WrSerializer wrser; - auto item = rx->NewItem(kNonShardedNs); + wrser.Reset(); + auto item = rx.NewItem(kNonShardedNs); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); reindexer::JsonBuilder jsonBuilder(wrser, ObjType::TypeObject); @@ -941,7 +943,7 @@ void ShardingApi::MultyThreadApplyConfigTest(ShardingApi::ApplyType type) { err = item.FromJSON(wrser.Slice()); ASSERT_TRUE(err.ok()) << err.what(); - err = rx->Insert(kNonShardedNs, item); + err = rx.Insert(kNonShardedNs, item); ASSERT_TRUE(err.ok()) << err.what(); } }; @@ -979,9 +981,9 @@ void ShardingApi::MultyThreadApplyConfigTest(ShardingApi::ApplyType type) { std::launch::async, [this, &config, type, kLocalSourceId](int shardId) { if (type == ApplyType::Local) { - return applyNewShardingConfig(svc_[shardId][0].Get()->api.reindexer, config, type, kLocalSourceId); + return applyNewShardingConfig(*svc_[shardId][0].Get()->api.reindexer, config, type, kLocalSourceId); } - return applyNewShardingConfig(svc_[shardId][0].Get()->api.reindexer, config, type); + return applyNewShardingConfig(*svc_[shardId][0].Get()->api.reindexer, config, type); }, i)); } @@ -1002,7 +1004,7 @@ void ShardingApi::MultyThreadApplyConfigTest(ShardingApi::ApplyType type) { if (type == ApplyType::Shared) { do { - auto err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + auto err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); if (err.ok()) break; ASSERT_THAT(err.what(), testing::MatchesRegex(".*(Config candidate is busy already|Attempt to apply|Attempt to reset).*")) @@ -1036,11 +1038,11 @@ TEST_F(ShardingApi, RuntimeShardingConfigLocallyResetTest) { cfg.shards = 4; Init(std::move(cfg)); - const auto &rx = svc_[0][0].Get()->api.reindexer; - auto shCfg = getShardingConfigFrom(*rx); + auto &rx = *svc_[0][0].Get()->api.reindexer; + auto shCfg = getShardingConfigFrom(rx); assertrx(shCfg.has_value()); - reindexer::client::Item item = rx->NewItem("#config"); + reindexer::client::Item item = rx.NewItem("#config"); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); { @@ -1056,10 +1058,10 @@ TEST_F(ShardingApi, RuntimeShardingConfigLocallyResetTest) { ASSERT_TRUE(err.ok()) << err.what(); } - auto err = rx->Upsert("#config", item); + auto err = rx.Upsert("#config", item); ASSERT_TRUE(err.ok()) << err.what(); - shCfg = getShardingConfigFrom(*rx); + shCfg = getShardingConfigFrom(rx); assertrx(!shCfg.has_value()); } @@ -1165,16 +1167,16 @@ TEST_F(ShardingApi, RuntimeUpdateShardingCfgWithClusterTest) { const int parallelSelectsCount = shardsCount; auto fillDataLocalNss = [this, &kNonShardedNs](int shardId) { - const auto &rx = svc_[shardId][0].Get()->api.reindexer; + auto &rx = *svc_[shardId][0].Get()->api.reindexer; NamespaceDef nsDef{kNonShardedNs}; nsDef.AddIndex(kFieldId, "hash", "int", IndexOpts().PK()); - Error err = rx->AddNamespace(nsDef); + Error err = rx.AddNamespace(nsDef); ASSERT_TRUE(err.ok()) << err.what(); for (int i = 0; i < 10; ++i) { WrSerializer wrser; - auto item = rx->NewItem(kNonShardedNs); + auto item = rx.NewItem(kNonShardedNs); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); reindexer::JsonBuilder jsonBuilder(wrser, ObjType::TypeObject); @@ -1183,7 +1185,7 @@ TEST_F(ShardingApi, RuntimeUpdateShardingCfgWithClusterTest) { err = item.FromJSON(wrser.Slice()); ASSERT_TRUE(err.ok()) << err.what(); - err = rx->Insert(kNonShardedNs, item); + err = rx.Insert(kNonShardedNs, item); ASSERT_TRUE(err.ok()) << err.what(); } }; @@ -1230,7 +1232,7 @@ TEST_F(ShardingApi, RuntimeUpdateShardingCfgWithClusterTest) { shardId); } - auto err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + auto err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); stopChangeLeader.store(true); for (auto &chLeaderThread : parallelChangeLeaders) chLeaderThread.join(); @@ -1242,14 +1244,13 @@ TEST_F(ShardingApi, RuntimeUpdateShardingCfgWithClusterTest) { // Applying without changing leader must be done correctly do { - err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); if (err.ok()) break; ASSERT_THAT(err.what(), testing::MatchesRegex(".*Config candidate is busy already.*")) << err.what(); std::this_thread::sleep_for(std::chrono::milliseconds(20)); } while (true); - - err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); ASSERT_TRUE(err.ok()) << err.what(); } @@ -1364,7 +1365,7 @@ TEST_F(ShardingApi, RuntimeUpdateShardingWithDisabledNodesTest) { shardId); } - auto err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + auto err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); stopDisableNodes.store(true); for (auto &th : parallelDisablingNodes) th.join(); @@ -1372,13 +1373,13 @@ TEST_F(ShardingApi, RuntimeUpdateShardingWithDisabledNodesTest) { if (!err.ok()) { // Applying without changing leader must be done correctly do { - err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); if (err.ok()) break; ASSERT_THAT(err.what(), testing::MatchesRegex(".*Config candidate is busy already.*")) << err.what(); } while (true); - err = applyNewShardingConfig(svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); + err = applyNewShardingConfig(*svc_[0][0].Get()->api.reindexer, config, ApplyType::Shared); ASSERT_TRUE(err.ok()) << err.what(); } @@ -1435,7 +1436,7 @@ TEST_F(ShardingApi, RuntimeUpdateShardingWithActualConfigTest) { auto &localCfg = useIncorrectCfg ? configIncorrect : config; localCfg.thisShardId = shardId; localCfg.sourceId = useIncorrectCfg ? kSourceId : kSourceId + 1; - auto err = applyNewShardingConfig(shard[nodeIds[i]].Get()->api.reindexer, localCfg, ApplyType::Local, localCfg.sourceId); + auto err = applyNewShardingConfig(*shard[nodeIds[i]].Get()->api.reindexer, localCfg, ApplyType::Local, localCfg.sourceId); ASSERT_TRUE(err.ok()) << err.what(); checkConfig(shard[nodeIds[i]].Get(), localCfg); } @@ -1511,12 +1512,12 @@ TEST_F(ShardingApi, CheckUpdCfgNsAfterApplySharingCfg) { cfg.needFillDefaultNs = false; Init(std::move(cfg)); - auto rx = svc_[0][0].Get()->api.reindexer; + auto &rx = *svc_[0][0].Get()->api.reindexer; auto newConfig = makeShardingConfigByDistrib(kNsName, shardDataDistrib, 3, 1); auto err = applyNewShardingConfig(rx, newConfig, ApplyType::Shared); ASSERT_TRUE(err.ok()) << err.what(); - auto readConfig = getShardingConfigFrom(*rx); + auto readConfig = getShardingConfigFrom(rx); assertrx(readConfig.has_value()); readConfig->thisShardId = -1; // it's necessary for the correct comparison of both configs @@ -1551,7 +1552,7 @@ TEST_F(ShardingApi, PartialClusterNodesSetInShardingCfgTest) { shardsMap[shard].push_back(fmt::format("cproto://127.0.0.1:{}/shard{}", GetDefaults().defaultRpcPort + nodeId, nodeId)); } // replace shards part in sharding config due to in one shard was only one of 7 nodes of sync cluster - cfg.shardsMap = std::move(shardsMap); + cfg.shardsMap = std::make_shared>>(std::move(shardsMap)); cfg.nodesInCluster = knodesInCluster; Init(std::move(cfg)); @@ -1940,7 +1941,7 @@ TEST_F(ShardingApi, EnumLocalNamespaces) { } } -const std::string ShardingApi::configTemplate = R"(version: 1 +const std::string_view ShardingApi::kConfigTemplate = R"(version: 1 namespaces: - namespace: namespace1 default_shard: 0 @@ -1971,6 +1972,7 @@ const std::string ShardingApi::configTemplate = R"(version: 1 this_shard_id: 0 reconnect_timeout_msec: 3000 shards_awaiting_timeout_sec: 30 +config_rollback_timeout_sec: 30 proxy_conn_count: 8 proxy_conn_concurrency: 8 proxy_conn_threads: 4 @@ -1997,7 +1999,7 @@ TEST_F(ShardingApi, ConfigYaml) { }; auto substRangesInTemplate = [](const std::vector &values) { - auto res = configTemplate; + std::string res(kConfigTemplate); for (size_t i = 0; i < 3; ++i) { auto tmplt = fmt::sprintf("${%d}", i); res.replace(res.find(tmplt), tmplt.size(), values[i]); @@ -2167,6 +2169,7 @@ TEST_F(ShardingApi, ConfigYaml) { this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Default shard id is not specified for namespace 'best_namespace'"}}, @@ -2190,6 +2193,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Dsns for shard id 1 are specified twice"}}, @@ -2210,6 +2214,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Scheme of sharding dsn must be cproto: 127.0.0.1:19001/shard1"}}, @@ -2233,6 +2238,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, @@ -2257,6 +2263,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Shard id 3 is not specified in the config but it is used in namespace keys"}}, @@ -2280,6 +2287,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Shard id should not be less than zero"}}, @@ -2313,6 +2321,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 30 proxy_conn_count: 15 )"s, Error{errParams, "Namespace 'best_namespacE' already specified in the config."} @@ -2364,6 +2373,7 @@ proxy_conn_count: 15 this_shard_id: 0 reconnect_timeout_msec: 5000 shards_awaiting_timeout_sec: 25 +config_rollback_timeout_sec: 17 proxy_conn_count: 15 proxy_conn_concurrency: 10 proxy_conn_threads: 5 @@ -2377,6 +2387,7 @@ proxy_conn_threads: 5 0, std::chrono::milliseconds(5000), std::chrono::seconds(25), + std::chrono::seconds(17), 15, 10, 5}}, @@ -2426,6 +2437,7 @@ proxy_conn_threads: 5 this_shard_id: 0 reconnect_timeout_msec: 3000 shards_awaiting_timeout_sec: 30 +config_rollback_timeout_sec: 30 proxy_conn_count: 8 proxy_conn_concurrency: 8 proxy_conn_threads: 4 @@ -2501,71 +2513,77 @@ TEST_F(ShardingApi, ConfigJson) { std::string json; Cfg expected; std::optional json4compare = std::nullopt; - } testCases[]{ - {R"({"version":1,"namespaces":[{"namespace":"best_namespace","default_shard":0,"index":"location","keys":[{"shard_id":1,"values":["south","west"]},{"shard_id":2,"values":["north"]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19001/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.2:19002/shard2"]}],"this_shard_id":0,"reconnect_timeout_msec":5000,"shards_awaiting_timeout_sec":20,"proxy_conn_count":4,"proxy_conn_concurrency":8,"proxy_conn_threads":4})"s, - Cfg{{{"best_namespace", - "location", - {Cfg::Key{1, ByValue, {sharding::Segment(Variant("south")), sharding::Segment(Variant("west"))}}, - {2, ByValue, {sharding::Segment(Variant("north"))}}}, - 0}}, - {{0, {"cproto://127.0.0.1:19000/shard0"}}, {1, {"cproto://127.0.0.1:19001/shard1"}}, {2, {"cproto://127.0.0.2:19002/shard2"}}}, - 0, - std::chrono::milliseconds(5000), - std::chrono::seconds(20), - 4}}, - {R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[0,10,20]},{"shard_id":2,"values":[1,5,7,9]},{"shard_id":3,"values":[100]}]},{"namespace":"namespace2","default_shard":3,"index":"city","keys":[{"shard_id":1,"values":["Moscow"]},{"shard_id":2,"values":["London"]},{"shard_id":3,"values":["Paris"]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0","cproto://127.0.0.1:19001/shard0","cproto://127.0.0.1:19002/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1","cproto://127.0.0.1:19011/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.2:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.2:19030/shard3","cproto://127.0.0.2:19031/shard3","cproto://127.0.0.2:19032/shard3","cproto://127.0.0.2:19033/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})"s, - Cfg{{{"namespace1", - "count", - {Cfg::Key{1, ByValue, {sharding::Segment(Variant(0)), sharding::Segment(Variant(10)), sharding::Segment(Variant(20))}}, - {2, - ByValue, - {sharding::Segment(Variant(1)), sharding::Segment(Variant(5)), sharding::Segment(Variant(7)), - sharding::Segment(Variant(9))}}, - {3, ByValue, {sharding::Segment(Variant(100))}}}, - 0}, - {"namespace2", - "city", - {Cfg::Key{1, ByValue, {sharding::Segment(Variant("Moscow"))}}, - {2, ByValue, {sharding::Segment(Variant("London"))}}, - {3, ByValue, {sharding::Segment(Variant("Paris"))}}}, - 3}}, - {{0, {"cproto://127.0.0.1:19000/shard0", "cproto://127.0.0.1:19001/shard0", "cproto://127.0.0.1:19002/shard0"}}, - {1, {"cproto://127.0.0.1:19010/shard1", "cproto://127.0.0.1:19011/shard1"}}, - {2, {"cproto://127.0.0.2:19020/shard2"}}, - {3, - {"cproto://127.0.0.2:19030/shard3", "cproto://127.0.0.2:19031/shard3", "cproto://127.0.0.2:19032/shard3", - "cproto://127.0.0.2:19033/shard3"}}}, - 0, - std::chrono::milliseconds(4000), - std::chrono::seconds(30), - 3, - 5, - 2}}, - {R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[1,2,3,4,[3,5],[10,15]]},{"shard_id":2,"values":[16,[36,40],[40,65],[100,150]]},{"shard_id":3,"values":[93,25,33,[24,35],[88,95]]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.1:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.1:19030/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})", - Cfg{{{"namespace1", - "count", - {Cfg::Key{1, - ByRange, - {sharding::Segment(Variant(1)), sharding::Segment(Variant(2)), sharding::Segment{Variant(3), Variant(5)}, - sharding::Segment{Variant(10), Variant(15)}}}, - {2, - ByRange, - {sharding::Segment(Variant(16)), sharding::Segment{Variant(36), Variant(65)}, - sharding::Segment{Variant(100), Variant(150)}}}, - {3, ByRange, {sharding::Segment{Variant(24), Variant(35)}, sharding::Segment{Variant(88), Variant(95)}}}}, - 0}}, - - {{0, {"cproto://127.0.0.1:19000/shard0"}}, - {1, {"cproto://127.0.0.1:19010/shard1"}}, - {2, {"cproto://127.0.0.1:19020/shard2"}}, - {3, {"cproto://127.0.0.1:19030/shard3"}}}, - 0, - std::chrono::milliseconds(4000), - std::chrono::seconds(30), - 3, - 5, - 2}, - R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[1,2,[3,5],[10,15]]},{"shard_id":2,"values":[16,[36,65],[100,150]]},{"shard_id":3,"values":[[24,35],[88,95]]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.1:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.1:19030/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})"}}; + } testCases[]{{R"({"version":1,"namespaces":[{"namespace":"best_namespace","default_shard":0,"index":"location","keys":[{"shard_id":1,"values":["south","west"]},{"shard_id":2,"values":["north"]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19001/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.2:19002/shard2"]}],"this_shard_id":0,"reconnect_timeout_msec":5000,"shards_awaiting_timeout_sec":20,"config_rollback_timeout_sec":33,"proxy_conn_count":4,"proxy_conn_concurrency":8,"proxy_conn_threads":4})"s, + Cfg{{{"best_namespace", + "location", + {Cfg::Key{1, ByValue, {sharding::Segment(Variant("south")), sharding::Segment(Variant("west"))}}, + {2, ByValue, {sharding::Segment(Variant("north"))}}}, + 0}}, + {{0, {"cproto://127.0.0.1:19000/shard0"}}, + {1, {"cproto://127.0.0.1:19001/shard1"}}, + {2, {"cproto://127.0.0.2:19002/shard2"}}}, + 0, + std::chrono::milliseconds(5000), + std::chrono::seconds(20), + std::chrono::seconds(33), + 4}}, + {R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[0,10,20]},{"shard_id":2,"values":[1,5,7,9]},{"shard_id":3,"values":[100]}]},{"namespace":"namespace2","default_shard":3,"index":"city","keys":[{"shard_id":1,"values":["Moscow"]},{"shard_id":2,"values":["London"]},{"shard_id":3,"values":["Paris"]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0","cproto://127.0.0.1:19001/shard0","cproto://127.0.0.1:19002/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1","cproto://127.0.0.1:19011/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.2:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.2:19030/shard3","cproto://127.0.0.2:19031/shard3","cproto://127.0.0.2:19032/shard3","cproto://127.0.0.2:19033/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"config_rollback_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})"s, + Cfg{{{"namespace1", + "count", + {Cfg::Key{ + 1, ByValue, {sharding::Segment(Variant(0)), sharding::Segment(Variant(10)), sharding::Segment(Variant(20))}}, + {2, + ByValue, + {sharding::Segment(Variant(1)), sharding::Segment(Variant(5)), sharding::Segment(Variant(7)), + sharding::Segment(Variant(9))}}, + {3, ByValue, {sharding::Segment(Variant(100))}}}, + 0}, + {"namespace2", + "city", + {Cfg::Key{1, ByValue, {sharding::Segment(Variant("Moscow"))}}, + {2, ByValue, {sharding::Segment(Variant("London"))}}, + {3, ByValue, {sharding::Segment(Variant("Paris"))}}}, + 3}}, + {{0, {"cproto://127.0.0.1:19000/shard0", "cproto://127.0.0.1:19001/shard0", "cproto://127.0.0.1:19002/shard0"}}, + {1, {"cproto://127.0.0.1:19010/shard1", "cproto://127.0.0.1:19011/shard1"}}, + {2, {"cproto://127.0.0.2:19020/shard2"}}, + {3, + {"cproto://127.0.0.2:19030/shard3", "cproto://127.0.0.2:19031/shard3", "cproto://127.0.0.2:19032/shard3", + "cproto://127.0.0.2:19033/shard3"}}}, + 0, + std::chrono::milliseconds(4000), + std::chrono::seconds(30), + std::chrono::seconds(30), + 3, + 5, + 2}}, + {R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[1,2,3,4,[3,5],[10,15]]},{"shard_id":2,"values":[16,[36,40],[40,65],[100,150]]},{"shard_id":3,"values":[93,25,33,[24,35],[88,95]]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.1:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.1:19030/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"config_rollback_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})", + Cfg{{{"namespace1", + "count", + {Cfg:: + Key{1, + ByRange, + {sharding::Segment(Variant(1)), sharding::Segment(Variant(2)), sharding::Segment{Variant(3), Variant(5)}, + sharding::Segment{Variant(10), Variant(15)}}}, + {2, + ByRange, + {sharding::Segment(Variant(16)), sharding::Segment{Variant(36), Variant(65)}, + sharding::Segment{Variant(100), Variant(150)}}}, + {3, ByRange, {sharding::Segment{Variant(24), Variant(35)}, sharding::Segment{Variant(88), Variant(95)}}}}, + 0}}, + + {{0, {"cproto://127.0.0.1:19000/shard0"}}, + {1, {"cproto://127.0.0.1:19010/shard1"}}, + {2, {"cproto://127.0.0.1:19020/shard2"}}, + {3, {"cproto://127.0.0.1:19030/shard3"}}}, + 0, + std::chrono::milliseconds(4000), + std::chrono::seconds(30), + std::chrono::seconds(30), + 3, + 5, + 2}, + R"({"version":1,"namespaces":[{"namespace":"namespace1","default_shard":0,"index":"count","keys":[{"shard_id":1,"values":[1,2,[3,5],[10,15]]},{"shard_id":2,"values":[16,[36,65],[100,150]]},{"shard_id":3,"values":[[24,35],[88,95]]}]}],"shards":[{"shard_id":0,"dsns":["cproto://127.0.0.1:19000/shard0"]},{"shard_id":1,"dsns":["cproto://127.0.0.1:19010/shard1"]},{"shard_id":2,"dsns":["cproto://127.0.0.1:19020/shard2"]},{"shard_id":3,"dsns":["cproto://127.0.0.1:19030/shard3"]}],"this_shard_id":0,"reconnect_timeout_msec":4000,"shards_awaiting_timeout_sec":30,"config_rollback_timeout_sec":30,"proxy_conn_count":3,"proxy_conn_concurrency":5,"proxy_conn_threads":2})"}}; for (const auto &[json, cfg, json4cmp] : testCases) { const auto generatedJson = cfg.GetJSON(); @@ -2779,24 +2797,24 @@ TEST_F(ShardingApi, RestrictionOnRequest) { ASSERT_EQ(err.code(), errLogic); } } -static void CheckTotalCount(bool cached, std::shared_ptr &rx, const std::string &ns, size_t expected) { +static void CheckTotalCount(bool cached, client::Reindexer &rx, const std::string &ns, size_t expected) { client::QueryResults qr; Error err; if (cached) { - err = rx->Select(Query(ns).CachedTotal().Limit(0), qr); + err = rx.Select(Query(ns).CachedTotal().Limit(0), qr); } else { - err = rx->Select(Query(ns).ReqTotal().Limit(0), qr); + err = rx.Select(Query(ns).ReqTotal().Limit(0), qr); } ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.TotalCount(), expected); } -static void CheckCachedCountAggregations(std::shared_ptr &rx, const std::string &ns, size_t dataPerShard, - size_t shardsCount, const std::string &fieldLocation) { +static void CheckCachedCountAggregations(client::Reindexer &rx, const std::string &ns, size_t dataPerShard, size_t shardsCount, + const std::string &fieldLocation) { { // Check distributed query without offset client::QueryResults qr; - Error err = rx->Select(Query(ns).CachedTotal().Limit(0), qr); + Error err = rx.Select(Query(ns).CachedTotal().Limit(0), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.TotalCount(), shardsCount * dataPerShard); auto &agg = qr.GetAggregationResults(); @@ -2808,7 +2826,7 @@ static void CheckCachedCountAggregations(std::shared_ptr &rx, { // Check distributed query with offset client::QueryResults qr; - Error err = rx->Select(Query(ns).CachedTotal().Limit(0).Offset(1), qr); + Error err = rx.Select(Query(ns).CachedTotal().Limit(0).Offset(1), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.TotalCount(), shardsCount * dataPerShard); auto &agg = qr.GetAggregationResults(); @@ -2820,7 +2838,7 @@ static void CheckCachedCountAggregations(std::shared_ptr &rx, { // Check single shard query without offset client::QueryResults qr; - Error err = rx->Select(Query(ns).Where(fieldLocation, CondEq, "key2").CachedTotal().Limit(0), qr); + Error err = rx.Select(Query(ns).Where(fieldLocation, CondEq, "key2").CachedTotal().Limit(0), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.TotalCount(), dataPerShard); auto &agg = qr.GetAggregationResults(); @@ -2832,7 +2850,7 @@ static void CheckCachedCountAggregations(std::shared_ptr &rx, { // Check single shard query with offset client::QueryResults qr; - Error err = rx->Select(Query(ns).Where(fieldLocation, CondEq, "key2").CachedTotal().Limit(0).Offset(1), qr); + Error err = rx.Select(Query(ns).Where(fieldLocation, CondEq, "key2").CachedTotal().Limit(0).Offset(1), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.TotalCount(), dataPerShard); auto &agg = qr.GetAggregationResults(); @@ -2866,7 +2884,7 @@ TEST_F(ShardingApi, SelectOffsetLimit) { Init(std::move(cfg)); const std::string_view kNsName = default_namespace; - std::shared_ptr rx = svc_[0][0].Get()->api.reindexer; + auto &rx = *svc_[0][0].Get()->api.reindexer; const unsigned long kMaxCountOnShard = 40; Fill(kNsName, rx, "key3", 0, kMaxCountOnShard); @@ -2887,13 +2905,13 @@ TEST_F(ShardingApi, SelectOffsetLimit) { {10, 50, 50}, {10, 70, 70}, {50, 70, 70}, {50, 100, 70}, {119, 1, 1}, {119, 10, 1}, {0, 0, 0}, {120, 10, 0}, {150, 10, 0}}; - auto execVariant = [rx, this](const LimitOffsetCase &v, bool checkCount) { + auto execVariant = [&rx, this](const LimitOffsetCase &v, bool checkCount) { Query q = Query(default_namespace).Limit(v.limit).Offset(v.offset); if (checkCount) { q.ReqTotal(); } client::QueryResults qr; - Error err = rx->Select(q, qr); + Error err = rx.Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), v.count) << q.GetSQL(); if (checkCount) { diff --git a/cpp_src/gtests/tests/unit/sharding_extras_test.cc b/cpp_src/gtests/tests/unit/sharding_extras_test.cc index 4cd3a12a2..30e8e9540 100644 --- a/cpp_src/gtests/tests/unit/sharding_extras_test.cc +++ b/cpp_src/gtests/tests/unit/sharding_extras_test.cc @@ -16,11 +16,11 @@ TEST_F(ShardingExtrasApi, LargeProxiedSelects) { // Distributed qr { - std::shared_ptr rx = getNode(0)->api.reindexer; + auto &rx = *getNode(0)->api.reindexer; client::QueryResults qr; Query q = Query(default_namespace); - Error err = rx->Select(q, qr); + Error err = rx.Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), kShardDataCount * kShards); TestCout() << "Distributed select done" << std::endl; @@ -43,11 +43,11 @@ TEST_F(ShardingExtrasApi, LargeProxiedSelects) { // Proxied qr { - std::shared_ptr rx = getNode(0)->api.reindexer; + auto &rx = *getNode(0)->api.reindexer; client::QueryResults qr; Query q = Query(default_namespace).Where(kFieldLocation, CondEq, "key2"); - Error err = rx->Select(q, qr); + Error err = rx.Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), kShardDataCount); TestCout() << "Proxied select done" << std::endl; @@ -73,17 +73,17 @@ TEST_F(ShardingExtrasApi, SelectFTSeveralShards) { InitShardingConfig cfg; cfg.nodesInCluster = 1; Init(std::move(cfg)); - std::shared_ptr rx = getNode(0)->api.reindexer; + auto &rx = *getNode(0)->api.reindexer; client::QueryResults qr1; Query q = Query(default_namespace).Where(kFieldFTData, CondEq, RandString()); - Error err = rx->Select(q, qr1); + Error err = rx.Select(q, qr1); ASSERT_FALSE(err.ok()); ASSERT_EQ(err.what(), "Full text query by several sharding hosts"); client::QueryResults qr2; q.Where(kFieldLocation, CondEq, "key1"); - err = rx->Select(q, qr2); + err = rx.Select(q, qr2); ASSERT_TRUE(err.ok()); } @@ -175,16 +175,17 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { waitSync(kShardWithLocalNs, kLocalNamespace); // Use local ns as right_ns on correct shard + const auto kNodesCount = NodesCount(); for (const bool local : {true, false}) { - for (size_t i = 0; i < NodesCount(); ++i) { - std::shared_ptr rx = getNode(i)->api.reindexer; + for (size_t i = 0; i < kNodesCount; ++i) { + auto &rx = *getNode(i)->api.reindexer; const std::string key = "key" + std::to_string(kShardWithLocalNs); client::QueryResults qr; Query q = Query(default_namespace) .Local(local) .Where(kFieldLocation, CondEq, key) .InnerJoin(kFieldId, kFieldId, CondEq, Query(kLocalNamespace)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local || getSCIdxs(i).first == kShardWithLocalNs) { ASSERT_TRUE(err.ok()) << err.what() << "; i = " << i << "; location = " << key; ASSERT_EQ(qr.Count(), kExpectedJoinResults1.size()) << "; i = " << i << "; location = " << key; @@ -207,14 +208,14 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { } // Use local ns as left_ns (sharded ns has proper shardin key) for (const bool local : {true, false}) { - for (size_t i = 0; i < NodesCount(); ++i) { - std::shared_ptr rx = getNode(i)->api.reindexer; + for (size_t i = 0; i < kNodesCount; ++i) { + auto &rx = *getNode(i)->api.reindexer; const std::string key = "key" + std::to_string(kShardWithLocalNs); client::QueryResults qr; Query q = Query(kLocalNamespace) .Local(local) .InnerJoin(kFieldId, kFieldId, CondEq, Query(default_namespace).Where(kFieldLocation, CondEq, key)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local || getSCIdxs(i).first == kShardWithLocalNs) { ASSERT_TRUE(err.ok()) << err.what() << "; i = " << i << "; location = " << key; ASSERT_EQ(qr.Count(), kExpectedJoinResults2.size()) << "; i = " << i << "; location = " << key; @@ -237,11 +238,11 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { } // Use local ns as left_ns (sharded ns does not have sharding key) for (const bool local : {true, false}) { - for (size_t i = 0; i < NodesCount(); ++i) { - std::shared_ptr rx = getNode(i)->api.reindexer; + for (size_t i = 0; i < kNodesCount; ++i) { + auto &rx = *getNode(i)->api.reindexer; client::QueryResults qr; Query q = Query(kLocalNamespace).Local(local).InnerJoin(kFieldId, kFieldId, CondEq, Query(default_namespace)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local) { ASSERT_EQ(err.code(), errLogic) << err.what() << "; i = " << i; ASSERT_EQ(err.what(), "Query to all shard can't contain JOIN, MERGE or SUBQUERY") << "; i = " << i; @@ -267,8 +268,8 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { } // Use local ns as right_ns or left ns on wrong shard (this shard does not have this local namespace) for (const bool local : {true, false}) { - for (size_t i = 0; i < NodesCount(); ++i) { - std::shared_ptr rx = getNode(i)->api.reindexer; + for (size_t i = 0; i < kNodesCount; ++i) { + auto &rx = *getNode(i)->api.reindexer; const std::string key = "key" + std::to_string((kShardWithLocalNs + 1) % kShards); { client::QueryResults qr; @@ -276,7 +277,7 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { .Local(local) .Where(kFieldLocation, CondEq, key) .InnerJoin(kFieldId, kFieldId, CondEq, Query(kLocalNamespace)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local) { ASSERT_EQ(err.code(), errNotFound) << err.what() << "; i = " << i << "; location = " << key; } else if (getSCIdxs(i).first == kShardWithLocalNs) { @@ -292,7 +293,7 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { Query q = Query(kLocalNamespace) .Local(local) .InnerJoin(kFieldId, kFieldId, CondEq, Query(default_namespace).Where(kFieldLocation, CondEq, key)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local) { ASSERT_EQ(err.code(), errNotFound) << err.what() << "; i = " << i << "; location = " << key; } else if (getSCIdxs(i).first == kShardWithLocalNs) { @@ -307,11 +308,11 @@ TEST_F(ShardingExtrasApi, JoinBetweenShardedAndNonSharded) { } // Use sharded ns as left_ns without sharding key for (const bool local : {true, false}) { - for (size_t i = 0; i < NodesCount(); ++i) { - std::shared_ptr rx = getNode(i)->api.reindexer; + for (size_t i = 0; i < kNodesCount; ++i) { + auto &rx = *getNode(i)->api.reindexer; client::QueryResults qr; Query q = Query(default_namespace).Local(local).InnerJoin(kFieldId, kFieldId, CondEq, Query(kLocalNamespace)); - err = rx->Select(q, qr); + err = rx.Select(q, qr); if (!local) { ASSERT_EQ(err.code(), errLogic) << err.what() << "; i = " << i; ASSERT_EQ(err.what(), "Query to all shard can't contain JOIN, MERGE or SUBQUERY") << "; i = " << i; @@ -382,14 +383,14 @@ TEST_F(ShardingExtrasApi, DiffTmInResultFromShards) { cfg.nodesInCluster = 1; Init(std::move(cfg)); - std::shared_ptr rx = svc_[0][0].Get()->api.reindexer; + auto &rx = *svc_[0][0].Get()->api.reindexer; const std::map> sampleData = { {1, {{kFieldLocation, "key2"}, {kFieldData, RandString()}, {"f1", RandString()}}}, {2, {{kFieldLocation, "key1"}, {kFieldData, RandString()}, {"f2", RandString()}}}}; - auto insertItem = [this, rx](int id, const std::map &data) { - client::Item item = rx->NewItem(default_namespace); + auto insertItem = [this, &rx](int id, const std::map &data) { + client::Item item = rx.NewItem(default_namespace); ASSERT_TRUE(item.Status().ok()); WrSerializer wrser; reindexer::JsonBuilder jsonBuilder(wrser, ObjType::TypeObject); @@ -400,7 +401,7 @@ TEST_F(ShardingExtrasApi, DiffTmInResultFromShards) { jsonBuilder.End(); Error err = item.FromJSON(wrser.Slice()); ASSERT_TRUE(err.ok()) << err.what(); - err = rx->Upsert(default_namespace, item); + err = rx.Upsert(default_namespace, item); ASSERT_TRUE(err.ok()) << err.what(); }; @@ -411,7 +412,7 @@ TEST_F(ShardingExtrasApi, DiffTmInResultFromShards) { client::QueryResults qr; Query q = Query(default_namespace); - auto err = rx->Select(q, qr); + auto err = rx.Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { @@ -437,8 +438,8 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { cfg.nodesInCluster = 1; Init(cfg); const unsigned int kShardCount = cfg.shards; - std::shared_ptr rx = svc_[0][0].Get()->api.reindexer; - Error err = rx->OpenNamespace(default_namespace); + auto &rx = *svc_[0][0].Get()->api.reindexer; + Error err = rx.OpenNamespace(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); const unsigned long kMaxCountOnShard = 40; @@ -460,15 +461,15 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { TestCout() << "Checking select queries" << std::endl; const Query q(default_namespace); for (unsigned int k = 0; k < kShardCount; k++) { - std::shared_ptr rxSel = svc_[k][0].Get()->api.reindexer; + auto &rxSel = *svc_[k][0].Get()->api.reindexer; { lsnsByShard.clear(); lsnsByShard.resize(kShards); client::QueryResults qr(flags); - err = rxSel->Select(q, qr); + err = rxSel.Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); - for (auto i = qr.begin(); i != qr.end(); ++i) { + for (auto &i : qr) { auto item = i.GetItem(); std::string_view json = item.GetJSON(); gason::JsonParser parser; @@ -498,10 +499,10 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { lsnsByShard.resize(kShards); for (unsigned int l = 0; l < kShardCount; l++) { client::QueryResults qr(flags); - err = rxSel->Select( + err = rxSel.Select( Query::FromSQL(fmt::sprintf("select * from %s where %s = 'key%d'", default_namespace, kFieldLocation, l)), qr); ASSERT_TRUE(err.ok()) << err.what() << "; " << l; - for (auto i = qr.begin(); i != qr.end(); ++i) { + for (auto &i : qr) { auto item = i.GetItem(); int shardId = item.GetShardID(); lsn_t lsn = item.GetLSN(); @@ -521,14 +522,14 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { TestCout() << "Checking update queries" << std::endl; for (unsigned int k = 0; k < kShardCount; k++) { - std::shared_ptr rxUpdate = svc_[k][0].Get()->api.reindexer; + auto &rxUpdate = *svc_[k][0].Get()->api.reindexer; for (int l = 0; l < 3; l++) { client::QueryResults qr(flags); - err = rxUpdate->Update(Query::FromSQL(fmt::sprintf("update %s set %s='datanew' where %s='key%d'", default_namespace, - kFieldData, kFieldLocation, l)), - qr); + err = rxUpdate.Update(Query::FromSQL(fmt::sprintf("update %s set %s='datanew' where %s='key%d'", default_namespace, + kFieldData, kFieldLocation, l)), + qr); ASSERT_TRUE(err.ok()) << err.what(); - for (auto i = qr.begin(); i != qr.end(); ++i) { + for (auto &i : qr) { auto item = i.GetItem(); int shardId = item.GetShardID(); lsn_t lsn = item.GetLSN(); @@ -550,14 +551,14 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { lsnsByShard.clear(); lsnsByShard.resize(kShards); for (unsigned int k = 0; k < kShardCount; k++) { - std::shared_ptr rxDelete = svc_[k][0].Get()->api.reindexer; + auto &rxDelete = *svc_[k][0].Get()->api.reindexer; for (unsigned int l = 0; l < kShardCount; l++) { client::QueryResults qr(flags); - err = rxDelete->Delete( + err = rxDelete.Delete( Query::FromSQL(fmt::sprintf("Delete from %s where %s = 'key%d'", default_namespace, kFieldLocation, l)), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), kMaxCountOnShard); - for (auto i = qr.begin(); i != qr.end(); ++i) { + for (auto &i : qr) { auto item = i.GetItem(); int shardId = item.GetShardID(); lsn_t lsn = item.GetLSN(); @@ -581,12 +582,12 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { TestCout() << "Checking transactions" << std::endl; for (unsigned int k = 0; k < kShardCount; k++) { - std::shared_ptr rxTx = svc_[k][0].Get()->api.reindexer; - err = rxTx->TruncateNamespace(default_namespace); + auto &rxTx = *svc_[k][0].Get()->api.reindexer; + err = rxTx.TruncateNamespace(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); int startId = 0; for (unsigned int l = 0; l < kShardCount; l++) { - auto tx = rxTx->NewTransaction(default_namespace); + auto tx = rxTx.NewTransaction(default_namespace); auto FillTx = [&](std::string_view key, const size_t from, const size_t count) { for (size_t index = from; index < from + count; ++index) { client::Item item = tx.NewItem(); @@ -609,10 +610,10 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { FillTx("key" + std::to_string(l), startId, kMaxCountOnShard); startId += kMaxCountOnShard; client::QueryResults qr(flags); - err = rxTx->CommitTransaction(tx, qr); + err = rxTx.CommitTransaction(tx, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(qr.Count(), kMaxCountOnShard); - for (auto i = qr.begin(); i != qr.end(); ++i) { + for (auto &i : qr) { auto item = i.GetItem(); int shardId = item.GetShardID(); lsn_t lsn = item.GetLSN(); @@ -631,14 +632,14 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { if (isExpectingID) { // Single insertions do not have QR options. Only format is available TestCout() << "Checking insertions" << std::endl; for (unsigned int k = 0; k < kShardCount; k++) { - std::shared_ptr rxTx = svc_[k][0].Get()->api.reindexer; - err = rxTx->TruncateNamespace(default_namespace); + auto &rxTx = *svc_[k][0].Get()->api.reindexer; + err = rxTx.TruncateNamespace(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); for (unsigned int l = 0; l < kShardCount; l++) { WrSerializer wrser; client::Item item = CreateItem(default_namespace, rx, "key" + std::to_string(l), 1000 + k, wrser); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); - err = rx->Upsert(default_namespace, item); + err = rx.Upsert(default_namespace, item); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(item.GetShardID(), l) << k; ASSERT_TRUE(item.GetLSN().isEmpty()) << k; // Item without precepts does not have lsn @@ -646,7 +647,7 @@ TEST_F(ShardingExtrasApi, QrContainCorrectShardingId) { item = CreateItem(default_namespace, rx, "key" + std::to_string(l), 1000 + k, wrser); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); item.SetPrecepts({kFieldId + "=SERIAL()"}); - err = rx->Upsert(default_namespace, item); + err = rx.Upsert(default_namespace, item); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(item.GetShardID(), l) << k; lsn_t lsn = item.GetLSN(); @@ -664,12 +665,12 @@ TEST_F(ShardingExtrasApi, StrictMode) { cfg.rowsInTableOnShard = 0; Init(std::move(cfg)); - std::shared_ptr rx = svc_[0][0].Get()->api.reindexer; + auto &rx = *svc_[0][0].Get()->api.reindexer; const std::string kFieldForSingleShard = "my_new_field"; const std::string kUnknownField = "unknown_field"; const std::string kValue = "value"; - client::Item item = rx->NewItem(default_namespace); + client::Item item = rx.NewItem(default_namespace); ASSERT_TRUE(item.Status().ok()); WrSerializer wrser; reindexer::JsonBuilder jsonBuilder(wrser, ObjType::TypeObject); @@ -679,7 +680,7 @@ TEST_F(ShardingExtrasApi, StrictMode) { jsonBuilder.End(); Error err = item.FromJSON(wrser.Slice()); ASSERT_TRUE(err.ok()) << err.what(); - err = rx->Upsert(default_namespace, item); + err = rx.Upsert(default_namespace, item); ASSERT_TRUE(err.ok()) << err.what(); waitSync(default_namespace); std::vector limits = {UINT_MAX, 10}; // delete when executeQueryOnShard has one branch for distributed select diff --git a/cpp_src/gtests/tests/unit/string_function_test.cc b/cpp_src/gtests/tests/unit/string_function_test.cc index 441a1e29c..62ad2cea4 100644 --- a/cpp_src/gtests/tests/unit/string_function_test.cc +++ b/cpp_src/gtests/tests/unit/string_function_test.cc @@ -1,4 +1,12 @@ +#if defined(__GNUC__) && (__GNUC__ == 12) && defined(REINDEX_WITH_ASAN) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #include +#pragma GCC diagnostic pop +#else // REINDEX_WITH_ASAN +#include +#endif // REINDEX_WITH_ASAN + #include "gtest/gtest.h" #include "reindexer_api.h" #include "tools/customlocal.h" diff --git a/cpp_src/server/contrib/CMakeLists.txt b/cpp_src/server/contrib/CMakeLists.txt index a162877a8..039475da5 100644 --- a/cpp_src/server/contrib/CMakeLists.txt +++ b/cpp_src/server/contrib/CMakeLists.txt @@ -40,6 +40,6 @@ if(python3) WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/server/contrib COMMENT "Generate query.json.h" ) - add_custom_target(query_json DEPENDS ${QUERY_SCHEMA}) + add_custom_target(query_json ALL DEPENDS ${QUERY_SCHEMA}) endif() endif () diff --git a/cpp_src/server/contrib/server.md b/cpp_src/server/contrib/server.md index f0bda9e00..aa9739fe9 100644 --- a/cpp_src/server/contrib/server.md +++ b/cpp_src/server/contrib/server.md @@ -79,6 +79,7 @@ * [FulltextConfig](#fulltextconfig) * [FulltextFieldConfig](#fulltextfieldconfig) * [FulltextSynonym](#fulltextsynonym) + * [StopWordObject](#stopwordobject) * [GlobalReplicationStats](#globalreplicationstats) * [Index](#index) * [IndexCacheMemStats](#indexcachememstats) @@ -2249,6 +2250,7 @@ Query execution explainings |**selectors**
*optional*|Filter selectors, used to proccess query conditions|< [selectors](#explaindef-selectors) > array| |**sort_by_uncommitted_index**
*optional*|Optimization of sort by uncompleted index has been performed|boolean| |**sort_index**
*optional*|Index, which used for sort results|string| +|**subqueries**
*optional*|Explain of subqueries preselect|< [subqueries](#explaindef-subqueries) > array| |**total_us**
*optional*|Total query execution time|integer| @@ -2285,6 +2287,7 @@ Query execution explainings |---|---|---| |**comparators**
*optional*|Count of comparators used, for this selector|integer| |**cost**
*optional*|Cost expectation of this selector|integer| +|**description**
*optional*|Description of the selector|string| |**explain_preselect**
*optional*|Preselect in joined namespace execution explainings|[ExplainDef](#explaindef)| |**explain_select**
*optional*|One of selects in joined namespace execution explainings|[ExplainDef](#explaindef)| |**field**
*optional*|Field or index name|string| @@ -2293,6 +2296,17 @@ Query execution explainings |**keys**
*optional*|Number of uniq keys, processed by this selector (may be incorrect, in case of internal query optimization/caching|integer| |**matched**
*optional*|Count of processed documents, matched this selector|integer| |**method**
*optional*|Method, used to process condition|enum (scan, index, inner_join, left_join)| +|**type**
*optional*|Type of the selector|string| + + +**subqueries** + +|Name|Description|Schema| +|---|---|---| +|**explain**
*optional*|Explain of the subquery's preselect|[ExplainDef](#explaindef)| +|**field**
*optional*|Name of field being compared with the subquery's result|string| +|**keys**
*optional*|Count of keys being compared with the subquery's result|integer| +|**namespace**
*optional*|Subquery's namespace name|string| @@ -2349,7 +2363,7 @@ Fulltext Index configuration |**position_boost**
*optional*|Boost of search query term position
**Default** : `1.0`
**Minimum value** : `0`
**Maximum value** : `10`|number (float)| |**position_weight**
*optional*|Weight of search query term position in final rank. 0: term position will not change final rank. 1: term position will affect to final rank in 0 - 100% range
**Default** : `0.1`
**Minimum value** : `0`
**Maximum value** : `1`|number (float)| |**stemmers**
*optional*|List of stemmers to use|< string > array| -|**stop_words**
*optional*|List of stop words. Words from this list will be ignored in documents and queries|< string > array| +|**stop_words**
*optional*|List of objects of stop words. Words from this list will be ignored when building indexes. |< [StopWordObject](#stopwordobject) > array| |**sum_ranks_by_fields_ratio**
*optional*|Ratio to summation of ranks of match one term in several fields. For example, if value of this ratio is K, request is '@+f1,+f2,+f3 word', ranks of match in fields are R1, R2, R3 and R2 < R1 < R3, final rank will be R = R2 + K*R1 + K*K*R3
**Default** : `0.0`
**Minimum value** : `0`
**Maximum value** : `1`|number (float)| |**synonyms**
*optional*|List of synonyms for replacement|< [FulltextSynonym](#fulltextsynonym) > array| |**term_len_boost**
*optional*|Boost of search query term length
**Default** : `1.0`
**Minimum value** : `0`
**Maximum value** : `10`|number (float)| @@ -2410,6 +2424,15 @@ Fulltext synonym definition +### StopWordObject +Stop word object definition +|Name|Description|Schema| +|---|---|---| +|**word**
*optional*|Stop word|string| +|**is_morpheme**
*optional*|If the value is true, the word can be included in search results in queries such as 'word*', 'word~' etc.|boolean| + + + ### GlobalReplicationStats |Name|Description|Schema| diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index d8ff8dd7e..018232ec4 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -2928,6 +2928,17 @@ definitions: description: "Descent or ascent sorting direction" type: boolean + FtStopWordObject: + type: object + properties: + word: + description: "Stop word" + type: string + is_morpheme: + type: boolean + description: "If the value is true, the word can be included in search results in queries such as 'word*', 'word~' etc." + default: false + FulltextConfig: type: object description: "Fulltext Index configuration" @@ -2966,9 +2977,9 @@ definitions: description: "List of symbols, which will be threated as word part, all other symbols will be thrated as wors separators" stop_words: type: array - description: "List of stop words. Words from this list will be ignored in documents and queries" + description: "List of objects of stop words. Words from this list will be ignored when building indexes" items: - type: string + $ref: "#/definitions/FtStopWordObject" stemmers: type: array default: ["en","ru"] @@ -3448,6 +3459,12 @@ definitions: keys: type: integer description: "Number of uniq keys, processed by this selector (may be incorrect, in case of internal query optimization/caching" + type: + type: string + description: "Type of the selector" + description: + type: string + description: "Description of the selector" explain_preselect: description: "Preselect in joined namespace execution explainings" $ref: "#/definitions/ExplainDef" @@ -3515,6 +3532,24 @@ definitions: values_count: type: integer description: resulting size of query values set + subqueries: + type: array + description: "Explain of subqueries preselect" + items: + type: object + properties: + namespace: + type: string + description: "Subquery's namespace name" + keys: + type: integer + description: "Count of keys being compared with the subquery's result" + field: + type: string + description: "Name of field being compared with the subquery's result" + explain: + description: "Explain of the subquery's preselect" + $ref: "#/definitions/ExplainDef" AggregationResDef: diff --git a/cpp_src/server/dbmanager.cc b/cpp_src/server/dbmanager.cc index 908b4299e..7891cd2cc 100644 --- a/cpp_src/server/dbmanager.cc +++ b/cpp_src/server/dbmanager.cc @@ -79,7 +79,7 @@ Error DBManager::OpenDatabase(const std::string &dbName, AuthContext &auth, bool status = dbConnect(it->second.get()); if (!status.ok()) return status; auth.db_ = it->second.get(); - return errOK; + return Error(); } lck.unlock(); @@ -99,7 +99,7 @@ Error DBManager::OpenDatabase(const std::string &dbName, AuthContext &auth, bool status = dbConnect(it->second.get()); if (!status.ok()) return status; auth.db_ = it->second.get(); - return errOK; + return Error(); } status = loadOrCreateDatabase(dbName, true, true, auth); @@ -110,7 +110,7 @@ Error DBManager::OpenDatabase(const std::string &dbName, AuthContext &auth, bool it = dbs_.find(dbName); assertrx(it != dbs_.end()); auth.db_ = it->second.get(); - return errOK; + return Error(); } Error DBManager::loadOrCreateDatabase(const std::string &dbName, bool allowDBErrors, bool withAutorepair, const AuthContext &auth) { @@ -121,7 +121,8 @@ Error DBManager::loadOrCreateDatabase(const std::string &dbName, bool allowDBErr std::string storagePath = !config_.StoragePath.empty() ? fs::JoinPath(config_.StoragePath, dbName) : ""; logPrintf(LogInfo, "Loading database %s", dbName); - auto db = std::make_unique(reindexer::ReindexerConfig().WithClientStats(clientsStats_).WithUpdatesSize(config_.MaxUpdatesSize)); + auto db = std::make_unique( + reindexer::ReindexerConfig().WithClientStats(clientsStats_).WithUpdatesSize(config_.MaxUpdatesSize)); StorageTypeOpt storageType = kStorageTypeOptLevelDB; switch (storageType_) { case datastorage::StorageType::LevelDB: diff --git a/cpp_src/tools/json2kv.cc b/cpp_src/tools/json2kv.cc index c693796a8..e7a7f1a15 100644 --- a/cpp_src/tools/json2kv.cc +++ b/cpp_src/tools/json2kv.cc @@ -32,7 +32,9 @@ Variant jsonValue2Variant(const gason::JsonValue &v, KeyValueType t, std::string -> Variant { throw Error(errLogic, "Error parsing json field '%s' - got number, expected %s", fieldName, t.Name()); }); case gason::JSON_STRING: return t.EvaluateOneOf( - [&](OneOf) { return Variant(p_string(json_string_ftr{v.sval.ptr})); }, + [&](OneOf) { + return Variant(p_string(json_string_ftr{v.sval.ptr}), Variant::no_hold_t{}); + }, [&](KeyValueType::Uuid) { return Variant{Uuid{v.toString()}}; }, [&](OneOf) -> Variant { @@ -59,7 +61,7 @@ Variant jsonValue2Variant(const gason::JsonValue &v, KeyValueType t, std::string [](KeyValueType::Double) noexcept { return Variant(0.0); }, [](KeyValueType::Bool) noexcept { return Variant(false); }, [](KeyValueType::Int) noexcept { return Variant(0); }, [](KeyValueType::Int64) noexcept { return Variant(static_cast(0)); }, - [](KeyValueType::String) { return Variant(p_string(static_cast(nullptr))); }, + [](KeyValueType::String) { return Variant(static_cast(nullptr)); }, [](KeyValueType::Uuid) noexcept { return Variant{Uuid{}}; }, [&](OneOf) -> Variant { throw Error(errLogic, "Error parsing json field '%s' - got null, expected %s", fieldName, t.Name()); diff --git a/cpp_src/tools/serializer.h b/cpp_src/tools/serializer.h index 5b7a27292..e6cc533ef 100644 --- a/cpp_src/tools/serializer.h +++ b/cpp_src/tools/serializer.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "core/cjson/ctag.h" #include "core/keyvalue/uuid.h" diff --git a/cpp_src/tools/stringstools.cc b/cpp_src/tools/stringstools.cc index 23e77fa50..3ac3371d9 100644 --- a/cpp_src/tools/stringstools.cc +++ b/cpp_src/tools/stringstools.cc @@ -10,7 +10,6 @@ #include "itoa/itoa.h" #include "stringstools.h" #include "tools/assertrx.h" -#include "tools/customlocal.h" #include "tools/randomgenerator.h" #include "tools/stringstools.h" #include "utf8cpp/utf8.h" @@ -296,24 +295,6 @@ std::pair calcUtf8BeforeDelims(const char *str, int pos, size_t return std::make_pair(str + pos - ptr, charCounter); } -void check_for_replacement(wchar_t &ch) { - if (ch == 0x451) { // 'ё' - ch = 0x435; // 'е' - } -} - -void check_for_replacement(uint32_t &ch) { - if (ch == 0x451) { // 'ё' - ch = 0x435; // 'е' - } -} - -bool is_number(std::string_view str) { - uint16_t i = 0; - while ((i < str.length() && IsDigit(str[i]))) i++; - return (i && i == str.length()); -} - void split(std::string_view str, std::string &buf, std::vector &words, const std::string &extraWordSymbols) { // assuming that the 'ToLower' function and the 'check for replacement' function should not change the character size in bytes buf.resize(str.length()); @@ -475,7 +456,7 @@ template bool checkIfEndsWith(std::string_view pattern, std: template bool checkIfEndsWith(std::string_view pattern, std::string_view src) noexcept; template <> -int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) { +int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) noexcept { auto itl = lhs.begin(); auto itr = rhs.begin(); @@ -497,11 +478,11 @@ int collateCompare(std::string_view lhs, std::string_view rhs, con } template <> -int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) { +int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) noexcept { auto itl = lhs.data(); auto itr = rhs.data(); - for (; itl != lhs.data() + lhs.size() && itr != rhs.size() + rhs.data();) { + for (auto lhsEnd = lhs.data() + lhs.size(), rhsEnd = rhs.size() + rhs.data(); itl != lhsEnd && itr != rhsEnd;) { auto chl = ToLower(utf8::unchecked::next(itl)); auto chr = ToLower(utf8::unchecked::next(itr)); @@ -518,7 +499,7 @@ int collateCompare(std::string_view lhs, std::string_view rhs, cons } template <> -int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) { +int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) noexcept { char *posl = nullptr; char *posr = nullptr; @@ -538,7 +519,7 @@ int collateCompare(std::string_view lhs, std::string_view rhs, c } template <> -int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &sortOrderTable) { +int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &sortOrderTable) noexcept { auto itl = lhs.data(); auto itr = rhs.data(); @@ -562,7 +543,7 @@ int collateCompare(std::string_view lhs, std::string_view rhs, co } template <> -int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) { +int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable &) noexcept { size_t l1 = lhs.size(); size_t l2 = rhs.size(); int res = memcmp(lhs.data(), rhs.data(), std::min(l1, l2)); @@ -793,7 +774,7 @@ Error getBytePosInMultilineString(std::string_view str, const size_t line, const } if ((currLine == line) && (charPos == currCharPos)) { bytePos = it - str.begin() - 1; - return errOK; + return Error(); } return Error(errNotValid, "Wrong cursor position: line=%d, pos=%d", line, charPos); } diff --git a/cpp_src/tools/stringstools.h b/cpp_src/tools/stringstools.h index 98d354930..3f62623a6 100644 --- a/cpp_src/tools/stringstools.h +++ b/cpp_src/tools/stringstools.h @@ -11,6 +11,7 @@ #include "core/keyvalue/variant.h" #include "core/type_consts.h" #include "tools/customhash.h" +#include "tools/customlocal.h" #include "tools/errors.h" namespace reindexer { @@ -107,18 +108,18 @@ template [[nodiscard]] Pos wordToByteAndCharPos(std::string_view str, int wordPosition, const std::string& extraWordSymbols); template -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable& sortOrderTable); +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable& sortOrderTable) noexcept; template <> -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&); +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&) noexcept; template <> -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&); +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&) noexcept; template <> -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&); +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&) noexcept; template <> -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&); +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&) noexcept; template <> -[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&); -[[nodiscard]] inline int collateCompare(std::string_view lhs, std::string_view rhs, const CollateOpts& collateOpts) { +[[nodiscard]] int collateCompare(std::string_view lhs, std::string_view rhs, const SortingPrioritiesTable&) noexcept; +[[nodiscard]] inline int collateCompare(std::string_view lhs, std::string_view rhs, const CollateOpts& collateOpts) noexcept { switch (collateOpts.mode) { case CollateASCII: return collateCompare(lhs, rhs, collateOpts.sortOrderTable); @@ -139,9 +140,18 @@ std::string utf16_to_utf8(const std::wstring& src); std::wstring& utf8_to_utf16(std::string_view src, std::wstring& dst); std::string& utf16_to_utf8(const std::wstring& src, std::string& dst); -void check_for_replacement(wchar_t& ch); -void check_for_replacement(uint32_t& ch); -bool is_number(std::string_view str); +inline void check_for_replacement(wchar_t& ch) noexcept { + ch = (ch == 0x451) ? 0x435 : ch; // 'ё' -> 'е' +} +inline void check_for_replacement(uint32_t& ch) noexcept { + ch = (ch == 0x451) ? 0x435 : ch; // 'ё' -> 'е' +} +inline bool is_number(std::string_view str) noexcept { + uint16_t i = 0; + for (; (i < str.length() && IsDigit(str[i])); ++i) + ; + return (i && i == str.length()); +} int fast_strftime(char* buf, const tm* tm); std::string urldecode2(std::string_view str); diff --git a/cpp_src/vendor/cpp-btree/btree.h b/cpp_src/vendor/cpp-btree/btree.h index c06695acc..4339dd5f9 100644 --- a/cpp_src/vendor/cpp-btree/btree.h +++ b/cpp_src/vendor/cpp-btree/btree.h @@ -719,10 +719,10 @@ struct btree_iterator { typedef btree_iterator const_iterator; typedef btree_iterator self_type; - btree_iterator() : node(NULL), position(-1) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} - btree_iterator(const iterator &x) : node(x.node), position(x.position) {} - btree_iterator &operator=(const iterator &x) { + btree_iterator() noexcept : node(NULL), position(-1) {} + btree_iterator(Node *n, int p) noexcept : node(n), position(p) {} + btree_iterator(const iterator &x) noexcept : node(x.node), position(x.position) {} + btree_iterator &operator=(const iterator &x) noexcept { if (reinterpret_cast(this) != &x) { node = x.node; position = x.position; @@ -731,45 +731,45 @@ struct btree_iterator { } // Increment/decrement the iterator. - void increment() { + void increment() noexcept { if (node->leaf() && ++position < node->count()) { return; } increment_slow(); } - void increment_by(int count); - void increment_slow(); + void increment_by(int count) noexcept; + void increment_slow() noexcept; - void decrement() { + void decrement() noexcept { if (node->leaf() && --position >= 0) { return; } decrement_slow(); } - void decrement_slow(); + void decrement_slow() noexcept; - bool operator==(const const_iterator &x) const { return node == x.node && position == x.position; } - bool operator!=(const const_iterator &x) const { return node != x.node || position != x.position; } + bool operator==(const const_iterator &x) const noexcept { return node == x.node && position == x.position; } + bool operator!=(const const_iterator &x) const noexcept { return node != x.node || position != x.position; } // Accessors for the key/value the iterator is pointing at. - const key_type &key() const { return node->key(position); } - reference operator*() const { return node->value(position); } - pointer operator->() const { return &node->value(position); } + const key_type &key() const noexcept { return node->key(position); } + reference operator*() const noexcept { return node->value(position); } + pointer operator->() const noexcept { return &node->value(position); } - self_type &operator++() { + self_type &operator++() noexcept { increment(); return *this; } - self_type &operator--() { + self_type &operator--() noexcept { decrement(); return *this; } - self_type operator++(int) { + self_type operator++(int) noexcept { self_type tmp = *this; ++*this; return tmp; } - self_type operator--(int) { + self_type operator--(int) noexcept { self_type tmp = *this; --*this; return tmp; @@ -1534,7 +1534,7 @@ void btree_node

::swap(btree_node *x) { //// // btree_iterator methods template -void btree_iterator::increment_slow() { +void btree_iterator::increment_slow() noexcept { if (node->leaf()) { assertrx(position >= node->count()); self_type save(*this); @@ -1557,7 +1557,7 @@ void btree_iterator::increment_slow() { } template -void btree_iterator::increment_by(int count) { +void btree_iterator::increment_by(int count) noexcept { while (count > 0) { if (node->leaf()) { int rest = node->count() - position; @@ -1574,7 +1574,7 @@ void btree_iterator::increment_by(int count) { } template -void btree_iterator::decrement_slow() { +void btree_iterator::decrement_slow() noexcept { if (node->leaf()) { assertrx(position <= -1); self_type save(*this); diff --git a/cpp_src/vendor/cpp-btree/btree_container.h b/cpp_src/vendor/cpp-btree/btree_container.h index 94469d201..759db216e 100644 --- a/cpp_src/vendor/cpp-btree/btree_container.h +++ b/cpp_src/vendor/cpp-btree/btree_container.h @@ -53,14 +53,14 @@ class btree_container { btree_container(const self_type &x) : tree_(x.tree_) {} // Iterator routines. - iterator begin() { return tree_.begin(); } - const_iterator begin() const { return tree_.begin(); } - iterator end() { return tree_.end(); } - const_iterator end() const { return tree_.end(); } - reverse_iterator rbegin() { return tree_.rbegin(); } - const_reverse_iterator rbegin() const { return tree_.rbegin(); } - reverse_iterator rend() { return tree_.rend(); } - const_reverse_iterator rend() const { return tree_.rend(); } + iterator begin() noexcept(noexcept(std::declval().begin())) { return tree_.begin(); } + const_iterator begin() const noexcept(noexcept(std::declval().begin())) { return tree_.begin(); } + iterator end() noexcept(noexcept(std::declval().end())) { return tree_.end(); } + const_iterator end() const noexcept(noexcept(std::declval().end())) { return tree_.end(); } + reverse_iterator rbegin() noexcept(noexcept(std::declval().rbegin())) { return tree_.rbegin(); } + const_reverse_iterator rbegin() const noexcept(noexcept(std::declval().rbegin())) { return tree_.rbegin(); } + reverse_iterator rend() noexcept(noexcept(std::declval().rend())) { return tree_.rend(); } + const_reverse_iterator rend() const noexcept(noexcept(std::declval().rend())) { return tree_.rend(); } // Lookup routines. iterator lower_bound(const key_type &key) { return tree_.lower_bound(key); } @@ -102,18 +102,18 @@ class btree_container { void verify() const { tree_.verify(); } // Size routines. - size_type size() const { return tree_.size(); } - size_type max_size() const { return tree_.max_size(); } - bool empty() const { return tree_.empty(); } - size_type height() const { return tree_.height(); } - size_type internal_nodes() const { return tree_.internal_nodes(); } - size_type leaf_nodes() const { return tree_.leaf_nodes(); } - size_type nodes() const { return tree_.nodes(); } - size_type bytes_used() const { return tree_.bytes_used(); } - static double average_bytes_per_value() { return Tree::average_bytes_per_value(); } - double fullness() const { return tree_.fullness(); } - double overhead() const { return tree_.overhead(); } - const key_compare &key_comp() const { return tree_.key_comp(); } + size_type size() const noexcept(noexcept(std::declval().size())) { return tree_.size(); } + size_type max_size() const noexcept(noexcept(std::declval().max_size())) { return tree_.max_size(); } + bool empty() const noexcept(noexcept(std::declval().empty())) { return tree_.empty(); } + size_type height() const noexcept(noexcept(std::declval().height())) { return tree_.height(); } + size_type internal_nodes() const noexcept(noexcept(std::declval().internal_nodes())) { return tree_.internal_nodes(); } + size_type leaf_nodes() const noexcept(noexcept(std::declval().leaf_nodes())) { return tree_.leaf_nodes(); } + size_type nodes() const noexcept(noexcept(std::declval().nodes())) { return tree_.nodes(); } + size_type bytes_used() const noexcept(noexcept(std::declval().bytes_used())) { return tree_.bytes_used(); } + static double average_bytes_per_value() noexcept(noexcept(Tree::average_bytes_per_value())) { return Tree::average_bytes_per_value(); } + double fullness() const noexcept(noexcept(std::declval().fullness())) { return tree_.fullness(); } + double overhead() const noexcept(noexcept(std::declval().overhead())) { return tree_.overhead(); } + const key_compare &key_comp() const noexcept(noexcept(std::declval().key_comp())) { return tree_.key_comp(); } bool operator==(const self_type &x) const { if (size() != x.size()) { @@ -315,4 +315,4 @@ class btree_multi_container : public btree_container { } // namespace btree -#endif // UTIL_BTREE_BTREE_CONTAINER_H__ +#endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/cpp_src/vendor/prometheus/family.h b/cpp_src/vendor/prometheus/family.h index 5c59122d0..6992bc161 100644 --- a/cpp_src/vendor/prometheus/family.h +++ b/cpp_src/vendor/prometheus/family.h @@ -157,7 +157,7 @@ T& Family::Add(std::map&& labels, int64_t epoch, Ar auto metrics_iter = metrics_.find(hash); if (metrics_iter != metrics_.end()) { -#ifndef NDEBUG +#if !defined(NDEBUG) && defined(WITH_STDLIB_DEBUG) auto labels_iter = labels_.find(hash); assertrx(labels_iter != labels_.end()); const auto& old_labels = labels_iter->second; @@ -166,7 +166,7 @@ T& Family::Add(std::map&& labels, int64_t epoch, Ar metrics_iter->second.epoch = epoch; return *metrics_iter->second.ptr; } else { -#ifndef NDEBUG +#if !defined(NDEBUG) && defined(WITH_STDLIB_DEBUG) for (auto& label_pair : labels) { auto& label_name = label_pair.first; assertrx(CheckLabelName(label_name)); diff --git a/cpp_src/vendor/prometheus/impl/check_names.cc b/cpp_src/vendor/prometheus/impl/check_names.cc index 0aabbc88c..6a800ce31 100644 --- a/cpp_src/vendor/prometheus/impl/check_names.cc +++ b/cpp_src/vendor/prometheus/impl/check_names.cc @@ -1,15 +1,21 @@ #include "prometheus/check_names.h" -#include - #if defined(__GLIBCXX__) && __GLIBCXX__ <= 20150623 #define STD_REGEX_IS_BROKEN #endif +#if defined(__GNUC__) && (__GNUC__ == 12) && (__GNUC_MINOR__ == 2) && defined(REINDEX_WITH_ASAN) +// regex header is broken in GCC 12.2 with ASAN +#define STD_REGEX_IS_BROKEN +#endif #if defined(_MSC_VER) && _MSC_VER < 1900 #define STD_REGEX_IS_BROKEN #endif +#ifndef STD_REGEX_IS_BROKEN +#include +#endif + namespace prometheus { bool CheckMetricName(const std::string& name) { // see https://prometheus.io/docs/concepts/data_model/ diff --git a/cpp_src/vendor/spdlog/details/os.h b/cpp_src/vendor/spdlog/details/os.h index cf8501181..a3f05d782 100644 --- a/cpp_src/vendor/spdlog/details/os.h +++ b/cpp_src/vendor/spdlog/details/os.h @@ -247,8 +247,9 @@ inline size_t filesize(FILE *f) #else // unix int fd = fileno(f); - //64 bits(but not in osx or cygwin, where fstat64 is deprecated) -#if !defined(__FreeBSD__) && !defined(__APPLE__) && (defined(__x86_64__) || defined(__ppc64__)) && !defined(__CYGWIN__) + // 64 bits(but not in osx, linux/musl or cygwin, where fstat64 is deprecated) +#if ((defined(__linux__) && defined(__GLIBC__)) || defined(__sun) || defined(_AIX)) && \ + (defined(__LP64__) || defined(_LP64)) struct stat64 st ; if (fstat64(fd, &st) == 0) return static_cast(st.st_size); diff --git a/dependencies.sh b/dependencies.sh index c2bfe65f1..bfd738a25 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -37,7 +37,7 @@ almalinux9_rpms="gcc-c++ make snappy-devel leveldb-devel gperftools-devel findut fedora_debs=" gcc-c++ make snappy-devel leveldb-devel gperftools-devel findutils curl tar unzip rpm-build rpmdevtools git" centos7_debs="centos-release-scl devtoolset-10-gcc devtoolset-10-gcc-c++ make snappy-devel leveldb-devel gperftools-devel findutils curl tar unzip rpm-build rpmdevtools git" debian_debs="build-essential g++ libunwind-dev libgoogle-perftools-dev libsnappy-dev libleveldb-dev make curl unzip git" -alpine_apks="g++ snappy-dev leveldb-dev libexecinfo-dev make curl cmake unzip git" +alpine_apks="g++ snappy-dev leveldb-dev libunwind-dev make curl cmake unzip git" arch_pkgs="gcc snappy leveldb make curl cmake unzip git" redos_rpms="gcc gcc-c++ make snappy-devel leveldb-devel gperftools-devel findutils curl tar unzip git cmake rpm-build python-srpm-macros" @@ -237,7 +237,7 @@ install_alpine() { if [ $? -eq 0 ]; then success_msg "Package '$pkg' was installed successfully." else - error_msg "Could not install '$pkg' package. Try 'apt-get update && apt-get install $pkg'" && return 1 + error_msg "Could not install '$pkg' package. Try 'apk update && apk add $pkg'" && return 1 fi fi done diff --git a/ftfastconfig.go b/ftfastconfig.go index 2eafbea2d..d55dfccb5 100644 --- a/ftfastconfig.go +++ b/ftfastconfig.go @@ -41,7 +41,6 @@ type FtTyposDetailedConfig struct { MaxExtraLetters int `json:"max_extra_letters"` } - type FtBaseRanking struct { // Relevancy of full word match // Values range: [0,500] @@ -81,6 +80,11 @@ type FtBaseRanking struct { Synonyms int `json:"synonyms_proc"` } +type StopWord struct { + Word string `json:"word"` + IsMorpheme bool `json:"is_morpheme"` +} + // FtFastConfig configurarion of FullText search index type FtFastConfig struct { // boost of bm25 ranking. default value 1. @@ -139,8 +143,10 @@ type FtFastConfig struct { EnableTranslit bool `json:"enable_translit"` // Enable wrong keyboard layout variants processing EnableKbLayout bool `json:"enable_kb_layout"` - // List of stop words. Words from this list will be ignored in documents and queries - StopWords []string `json:"stop_words"` + // List of objects of stop words. Words from this list will be ignored when building indexes + // but can be included in search results in queries such as 'word*', 'word~' etc. if for the stop-word attribute is_morpheme is true. + // The list item can be either a reindexer.StopWord, or string + StopWords []interface{} `json:"stop_words"` // List of synonyms for replacement Synonyms []struct { // List source tokens in query, which will be replaced with alternatives @@ -201,8 +207,7 @@ func DefaultFtFastConfig() FtFastConfig { MaxTotalAreasToCache: -1, Optimization: "Memory", EnablePreselectBeforeFt: false, - FtBaseRankingConfig: &FtBaseRanking{FullMatch: 100, PrefixMin: 50, SuffixMin:10, Typo:85, TypoPenalty: 15, StemmerPenalty: 15, Kblayout: 90, Translit:90, Synonyms:95}, - + FtBaseRankingConfig: &FtBaseRanking{FullMatch: 100, PrefixMin: 50, SuffixMin: 10, Typo: 85, TypoPenalty: 15, StemmerPenalty: 15, Kblayout: 90, Translit: 90, Synonyms: 95}, } } diff --git a/ftfuzzyconfig.go b/ftfuzzyconfig.go index 570970a54..223faed4b 100644 --- a/ftfuzzyconfig.go +++ b/ftfuzzyconfig.go @@ -39,15 +39,15 @@ type FtFuzzyConfig struct { EnableTranslit bool `json:"enable_translit"` // Enable wrong keyboard layout variants processing EnableKbLayout bool `json:"enable_kb_layout"` - // List of stop words. Words from this list will be ignored in documents and queries - StopWords []string `json:"stop_words"` + // List of objects of stop words. Words from this list will be ignored when building indexes + // but can be included in search results in queries such as 'word*', 'word~' etc. if for the stop-word attribute is_morpheme is true + StopWords []interface{} `json:"stop_words"` // Log level of full text search engine LogLevel int `json:"log_level"` // Extra symbols, which will be threated as parts of word to addition to letters and digits ExtraWordSymbols string `json:"extra_word_symbols"` // Config for subterm rank multiplier FtBaseRankingConfig *FtBaseRanking `json:"base_ranking,omitempty"` - } func DefaultFtFuzzyConfig() FtFuzzyConfig { @@ -69,6 +69,6 @@ func DefaultFtFuzzyConfig() FtFuzzyConfig { EnableKbLayout: true, LogLevel: 0, ExtraWordSymbols: "/-+", - FtBaseRankingConfig: &FtBaseRanking{FullMatch: 100, PrefixMin: 50, SuffixMin:10, Typo:85, TypoPenalty: 15, StemmerPenalty: 15, Kblayout: 90, Translit:90, Synonyms:95}, + FtBaseRankingConfig: &FtBaseRanking{FullMatch: 100, PrefixMin: 50, SuffixMin: 10, Typo: 85, TypoPenalty: 15, StemmerPenalty: 15, Kblayout: 90, Translit: 90, Synonyms: 95}, } } diff --git a/fulltext.md b/fulltext.md index 672ab7e52..7226e8a64 100644 --- a/fulltext.md +++ b/fulltext.md @@ -6,11 +6,11 @@ Reindexer has builtin full text search engine. This document describes usage of - [Define full text index fields](#define-full-text-index-fields) - [Query to full text index](#query-to-full-text-index) - [Text query format](#text-query-format) - - [Patterns](#patterns) - - [Field selection](#field-selection) - - [Binary operators](#binary-operators) - - [Escape character](#escape-character) - - [Phrase search](#phrase-search) + - [Patterns](#patterns) + - [Field selection](#field-selection) + - [Binary operators](#binary-operators) + - [Escape character](#escape-character) + - [Phrase search](#phrase-search) - [Examples of text queris](#examples-of-text-queris) - [Natural language processing](#natural-language-processing) - [Merging queries results](#merging-queries-results) @@ -24,9 +24,10 @@ Reindexer has builtin full text search engine. This document describes usage of - [Performance and memory usage](#performance-and-memory-usage) - [Configuration](#configuration) - [Base config parameters](#base-config-parameters) + - [Stopwords details](#stopwords-details) - [Detailed typos config](#detailed-typos-config) - [Base ranking config](#base-ranking-config) - - [Limitations and know issues](#limitations-and-know-issues) + - [Limitations and know issues](#limitations-and-know-issues) ## LIKE @@ -34,17 +35,17 @@ Reindexer has builtin full text search engine. This document describes usage of For simple search in text can be used operator `LIKE`. It search strings which match a pattern. In the pattern `_` means any char and `%` means any sequence of chars. ``` - In Go: - query := db.Query("items"). - Where("field", reindexer.LIKE, "pattern") + In Go: + query := db.Query("items"). + Where("field", reindexer.LIKE, "pattern") - In SQL: - SELECT * FROM items WHERE fields LIKE 'pattern' + In SQL: + SELECT * FROM items WHERE fields LIKE 'pattern' ``` ``` - 'me_t' corresponds to 'meet', 'meat', 'melt' and so on - '%tion' corresponds to 'tion', 'condition', 'creation' and so on + 'me_t' corresponds to 'meet', 'meat', 'melt' and so on + '%tion' corresponds to 'tion', 'condition', 'creation' and so on ``` @@ -54,8 +55,8 @@ Full text search is performed in fields marked with `text` tag: ```go type Item struct { - ID int64 `reindex:"id,,pk"` - Description string `reindex:"description,text"` + ID int64 `reindex:"id,,pk"` + Description string `reindex:"description,text"` } ``` @@ -63,10 +64,10 @@ Full text search is also available for multiple fields of composite index marked ```go type Item struct { - ID int64 `reindex:"id,,pk"` - Name string `reindex:"name,-"` - Description string `reindex:"description,-"` - _ struct{} `reindex:"name+description=text_search,text,composite` + ID int64 `reindex:"id,,pk"` + Name string `reindex:"name,-"` + Description string `reindex:"description,-"` + _ struct{} `reindex:"name+description=text_search,text,composite` } ``` In this example full text index will include fields `name` and `description`,`text_search` is short alias of composite index name for using in Queries. @@ -78,22 +79,22 @@ Full text index is case insensitive. The source text is tokenized to set of word Queries to full text index are constructed by usual query interface ```go - query := db.Query ("items"). - Match ("name+description","text query","") + query := db.Query ("items"). + Match ("name+description","text query","") ``` Or equivalent query using name alias: ```go - query := db.Query ("items"). - Match ("text_search","text query","") + query := db.Query ("items"). + Match ("text_search","text query","") ``` Queries to full text index can be combined with conditions on another fields. e.g: ```go - query := db.Query ("items"). - Match ("description","text query"). - WhereInt("year",reindexer.GT,2010) + query := db.Query ("items"). + Match ("description","text query"). + WhereInt("year",reindexer.GT,2010) ``` Each result of query contains rank of match. Rank is integer from 0 to 255. 0 - lowest relevancy, 255 - best relevancy. The query Iterator has method `Rank()`, which returns rank of current result @@ -166,26 +167,26 @@ There are built in stemmers support in full text search. It enables natural lang It is possible to merge multiple queries results and sort final result by relevancy. ```go - query := db.Query ("items"). - Match ("description","text query1") - q2 := db.Query ("another_items"). - Match ("description","text query2") - query.Merge (q2) + query := db.Query ("items"). + Match ("description","text query1") + q2 := db.Query ("another_items"). + Match ("description","text query2") + query.Merge (q2) iterator = query.Exec () - // Check the error - if err := iterator.Error(); err != nil { - panic(err) - } - // Iterate over results - for iterator.Next() { - // Get the next document and cast it to a pointer - switch elem := iterator.Object().(type) { - case Item: - fmt.Printf ("%v,rank=%d\n",*elem,iterator.Rank()) - case AnotherItem: - fmt.Printf ("%v,rank=%d\n",*elem,iterator.Rank()) - } - } + // Check the error + if err := iterator.Error(); err != nil { + panic(err) + } + // Iterate over results + for iterator.Next() { + // Get the next document and cast it to a pointer + switch elem := iterator.Object().(type) { + case Item: + fmt.Printf ("%v,rank=%d\n",*elem,iterator.Rank()) + case AnotherItem: + fmt.Printf ("%v,rank=%d\n",*elem,iterator.Rank()) + } + } ``` ## Using select functions It is possible to use select functions to process result data. @@ -319,20 +320,20 @@ Several parameters of full text search engine can be configured from application ```go ... ftconfig := reindexer.DefaultFtFastConfig() - // Setup configuration - ftconfig.LogLevel = reindexer.TRACE - // Setup another parameters - // ... - // Create index definition - indexDef := reindexer.IndexDef { - Name: "description", - JSONPaths: []string{"description"}, - IndexType: "text", - FieldType: "string", - Config: ftconfig, - } - // Add index with configuration - return db.AddIndex ("items",indexDef) + // Setup configuration + ftconfig.LogLevel = reindexer.TRACE + // Setup another parameters + // ... + // Create index definition + indexDef := reindexer.IndexDef { + Name: "description", + JSONPaths: []string{"description"}, + IndexType: "text", + FieldType: "string", + Config: ftconfig, + } + // Add index with configuration + return db.AddIndex ("items",indexDef) ``` @@ -361,7 +362,7 @@ Several parameters of full text search engine can be configured from application | | Stemmers | []string | List of stemmers to use | "en","ru" | | | EnableTranslit | bool | Enable russian translit variants processing. e.g. term "luntik" will match word "лунтик" | true | | | EnableKbLayout | bool | Enable wrong keyboard layout variants processing. e.g. term "keynbr" will match word "лунтик" | true | -| | StopWords | []string | List of stop words. Words from this list will be ignored in documents and queries | | +| | StopWords | []struct | List of objects of stopwords. Words from this list will be ignored when building indexes, but may be used in fulltext queries (such as 'word*', 'word~' etc) and produce non-empty search results. [More...](#stopwords-details) | | | | SumRanksByFieldsRatio | float | Ratio of summation of ranks of match one term in several fields | 0.0 | | | LogLevel | int | Log level of full text search engine | 0 | | | FieldsCfg | []struct | Configs for certain fields. Overlaps parameters from main config. Contains parameters: FieldName, Bm25Boost, Bm25Weight, TermLenBoost, TermLenWeight, PositionBoost, PositionWeight. | empty | @@ -372,6 +373,42 @@ Several parameters of full text search engine can be configured from application | | Optimization | string | Optimize the index by 'memory' or by 'cpu' | "memory" | | | FtBaseRanking | struct | Relevance of the word in different forms | | + +### Stopwords details +The list item can be either a string or a structure containing a string (the stopword) and a bool attribute (`is_morpheme`) indicating whether the stopword can be part of a word that can be shown in query-results. +If the stopword is set as a string, then the `is_morpheme` attribute is `false` by default and following entries are equivalent: +```json +"StopWords":[ + { + "word": "some_word", + "is_morpheme": false + }, + ///... +] +``` +, +```json +"StopWords":[ + "some_word", + ///... +] +``` + +#### Example: +If the list of stopwords looks like this: +```json +"StopWords":[ + { + "word": "under", + "is_morpheme": true + }, + ///... +] +``` +and there are pair of documents containing this word: `{"...under the roof ..."}, {"... to understand and forgive..."}`. Then for the query 'under*' we will get as a result only document `{"... to understand and forgive..."}` and for the query 'under' we will get nothing as a result. + +If the "StopWords" section is not specified in the config, then the [default](./cpp_src/core/ft/stopwords/stop_en.cc) stopwords list will be used, and if it is explicitly specified empty, it means that there are no stopwords. + ### Detailed typos config FtTyposDetailedConfig: config for more precise typos algorithm tuning. diff --git a/iterator.go b/iterator.go index 347cad44e..56bbd1a80 100644 --- a/iterator.go +++ b/iterator.go @@ -15,11 +15,11 @@ import ( type ExplainSelector struct { // Field or index name - Field string `json:"field"` + Field string `json:"field,omitempty"` // Field type enum: indexed, non-indexed FieldType string `json:"field_type,omitempty"` // Method, used to process condition - Method string `json:"method"` + Method string `json:"method,omitempty"` // Number of uniq keys, processed by this selector (may be incorrect, in case of internal query optimization/caching Keys int `json:"keys"` // Count of comparators used, for this selector @@ -30,6 +30,8 @@ type ExplainSelector struct { Matched int `json:"matched"` // Count of scanned documents by this selector Items int `json:"items"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` // Preselect in joined namespace execution explainings ExplainPreselect *ExplainResults `json:"explain_preselect,omitempty"` // One of selects in joined namespace execution explainings @@ -37,6 +39,13 @@ type ExplainSelector struct { Selectors []ExplainSelector `json:"selectors,omitempty"` } +type ExplainSubQuery struct { + Namespace string `json:"namespace"` + Explain ExplainResults `json:"explain"` + Keys int `json:"keys,omitempty"` + Field string `json:"field,omitempty"` +} + // ExplainResults presents query plan type ExplainResults struct { // Total query execution time @@ -61,6 +70,8 @@ type ExplainResults struct { Selectors []ExplainSelector `json:"selectors"` // Explaining attempts to inject Join queries ON-conditions into the Main Query WHERE clause OnConditionsInjections []ExplainJoinOnInjections `json:"on_conditions_injections,omitempty"` + // Explaining of subqueries' preselect + SubQueriesExplains []ExplainSubQuery `json:"subqueries,omitempty"` } // Describes the process of a single JOIN-query ON-conditions injection into the Where clause of a main query diff --git a/query.go b/query.go index 88c8eacae..985bdfeb8 100644 --- a/query.go +++ b/query.go @@ -555,20 +555,24 @@ func (q *Query) DWithin(index string, point Point, distance float64) *Query { return q } -func (q *Query) AggregateSum(field string) { +func (q *Query) AggregateSum(field string) *Query { q.ser.PutVarCUInt(queryAggregation).PutVarCUInt(AggSum).PutVarCUInt(1).PutVString(field) + return q } -func (q *Query) AggregateAvg(field string) { +func (q *Query) AggregateAvg(field string) *Query { q.ser.PutVarCUInt(queryAggregation).PutVarCUInt(AggAvg).PutVarCUInt(1).PutVString(field) + return q } -func (q *Query) AggregateMin(field string) { +func (q *Query) AggregateMin(field string) *Query { q.ser.PutVarCUInt(queryAggregation).PutVarCUInt(AggMin).PutVarCUInt(1).PutVString(field) + return q } -func (q *Query) AggregateMax(field string) { +func (q *Query) AggregateMax(field string) *Query { q.ser.PutVarCUInt(queryAggregation).PutVarCUInt(AggMax).PutVarCUInt(1).PutVString(field) + return q } type AggregateFacetRequest struct { diff --git a/readme.md b/readme.md index e2b9159bd..d71b2a5e6 100644 --- a/readme.md +++ b/readme.md @@ -1616,3 +1616,4 @@ Landing: https://reindexer.io/ Packages repo: https://repo.reindexer.io/ More documentation (RU): https://reindexer.io/reindexer-docs/ + diff --git a/test/config_test.go b/test/config_test.go index abe83fc83..625600b56 100644 --- a/test/config_test.go +++ b/test/config_test.go @@ -10,6 +10,18 @@ import ( "github.com/restream/reindexer/v4" ) +type FtConfCheck struct { + ID int `reindex:"id,,pk"` +} + +const ( + ftCfgNsName = "ft_cfg_check" +) + +func init() { + tnamespaces[ftCfgNsName] = FtConfCheck{} +} + func TestSetDefaultQueryDebug(t *testing.T) { t.Run("set debug level to exist ns config", func(t *testing.T) { ns := "ns_with_config" @@ -89,3 +101,78 @@ func TestSetDefaultQueryDebug(t *testing.T) { assert.True(t, found) }) } + +func TestFtConfigCompablitity(t *testing.T) { + config := reindexer.DefaultFtFastConfig() + + stopWordsStrs := append(make([]interface{}, 0), "под", "на", "из") + stopWordsObjs := append(make([]interface{}, 0), + reindexer.StopWord{ + Word: "пред", + IsMorpheme: true, + }, reindexer.StopWord{ + Word: "над", + IsMorpheme: true, + }, reindexer.StopWord{ + Word: "за", + IsMorpheme: false, + }) + + stopWordsMix := append(make([]interface{}, 0), + "под", + reindexer.StopWord{ + Word: "пред", + IsMorpheme: true, + }, + reindexer.StopWord{ + Word: "за", + IsMorpheme: false, + }, + "на", + reindexer.StopWord{ + Word: "над", + IsMorpheme: true, + }, + "из") + + checkFtConfigAfterAddIndex := func(index string) { + err := DB.AddIndex(ftCfgNsName, reindexer.IndexDef{ + Name: index, + JSONPaths: []string{index}, + Config: config, + IndexType: "text", + FieldType: "string", + }) + assert.NoError(t, err) + + item, err := DBD.Query(reindexer.NamespacesNamespaceName).Where("name", reindexer.EQ, ftCfgNsName).Exec().FetchOne() + assert.NoError(t, err) + + indexes := item.(*reindexer.NamespaceDescription).Indexes + ftConf := indexes[len(indexes)-1].Config.(map[string]interface{}) + + actual := ftConf["stop_words"].([]interface{}) + assert.Equal(t, len(actual), len(config.StopWords)) + + for idx, wordI := range config.StopWords { + switch wordI.(type) { + case string: + assert.Equal(t, wordI, actual[idx]) + case reindexer.StopWord: + word := wordI.(reindexer.StopWord) + assert.Equal(t, word.Word, actual[idx].(map[string]interface{})["word"]) + assert.Equal(t, word.IsMorpheme, actual[idx].(map[string]interface{})["is_morpheme"]) + } + } + } + + config.StopWords = stopWordsStrs + checkFtConfigAfterAddIndex("textStrs") + + config.StopWords = stopWordsObjs + checkFtConfigAfterAddIndex("textObjs") + + config.StopWords = stopWordsMix + checkFtConfigAfterAddIndex("textMix") + +} diff --git a/test/dsl_test.go b/test/dsl_test.go index 0392431b6..07d193bbc 100644 --- a/test/dsl_test.go +++ b/test/dsl_test.go @@ -126,6 +126,7 @@ func execDSLTwice(t *testing.T, testF func(*testing.T, *reindexer.Query), jsonDS q, err := DBD.QueryFrom(dslQ) require.NoError(t, err) require.NotNil(t, q) + q.Debug(reindexer.TRACE) testF(t, q) marshaledJSON, err = json.Marshal(dslQ) require.NoError(t, err) @@ -138,6 +139,7 @@ func execDSLTwice(t *testing.T, testF func(*testing.T, *reindexer.Query), jsonDS q, err := DBD.QueryFrom(dslQ) require.NoError(t, err) require.NotNil(t, q) + q.Debug(reindexer.TRACE) testF(t, q) } } diff --git a/test/ft/fx.go b/test/ft/fx.go index d4e5bcce0..4e909cf96 100644 --- a/test/ft/fx.go +++ b/test/ft/fx.go @@ -38,7 +38,7 @@ func createReindexDbInstance(rx *reindexer.Reindexer, namespace string, indexTyp if indexType == "fuzzytext" { // Disable non exact searchers, disable stop word dictionat cfg := reindexer.DefaultFtFuzzyConfig() - cfg.StopWords = []string{} + cfg.StopWords = make([]interface{}, 0) cfg.Stemmers = []string{} cfg.EnableKbLayout = false cfg.EnableTranslit = false @@ -48,7 +48,7 @@ func createReindexDbInstance(rx *reindexer.Reindexer, namespace string, indexTyp config = cfg } else { cfg := reindexer.DefaultFtFastConfig() - cfg.StopWords = []string{} + cfg.StopWords = make([]interface{}, 0) cfg.Stemmers = []string{} cfg.EnableKbLayout = false cfg.EnableTranslit = false diff --git a/test/join_test.go b/test/join_test.go index 3a7220a9e..8f4fca7fd 100644 --- a/test/join_test.go +++ b/test/join_test.go @@ -471,6 +471,7 @@ type expectedExplain struct { Field string FieldType string Method string + Description string Keys int Comparators int Matched int @@ -499,6 +500,13 @@ type expectedExplainJoinOnInjections struct { Conditions []expectedExplainConditionInjection } +type expectedExplainSubQuery struct { + Namespace string + Keys int + Field string + Selectors []expectedExplain +} + func checkExplain(t *testing.T, res []reindexer.ExplainSelector, expected []expectedExplain, fieldName string) { require.Equal(t, len(expected), len(res)) for i := 0; i < len(expected); i++ { @@ -514,6 +522,7 @@ func checkExplain(t *testing.T, res []reindexer.ExplainSelector, expected []expe assert.Equalf(t, expected[i].Matched, res[i].Matched, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Keys, res[i].Keys, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Comparators, res[i].Comparators, fieldName+expected[i].Field) + assert.Equalf(t, expected[i].Description, res[i].Description, fieldName+expected[i].Field) if len(expected[i].Preselect) == 0 { assert.Nil(t, res[i].ExplainPreselect, fieldName+expected[i].Field) } else { @@ -561,6 +570,16 @@ func checkExplainJoinOnInjections(t *testing.T, res []reindexer.ExplainJoinOnInj } } +func checkExplainSubqueries(t *testing.T, res []reindexer.ExplainSubQuery, expected []expectedExplainSubQuery) { + require.Equal(t, len(expected), len(res)) + for i := 0; i < len(expected); i++ { + assert.Equal(t, expected[i].Namespace, res[i].Namespace) + assert.Equal(t, expected[i].Field, res[i].Field) + assert.Equal(t, expected[i].Keys, res[i].Keys) + checkExplain(t, res[i].Explain.Selectors, expected[i].Selectors, "") + } +} + func TestExplainJoin(t *testing.T) { nsMain := "test_explain_main" nsJoined := "test_explain_joined" diff --git a/test/queries_test.go b/test/queries_test.go index d6bf7e8df..e38a049cd 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -241,6 +241,8 @@ func init() { tnamespaces["test_items_eqaul_position"] = TestItemEqualPosition{} tnamespaces["test_items_strict"] = TestItem{} tnamespaces["test_items_strict_joined"] = TestJoinItem{} + + tnamespaces["test_items_explain"] = TestItemSimple{} } func FillTestItemsForNot() { @@ -2192,3 +2194,183 @@ func TestQrIdleTimeout(t *testing.T) { } }) } + +func TestQueryExplain(t *testing.T) { + t.Parallel() + + ns := "test_items_explain" + + tx := newTestTx(DB, ns) + for i := 0; i < 5; i++ { + tx.Upsert(TestItemSimple{ID: i, Year: i, Name: randString()}) + } + tx.MustCommit() + + t.Run("Subquery explain check (WhereQuery)", func(t *testing.T) { + q := DB.Query(ns).Explain(). + WhereQuery(t, DB.Query(ns).Select("id").Where("year", reindexer.EQ, 1), reindexer.GE, 0) + it := q.MustExec(t) + defer it.Close() + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "-scan", + Method: "scan", + Keys: 0, + Comparators: 0, + Matched: 5, + }, + { + Description: "always true", + Keys: 0, + Comparators: 0, + Matched: 0, + }, + }, "") + checkExplainSubqueries(t, explainRes.SubQueriesExplains, []expectedExplainSubQuery{ + { + Namespace: ns, + Selectors: []expectedExplain{ + { + Field: "year", + FieldType: "indexed", + Method: "index", + Keys: 1, + Comparators: 0, + Matched: 1, + }, + { + Field: "id", + FieldType: "indexed", + Method: "scan", + Keys: 0, + Comparators: 1, + Matched: 1, + }, + }, + }, + }) + }) + + t.Run("Subquery explain check (Where)", func(t *testing.T) { + q := DB.Query(ns).Explain(). + Where("id", reindexer.EQ, DB.Query(ns).Select("id").Where("year", reindexer.EQ, 3)) + it := q.MustExec(t) + defer it.Close() + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Comparators: 0, + Matched: 1, + }, + }, "") + checkExplainSubqueries(t, explainRes.SubQueriesExplains, []expectedExplainSubQuery{ + { + Namespace: ns, + Field: "id", + Selectors: []expectedExplain{ + { + Field: "year", + FieldType: "indexed", + Method: "index", + Keys: 1, + Comparators: 0, + Matched: 1, + }, + }, + }, + }) + }) + + t.Run("Subquery explain check (Where + WhereQuery)", func(t *testing.T) { + q := DB.Query(ns).Explain(). + Where("id", reindexer.SET, DB.Query(ns).Select("id").Where("year", reindexer.SET, []int{1, 2})). + WhereQuery(t, DB.Query(ns).Select("id").Where("year", reindexer.EQ, 5), reindexer.LE, 10) + it := q.MustExec(t) + defer it.Close() + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "-scan", + Method: "scan", + Keys: 0, + Comparators: 0, + Matched: 1, + }, + { + Description: "always false", + Keys: 0, + Comparators: 0, + Matched: 0, + }, + { + Field: "id", + FieldType: "indexed", + Method: "scan", + Keys: 0, + Comparators: 1, + Matched: 0, + }, + }, "") + checkExplainSubqueries(t, explainRes.SubQueriesExplains, []expectedExplainSubQuery{ + { + Namespace: ns, + Field: "id", + Selectors: []expectedExplain{ + { + Field: "-scan", + Method: "scan", + Keys: 0, + Comparators: 0, + Matched: 5, + }, + { + Field: "year", + FieldType: "indexed", + Method: "scan", + Keys: 0, + Comparators: 1, + Matched: 2, + }, + }, + }, + { + Namespace: ns, + Selectors: []expectedExplain{ + { + Field: "year", + FieldType: "indexed", + Method: "index", + Keys: 0, + Comparators: 0, + Matched: 0, + }, + { + Field: "id", + FieldType: "indexed", + Method: "scan", + Keys: 0, + Comparators: 1, + Matched: 0, + }, + }, + }, + }) + }) +} diff --git a/test/reindexer_bench_test.go b/test/reindexer_bench_test.go index 86dbcd9d8..052b45b59 100644 --- a/test/reindexer_bench_test.go +++ b/test/reindexer_bench_test.go @@ -409,6 +409,31 @@ func Benchmark2CondQueryTotal(b *testing.B) { } } +func BenchmarkSubQueryEq(b *testing.B) { + for i := 0; i < b.N; i++ { + prices := priceIds[rand.Int()%len(priceIds)] + q := DBD.Query("test_items_bench").Where("price_id", reindexer.EQ, DBD.Query("test_join_items").Select("id").WhereInt32("id", reindexer.EQ, prices[rand.Int()%len(prices)])).Limit(20) + q.MustExec().FetchAll() + } +} + +func BenchmarkSubQuerySet(b *testing.B) { + for i := 0; i < b.N; i++ { + prices := priceIds[rand.Int()%len(priceIds)] + rangeMin := prices[rand.Int()%len(prices)] + q := DBD.Query("test_items_bench").Where("price_id", reindexer.SET, DBD.Query("test_join_items").Select("id").WhereInt32("id", reindexer.RANGE, rangeMin, rangeMin + 500)).Limit(20) + q.MustExec().FetchAll() + } +} + +func BenchmarkSubQueryAggregate(b *testing.B) { + for i := 0; i < b.N; i++ { + prices := priceIds[rand.Int()%len(priceIds)] + q := DBD.Query("test_items_bench").Where("price_id", reindexer.LT, DBD.Query("test_join_items").AggregateAvg("id").WhereInt32("id", reindexer.SET, prices...).Limit(500)).Limit(20) + q.MustExec().FetchAll() + } +} + func Benchmark2CondQueryLeftJoin(b *testing.B) { ctx := &TestJoinCtx{} for i := 0; i < b.N; i++ {