From deb68415efac51c183db38d2e2110cf63571e37f Mon Sep 17 00:00:00 2001 From: yaphet <4414314+darionyaphet@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:41:12 +0800 Subject: [PATCH] fix meta upgrade for multi instance (#3734) --- src/daemons/MetaDaemonInit.cpp | 20 +++-- src/meta/MetaVersionMan.cpp | 114 ++++++++++++++------------- src/meta/MetaVersionMan.h | 11 +-- src/meta/upgrade/MetaDataUpgrade.cpp | 4 +- src/meta/upgrade/MetaDataUpgrade.h | 52 ++---------- 5 files changed, 88 insertions(+), 113 deletions(-) diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index 09640d5dc58..4f004933682 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -94,6 +94,13 @@ std::unique_ptr initKV(std::vector p return nullptr; } + auto engineRet = kvstore->part(nebula::kDefaultSpaceId, nebula::kDefaultPartId); + if (!nebula::ok(engineRet)) { + LOG(ERROR) << "Get nebula store engine failed"; + return nullptr; + } + + auto engine = nebula::value(engineRet)->engine(); LOG(INFO) << "Waiting for the leader elected..."; nebula::HostAddr leader; while (true) { @@ -138,22 +145,21 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(kvstore.get()); + auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(engine); if (!ret.ok()) { - LOG(ERROR) << ret; + LOG(ERROR) << "Update meta from V1 to V2 failed " << ret; return nullptr; } - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V2); + nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V2); } else if (version == nebula::meta::MetaVersion::V2) { - LOG(INFO) << "version 3"; - auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(kvstore.get()); + auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine); if (!ret.ok()) { - LOG(ERROR) << ret; + LOG(ERROR) << "Update meta from V2 to V3 failed " << ret; return nullptr; } - nebula::meta::MetaVersionMan::setMetaVersionToKV(kvstore.get(), nebula::meta::MetaVersion::V3); + nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V3); } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index 292878899ad..82bc7a8c842 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -5,6 +5,7 @@ #include "meta/MetaVersionMan.h" +#include "common/fs/FileUtils.h" #include "meta/ActiveHostsMan.h" #include "meta/processors/job/JobDescription.h" #include "meta/processors/job/JobUtils.h" @@ -51,78 +52,86 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { } // static -bool MetaVersionMan::setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version) { - CHECK_NOTNULL(kv); - std::vector data; - data.emplace_back(kMetaVersionKey, - std::string(reinterpret_cast(&version), sizeof(MetaVersion))); - bool ret = true; - folly::Baton baton; - kv->asyncMultiPut( - kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) { - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Put failed, error: " << static_cast(code); - ret = false; - } else { - LOG(INFO) << "Write meta version 3 succeeds"; - } - baton.post(); - }); - baton.wait(); - return ret; +bool MetaVersionMan::setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version) { + CHECK_NOTNULL(engine); + std::string versionValue = + std::string(reinterpret_cast(&version), sizeof(MetaVersion)); + auto code = engine->put(kMetaVersionKey, std::move(versionValue)); + return code == nebula::cpp2::ErrorCode::SUCCEEDED; } // static -Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVStore* kv) { - CHECK_NOTNULL(kv); +Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVEngine* engine) { + CHECK_NOTNULL(engine); auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()); - auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); - if (meteRet.isLeftType()) { + + std::string path = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot); + if (!fs::FileUtils::exist(path) && !fs::FileUtils::makeDir(path)) { + LOG(ERROR) << "Make checkpoint dir: " << path << " failed"; + return Status::Error("Create snapshot file failed"); + } + + std::string dataPath = folly::sformat("{}/data", path); + auto code = engine->createCheckpoint(dataPath); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgradeV1ToV2(kv); + + auto status = doUpgradeV1ToV2(engine); if (!status.ok()) { // rollback by snapshot return status; } + // delete snapshot file - auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); - if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + auto checkpointPath = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot); + + if (fs::FileUtils::exist(checkpointPath) && !fs::FileUtils::remove(checkpointPath.data(), true)) { LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; } return Status::OK(); } -Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVStore* kv) { - CHECK_NOTNULL(kv); +Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) { + CHECK_NOTNULL(engine); auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()); - auto meteRet = kv->createCheckpoint(kDefaultSpaceId, snapshot); - if (meteRet.isLeftType()) { + + std::string path = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot); + if (!fs::FileUtils::exist(path) && !fs::FileUtils::makeDir(path)) { + LOG(ERROR) << "Make checkpoint dir: " << path << " failed"; + return Status::Error("Create snapshot file failed"); + } + + std::string dataPath = folly::sformat("{}/data", path); + auto code = engine->createCheckpoint(dataPath); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Create snapshot failed: " << snapshot; return Status::Error("Create snapshot failed"); } - auto status = doUpgradeV2ToV3(kv); + + auto status = doUpgradeV2ToV3(engine); if (!status.ok()) { // rollback by snapshot return status; } + // delete snapshot file - auto dmRet = kv->dropCheckpoint(kDefaultSpaceId, snapshot); - if (dmRet != nebula::cpp2::ErrorCode::SUCCEEDED) { + auto checkpointPath = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot); + if (fs::FileUtils::exist(checkpointPath) && !fs::FileUtils::remove(checkpointPath.data(), true)) { LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually"; } return Status::OK(); } // static -Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { - MetaDataUpgrade upgrader(kv); +Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVEngine* engine) { + MetaDataUpgrade upgrader(engine); { // kSpacesTable auto prefix = nebula::meta::v1::kSpacesTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -143,7 +152,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kPartsTable auto prefix = nebula::meta::v1::kPartsTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -164,7 +173,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kHostsTable auto prefix = nebula::meta::v1::kHostsTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -185,7 +194,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kLeadersTable auto prefix = nebula::meta::v1::kLeadersTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -206,7 +215,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kTagsTable auto prefix = nebula::meta::v1::kTagsTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -227,7 +236,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kEdgesTable auto prefix = nebula::meta::v1::kEdgesTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -248,7 +257,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kIndexesTable auto prefix = nebula::meta::v1::kIndexesTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -269,7 +278,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kConfigsTable auto prefix = nebula::meta::v1::kConfigsTable; std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -290,7 +299,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { // kJob auto prefix = JobUtil::jobPrefix(); std::unique_ptr iter; - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -324,7 +333,7 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { nebula::meta::v1::kJobArchive}); std::unique_ptr iter; for (auto& prefix : prefixes) { - auto ret = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto ret = engine->prefix(prefix, &iter); if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) { Status status = Status::OK(); while (iter->valid()) { @@ -351,22 +360,22 @@ Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVStore* kv) { } } } - if (!setMetaVersionToKV(kv, MetaVersion::V2)) { + if (!setMetaVersionToKV(engine, MetaVersion::V2)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); } } -Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { - MetaDataUpgrade upgrader(kv); +Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVEngine* engine) { + MetaDataUpgrade upgrader(engine); // Step 1: Upgrade HeartBeat into machine list { // collect all hosts association with zone std::vector zoneHosts; const auto& zonePrefix = MetaKeyUtils::zonePrefix(); std::unique_ptr zoneIter; - auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &zoneIter); + auto code = engine->prefix(zonePrefix, &zoneIter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get active hosts failed"; return Status::Error("Get hosts failed"); @@ -382,7 +391,7 @@ Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { const auto& prefix = MetaKeyUtils::hostPrefix(); std::unique_ptr iter; - code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + code = engine->prefix(prefix, &iter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get active hosts failed"; return Status::Error("Get hosts failed"); @@ -420,7 +429,7 @@ Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { { const auto& prefix = MetaKeyUtils::spacePrefix(); std::unique_ptr iter; - auto code = kv->prefix(kDefaultSpaceId, kDefaultPartId, prefix, &iter); + auto code = engine->prefix(prefix, &iter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(ERROR) << "Get spaces failed"; return Status::Error("Get spaces failed"); @@ -430,7 +439,6 @@ Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { if (FLAGS_print_info) { upgrader.printSpacesV2(iter->val()); } - auto spaceProperties = meta::v2::MetaServiceUtilsV2::parseSpace(iter->val()); auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); if (!status.ok()) { LOG(ERROR) << status; @@ -439,7 +447,7 @@ Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVStore* kv) { iter->next(); } } - if (!setMetaVersionToKV(kv, MetaVersion::V3)) { + if (!setMetaVersionToKV(engine, MetaVersion::V3)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index c6e31b8d01d..07d7331f906 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -8,6 +8,7 @@ #include "common/base/Base.h" #include "common/utils/MetaKeyUtils.h" +#include "kvstore/KVEngine.h" #include "kvstore/KVStore.h" namespace nebula { @@ -29,18 +30,18 @@ class MetaVersionMan final { static MetaVersion getMetaVersionFromKV(kvstore::KVStore* kv); - static bool setMetaVersionToKV(kvstore::KVStore* kv, MetaVersion version); + static bool setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version); - static Status updateMetaV1ToV2(kvstore::KVStore* kv); + static Status updateMetaV1ToV2(kvstore::KVEngine* engine); - static Status updateMetaV2ToV3(kvstore::KVStore* kv); + static Status updateMetaV2ToV3(kvstore::KVEngine* engine); private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgradeV1ToV2(kvstore::KVStore* kv); + static Status doUpgradeV1ToV2(kvstore::KVEngine* engine); - static Status doUpgradeV2ToV3(kvstore::KVStore* kv); + static Status doUpgradeV2ToV3(kvstore::KVEngine* engine); }; } // namespace meta diff --git a/src/meta/upgrade/MetaDataUpgrade.cpp b/src/meta/upgrade/MetaDataUpgrade.cpp index e057e8d6f85..e25e2ffb236 100644 --- a/src/meta/upgrade/MetaDataUpgrade.cpp +++ b/src/meta/upgrade/MetaDataUpgrade.cpp @@ -97,7 +97,7 @@ Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, auto groupName = *oldProps.group_name_ref(); auto groupKey = meta::v2::MetaServiceUtilsV2::groupKey(groupName); std::string zoneValue; - auto code = kv_->get(kDefaultSpaceId, kDefaultPartId, std::move(groupKey), &zoneValue); + auto code = engine_->get(std::move(groupKey), &zoneValue); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { return Status::Error("Get Group Failed"); } @@ -107,7 +107,7 @@ Status MetaDataUpgrade::rewriteSpacesV2ToV3(const folly::StringPiece &key, } else { const auto &zonePrefix = MetaKeyUtils::zonePrefix(); std::unique_ptr iter; - auto code = kv_->prefix(kDefaultSpaceId, kDefaultPartId, zonePrefix, &iter); + auto code = engine_->prefix(zonePrefix, &iter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { return Status::Error("Get Zones Failed"); } diff --git a/src/meta/upgrade/MetaDataUpgrade.h b/src/meta/upgrade/MetaDataUpgrade.h index bd4ffea08a1..e62cac8ba47 100644 --- a/src/meta/upgrade/MetaDataUpgrade.h +++ b/src/meta/upgrade/MetaDataUpgrade.h @@ -5,12 +5,11 @@ #pragma once -#include - #include "common/base/Base.h" #include "common/base/Status.h" #include "common/utils/MetaKeyUtils.h" #include "interface/gen-cpp2/meta_types.h" +#include "kvstore/KVEngine.h" #include "kvstore/KVStore.h" #include "meta/processors/Common.h" #include "meta/upgrade/v1/gen-cpp2/meta_types.h" @@ -21,7 +20,7 @@ namespace meta { class MetaDataUpgrade final { public: - explicit MetaDataUpgrade(kvstore::KVStore *kv) : kv_(kv) {} + explicit MetaDataUpgrade(kvstore::KVEngine *engine) : engine_(engine) {} ~MetaDataUpgrade() = default; @@ -52,21 +51,7 @@ class MetaDataUpgrade final { private: Status put(const folly::StringPiece &key, const folly::StringPiece &val) { - std::vector data; - data.emplace_back(key.str(), val.str()); - folly::Baton baton; - auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; - kv_->asyncMultiPut(kDefaultSpaceId, - kDefaultPartId, - std::move(data), - [&ret, &baton](nebula::cpp2::ErrorCode code) { - if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { - ret = code; - LOG(INFO) << "Put data error on meta server"; - } - baton.post(); - }); - baton.wait(); + auto ret = engine_->put(key.str(), val.str()); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return Status::Error("Put data failed"); } @@ -74,19 +59,7 @@ class MetaDataUpgrade final { } Status put(std::vector data) { - folly::Baton baton; - auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; - kv_->asyncMultiPut(kDefaultSpaceId, - kDefaultPartId, - std::move(data), - [&ret, &baton](nebula::cpp2::ErrorCode code) { - if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { - ret = code; - LOG(INFO) << "Put data error on meta server"; - } - baton.post(); - }); - baton.wait(); + auto ret = engine_->multiPut(data); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return Status::Error("Put data failed"); } @@ -94,20 +67,7 @@ class MetaDataUpgrade final { } Status remove(const folly::StringPiece &key) { - std::vector keys{key.str()}; - folly::Baton baton; - auto ret = nebula::cpp2::ErrorCode::SUCCEEDED; - kv_->asyncMultiRemove(kDefaultSpaceId, - kDefaultPartId, - std::move(keys), - [&ret, &baton](nebula::cpp2::ErrorCode code) { - if (nebula::cpp2::ErrorCode::SUCCEEDED != code) { - ret = code; - LOG(INFO) << "Remove data error on meta server"; - } - baton.post(); - }); - baton.wait(); + auto ret = engine_->remove(key.str()); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { return Status::Error("Remove data failed"); } @@ -125,7 +85,7 @@ class MetaDataUpgrade final { nebula::meta::cpp2::GeoShape convertToGeoShape(nebula::meta::v2::cpp2::GeoShape shape); private: - kvstore::KVStore *kv_ = nullptr; + kvstore::KVEngine *engine_ = nullptr; }; } // namespace meta