Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

parse diskPartsKey and add unit test #3416

Merged
merged 7 commits into from
Dec 8, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions src/common/utils/MetaKeyUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1152,9 +1152,30 @@ GraphSpaceID MetaKeyUtils::parseLocalIdSpace(folly::StringPiece rawData) {
return *reinterpret_cast<const GraphSpaceID*>(rawData.data() + offset);
}

GraphSpaceID MetaKeyUtils::parseDiskPartsSpace(folly::StringPiece rawData) {
HostAddr MetaKeyUtils::parseDiskPartsHost(const folly::StringPiece& rawData) {
Nivras marked this conversation as resolved.
Show resolved Hide resolved
auto offset = kDiskPartsTable.size();
Nivras marked this conversation as resolved.
Show resolved Hide resolved
return *reinterpret_cast<const GraphSpaceID*>(rawData.data() + offset);
size_t len = *reinterpret_cast<const size_t*>(rawData.begin() + offset);
std::string hostStr;
hostStr.reserve(sizeof(size_t) + len + sizeof(Port));
hostStr.append(rawData.begin() + offset, sizeof(size_t) + len + sizeof(Port));
return deserializeHostAddr(hostStr);
}

GraphSpaceID MetaKeyUtils::parseDiskPartsSpace(const folly::StringPiece& rawData) {
auto offset = kDiskPartsTable.size();
size_t len = *reinterpret_cast<const size_t*>(rawData.begin() + offset);
offset += sizeof(size_t) + len + sizeof(Port);
return *reinterpret_cast<const GraphSpaceID*>(rawData.begin() + offset);
}

std::string MetaKeyUtils::parseDiskPartsPath(const folly::StringPiece& rawData) {
auto offset = kDiskPartsTable.size();
size_t len = *reinterpret_cast<const size_t*>(rawData.begin() + offset);
offset += sizeof(size_t) + len + sizeof(Port) + sizeof(GraphSpaceID);
std::string path;
path.reserve(rawData.size() - offset);
path.append(rawData.begin() + offset, rawData.size() - offset);
return path;
}

std::string MetaKeyUtils::diskPartsPrefix() { return kDiskPartsTable; }
Expand All @@ -1176,7 +1197,9 @@ std::string MetaKeyUtils::diskPartsPrefix(HostAddr addr, GraphSpaceID spaceId) {
return key;
}

std::string MetaKeyUtils::diskPartsKey(HostAddr addr, GraphSpaceID spaceId, std::string path) {
std::string MetaKeyUtils::diskPartsKey(HostAddr addr,
GraphSpaceID spaceId,
const std::string& path) {
std::string key;
std::string prefix = diskPartsPrefix(addr, spaceId);
key.reserve(prefix.size() + path.size());
Expand Down
8 changes: 6 additions & 2 deletions src/common/utils/MetaKeyUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -376,15 +376,19 @@ class MetaKeyUtils final {

static std::unordered_map<std::string, std::pair<std::string, bool>> getSystemTableMaps();

static GraphSpaceID parseDiskPartsSpace(folly::StringPiece rawData);
static GraphSpaceID parseDiskPartsSpace(const folly::StringPiece& rawData);

static HostAddr parseDiskPartsHost(const folly::StringPiece& rawData);

static std::string parseDiskPartsPath(const folly::StringPiece& rawData);

static std::string diskPartsPrefix();

static std::string diskPartsPrefix(HostAddr addr);

static std::string diskPartsPrefix(HostAddr addr, GraphSpaceID spaceId);

static std::string diskPartsKey(HostAddr addr, GraphSpaceID spaceId, std::string path);
static std::string diskPartsKey(HostAddr addr, GraphSpaceID spaceId, const std::string& path);

static std::string diskPartsVal(const meta::cpp2::PartitionList& partList);

Expand Down
11 changes: 11 additions & 0 deletions src/common/utils/test/MetaKeyUtilsTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,17 @@ TEST(MetaKeyUtilsTest, ZoneTest) {
ASSERT_EQ(nodes, MetaKeyUtils::parseZoneHosts(zoneValue));
}

TEST(MetaKeyUtilsTest, DiskPathsTest) {
HostAddr addr{"192.168.0.1", 1234};
GraphSpaceID spaceId = 1;
std::string path = "/data/storage/test_part1";

auto diskPartsKey = MetaKeyUtils::diskPartsKey(addr, spaceId, path);
ASSERT_EQ(addr, MetaKeyUtils::parseDiskPartsHost(diskPartsKey));
ASSERT_EQ(spaceId, MetaKeyUtils::parseDiskPartsSpace(diskPartsKey));
ASSERT_EQ(path, MetaKeyUtils::parseDiskPartsPath(diskPartsKey));
}

} // namespace nebula

int main(int argc, char** argv) {
Expand Down
33 changes: 33 additions & 0 deletions src/kvstore/test/DiskManagerTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,39 @@ TEST(DiskManagerTest, WalNoSpaceTest) {
}
}

TEST(DiskManagerTest, GetDiskPartsTest) {
GraphSpaceID spaceId = 1;
fs::TempDir disk1("/tmp/get_disk_part_test.XXXXXX");
auto path1 = folly::stringPrintf("%s/nebula/%d", disk1.path(), spaceId);
boost::filesystem::create_directories(path1);
fs::TempDir disk2("/tmp/get_disk_part_test.XXXXXX");
auto path2 = folly::stringPrintf("%s/nebula/%d", disk2.path(), spaceId);
boost::filesystem::create_directories(path2);
GraphSpaceID spaceId2 = 2;
fs::TempDir disk3("/tmp/get_disk_part_test.XXXXXX");
auto path3 = folly::stringPrintf("%s/nebula/%d", disk3.path(), spaceId2);
boost::filesystem::create_directories(path3);

std::vector<std::string> dataPaths = {disk1.path(), disk2.path(), disk3.path()};
DiskManager diskMan(dataPaths);
for (PartitionID partId = 1; partId <= 10; partId++) {
diskMan.addPartToPath(spaceId, partId, path1);
}
for (PartitionID partId = 11; partId <= 20; partId++) {
diskMan.addPartToPath(spaceId, partId, path2);
}
for (PartitionID partId = 1; partId <= 10; partId++) {
diskMan.addPartToPath(spaceId2, partId, path3);
}

SpaceDiskPartsMap diskParts;
diskMan.getDiskParts(diskParts);
ASSERT_EQ(2, diskParts.size());
ASSERT_EQ(2, diskParts[spaceId].size());
ASSERT_EQ(1, diskParts[spaceId2].size());
ASSERT_EQ(10, diskParts[spaceId2][path3].get_part_list().size());
}

} // namespace kvstore
} // namespace nebula

Expand Down