fix: local_service compile error loading file metadata (#666)

src/block_service/local/local_service.cpp

@@ -6,9 +6,11 @@
 #include <dsn/utility/utils.h>
 #include <dsn/utility/strings.h>
 #include <dsn/utility/safe_strerror_posix.h>
+#include <dsn/dist/fmt_logging.h>
 
 #include <dsn/cpp/json_helper.h>
 #include <dsn/tool-api/task_tracker.h>
+#include <nlohmann/json.hpp>
 #include "local_service.h"
 
 static const int max_length = 2048; // max data length read from file each time
@@ -24,8 +26,21 @@ struct file_metadata
 {
     uint64_t size;
     std::string md5;
-    DEFINE_JSON_SERIALIZATION(size, md5)
 };
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(file_metadata, size, md5)
+
+bool file_metadata_from_json(std::ifstream &fin, file_metadata &fmeta) noexcept
+{
+    std::string data;
+    fin >> data;
+    try {
+        nlohmann::json::parse(data).get_to(fmeta);
+        return true;
+    } catch (nlohmann::json::exception &exp) {
+        dwarn_f("decode meta data from json failed: {} [{}]", exp.what(), data);
+        return false;
+    }
+}
 
 std::string local_service::get_metafile(const std::string &filepath)
 {
@@ -239,14 +254,9 @@ error_code local_file_object::load_metadata()
     }
     auto cleanup = dsn::defer([&is]() { is.close(); });
 
-    std::string data;
-    is >> data;
-
     file_metadata meta;
-    bool ans = dsn::json::json_forwarder<file_metadata>::decode(
-        dsn::blob(data.c_str(), 0, data.size()), meta);
+    bool ans = file_metadata_from_json(is, meta);
     if (!ans) {
-        dwarn("decode meta data from json(%s) failed", data.c_str());
         return ERR_FS_INTERNAL;
     }
     _size = meta.size;
@@ -270,7 +280,7 @@ error_code local_file_object::store_metadata()
         return ERR_FS_INTERNAL;
     }
     auto cleanup = dsn::defer([&os]() { os.close(); });
-    dsn::json::json_forwarder<file_metadata>::encode(os, meta);
+    os << nlohmann::json(meta);
 
     return ERR_OK;
 }

src/block_service/test/local_service_test.cpp

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <boost/filesystem.hpp>
+#include <nlohmann/json.hpp>
+
+#include "block_service/local/local_service.h"
+
+namespace dsn {
+namespace dist {
+namespace block_service {
+
+// Simple tests for nlohmann::json serialization, via NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE.
+
+TEST(local_service, store_metadata)
+{
+    local_file_object file("a.txt");
+    error_code ec = file.store_metadata();
+    ASSERT_EQ(ec, ERR_OK);
+
+    auto meta_file_path = local_service::get_metafile(file.file_name());
+    ASSERT_TRUE(boost::filesystem::exists(meta_file_path));
+
+    std::ifstream ifs(meta_file_path);
+    nlohmann::json j;
+    ifs >> j;
+    ASSERT_EQ(j["md5"], "");
+    ASSERT_EQ(j["size"], 0);
+}
+
+TEST(local_service, load_metadata)
+{
+    local_file_object file("a.txt");
+    auto meta_file_path = local_service::get_metafile(file.file_name());
+
+    {
+        std::ofstream ofs(meta_file_path);
+        nlohmann::json j({{"md5", "abcde"}, {"size", 5}});
+        ofs << j;
+        ofs.close();
+
+        ASSERT_EQ(file.load_metadata(), ERR_OK);
+        ASSERT_EQ(file.get_md5sum(), "abcde");
+        ASSERT_EQ(file.get_size(), 5);
+    }
+
+    {
+        std::ofstream ofs(meta_file_path);
+        ofs << "invalid json string";
+        ofs.close();
+
+        local_file_object file2("a.txt");
+        ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL);
+    }
+
+    {
+        std::ofstream ofs(meta_file_path);
+        nlohmann::json j({{"md5", "abcde"}, {"no such key", "illegal"}});
+        ofs << j;
+        ofs.close();
+
+        local_file_object file2("a.txt");
+        ASSERT_EQ(file2.load_metadata(), ERR_FS_INTERNAL);
+    }
+}
+
+} // namespace block_service
+} // namespace dist
+} // namespace dsn

thirdparty/CMakeLists.txt

@@ -1,12 +1,12 @@
 project(pegasus_thirdparties)
 cmake_minimum_required(VERSION 3.0.2)
 
-if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     # require at least gcc 4.8
     if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8)
         message(FATAL_ERROR "GCC version must be at least 4.8!")
-    endif()
-endif()
+    endif ()
+endif ()
 
 include(ExternalProject)
 
@@ -182,9 +182,9 @@ ExternalProject_Add(fds
         )
 
 ExternalProject_Add(fmt
-        URL ${OSS_URL_PREFIX}/fmt-4.0.0.tar.gz
-        https://codeload.github.com/fmtlib/fmt/tar.gz/4.0.0
-        URL_MD5 c9be9a37bc85493d1116b0af59a25eba
+        URL ${OSS_URL_PREFIX}/fmt-7.1.2.zip
+        https://github.com/fmtlib/fmt/releases/download/7.1.2/fmt-7.1.2.zip
+        URL_MD5 cd838fb47c92b3221986a2f454331178
         CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT}
         -DCMAKE_BUILD_TYPE=release
         -DFMT_TEST=false
@@ -256,9 +256,9 @@ ExternalProject_Add(prometheus-cpp
 
 # header-only
 ExternalProject_Add(nlohmann_json
-        URL ${OSS_URL_PREFIX}/nlohmann_json-3.7.3.zip
-        https://github.com/nlohmann/json/releases/download/v3.7.3/include.zip
-        URL_MD5 7249387593792b565dcb30d87bca0de3
+        URL ${OSS_URL_PREFIX}/nlohmann_json-3.9.1.zip
+        https://github.com/nlohmann/json/releases/download/v3.9.1/include.zip
+        URL_MD5 d2f66c608af689e21d69a33c220e974e
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""
         INSTALL_COMMAND cp -R include/nlohmann ${TP_OUTPUT}/include