From 565f9e5bf93315f016eddae475bf2b708711db03 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Tue, 4 Apr 2023 19:50:50 +0200 Subject: [PATCH 01/10] Fix build and warnings for realm tools --- src/realm/exec/realm_browser.cpp | 15 ++++--- src/realm/exec/realm_decrypt.cpp | 4 +- src/realm/exec/realm_dump.c | 66 +++++++++++++++++------------- src/realm/exec/realm_encrypt.cpp | 2 - src/realm/exec/realm_enumerate.cpp | 4 +- src/realm/exec/realm_trawler.cpp | 14 +++---- 6 files changed, 57 insertions(+), 48 deletions(-) mode change 100755 => 100644 src/realm/exec/realm_dump.c diff --git a/src/realm/exec/realm_browser.cpp b/src/realm/exec/realm_browser.cpp index a5990a96fa0..ab97123d77f 100644 --- a/src/realm/exec/realm_browser.cpp +++ b/src/realm/exec/realm_browser.cpp @@ -5,7 +5,10 @@ using namespace realm; -bool get_table_ndx(size_t& ndx) +#define LL long long +#define ULL unsigned long long + +static bool get_table_ndx(size_t& ndx) { std::cout << "Table ndx? "; std::string inp; @@ -16,7 +19,7 @@ bool get_table_ndx(size_t& ndx) return *endp == '\0'; } -bool get_range(size_t size, size_t& begin, size_t& end) +static bool get_range(size_t size, size_t& begin, size_t& end) { std::cout << "Size " << size << ". Range? "; std::string inp; @@ -46,7 +49,7 @@ bool get_range(size_t size, size_t& begin, size_t& end) return false; } -void print_objects(ConstTableRef table, size_t begin, size_t end) +static void print_objects(ConstTableRef table, size_t begin, size_t end) { printf(" Object key"); auto col_keys = table->get_column_keys(); @@ -57,7 +60,7 @@ void print_objects(ConstTableRef table, size_t begin, size_t end) for (size_t row = begin; row < end; row++) { printf("%5zu ", row); Obj obj = table->get_object(row); - printf(" %20zx", obj.get_key().value); + printf(" %20llx", (ULL)obj.get_key().value); for (auto col : col_keys) { auto col_type = table->get_column_type(col); if (table->get_column_attr(col).test(col_attr_Nullable) && obj.is_null(col)) { @@ -70,7 +73,7 @@ void print_objects(ConstTableRef table, size_t begin, size_t end) } switch (col_type) { case type_Int: - printf(" %20ld", obj.get(col)); + printf(" %20lld", (LL)obj.get(col)); break; case type_Bool: printf(" %20s", obj.get(col) ? "true" : "false"); @@ -106,7 +109,7 @@ void print_objects(ConstTableRef table, size_t begin, size_t end) break; } case type_Link: { - printf(" -> %12zx", obj.get(col).value); + printf(" -> %12llx", (ULL)obj.get(col).value); break; } case type_LinkList: { diff --git a/src/realm/exec/realm_decrypt.cpp b/src/realm/exec/realm_decrypt.cpp index eb841e13d9d..7e08063850c 100644 --- a/src/realm/exec/realm_decrypt.cpp +++ b/src/realm/exec/realm_decrypt.cpp @@ -33,8 +33,8 @@ int main(int argc, const char* argv[]) util::File file; file.open(path); file.set_encryption_key(key); - size_t size = file.get_size(); - size_t pos = 0; + auto size = file.get_size(); + decltype(size) pos = 0; util::AESCryptor cryptor(key_ptr); cryptor.set_file_size(size); while (pos < size) { diff --git a/src/realm/exec/realm_dump.c b/src/realm/exec/realm_dump.c old mode 100755 new mode 100644 index 5791997fa44..e45e84b5500 --- a/src/realm/exec/realm_dump.c +++ b/src/realm/exec/realm_dump.c @@ -11,6 +11,9 @@ #define do_seek fseek #endif +#define LL long long +#define ULL unsigned long long + typedef struct _FileHeader { uint64_t m_top_ref[2]; // 2 * 8 bytes // Info-block 8-bytes @@ -32,12 +35,12 @@ typedef struct _NodeHeader { size_t num_bytes; } NodeHeader; -char to_print(unsigned char ch) +static char to_print(unsigned char ch) { return (ch >= 0x20 && ch <= 0x7e) ? (char)ch : '.'; } -void dump_buffer(unsigned char* buffer, uint64_t addr, size_t sz) +static void dump_buffer(unsigned char* buffer, uint64_t addr, size_t sz) { char printable[20]; unsigned char* ptr = buffer; @@ -46,7 +49,7 @@ void dump_buffer(unsigned char* buffer, uint64_t addr, size_t sz) size_t len = (end - ptr); if (len > 16) len = 16; - printf("%08zx ", addr + (ptr - buffer)); + printf("%08llx ", (ULL)(addr + (ptr - buffer))); char* trans = printable; *trans++ = '|'; for (size_t i = 0; i < 16; i++) { @@ -66,7 +69,7 @@ void dump_buffer(unsigned char* buffer, uint64_t addr, size_t sz) } } -void dump(FILE* fp, int64_t offset, size_t sz) +static void dump(FILE* fp, int64_t offset, size_t sz) { if (sz) { do_seek(fp, offset, SEEK_SET); @@ -86,14 +89,14 @@ void dump(FILE* fp, int64_t offset, size_t sz) } } -int get_header(NodeHeader* node_header, FILE* fp, int64_t offset) +static int get_header(NodeHeader* node_header, FILE* fp, int64_t offset) { memset(node_header, 0, sizeof(NodeHeader)); unsigned char header[8]; do_seek(fp, offset, SEEK_SET); fread(header, 1, 8, fp); if (strncmp((const char*)header, "AAAA", 4) != 0) { - printf("Ref '0x%zx' does not point to an array\n", offset); + printf("Ref '0x%llx' does not point to an array\n", (ULL)offset); dump(fp, offset, 64); return 0; } @@ -129,24 +132,24 @@ int get_header(NodeHeader* node_header, FILE* fp, int64_t offset) return 1; } -size_t dump_header(FILE* fp, int64_t offset) +static size_t dump_header(FILE* fp, int64_t offset) { NodeHeader header; if (get_header(&header, fp, offset)) { if (header.is_inner && header.has_refs) { - printf("Ref: 0x%zx, Size: %zd, width: %d %s Inner B+tree node\n", offset, header.size, header.width, + printf("Ref: 0x%llx, Size: %zd, width: %d %s Inner B+tree node\n", (ULL)offset, header.size, header.width, header.type); } else { - printf("Ref: 0x%zx, Size: %zd, width: %d %s, hasRefs: %d, flag: %d\n", offset, header.size, header.width, - header.type, header.has_refs, header.context); + printf("Ref: 0x%llx, Size: %zd, width: %d %s, hasRefs: %d, flag: %d\n", (ULL)offset, header.size, + header.width, header.type, header.has_refs, header.context); } } return header.num_bytes; } -void dump_file_header(FILE* fp) +static void dump_file_header(FILE* fp) { FileHeader header; do_seek(fp, 0, SEEK_SET); @@ -158,7 +161,7 @@ void dump_file_header(FILE* fp) dump(fp, header.m_top_ref[1] + 8, sz); } -int64_t get_top_ref(FILE* fp) +static int64_t get_top_ref(FILE* fp) { FileHeader header; do_seek(fp, 0, SEEK_SET); @@ -166,14 +169,15 @@ int64_t get_top_ref(FILE* fp) return header.m_top_ref[header.m_flags]; } -int search_ref(FILE* fp, int64_t ref, int64_t target, size_t level, int* stack) +static int search_ref(FILE* fp, int64_t ref, int64_t target, size_t level, size_t* stack) { + int ret = 0; NodeHeader header; get_header(&header, fp, ref); if (header.has_refs) { assert(header.width >= 8); size_t byte_size = header.width / 8; - char buffer[byte_size * header.size]; + char* buffer = malloc(byte_size * header.size); do_seek(fp, ref + 8, SEEK_SET); fread(buffer, byte_size * header.size, 1, fp); for (size_t i = 0; i < header.size; i++) { @@ -195,37 +199,41 @@ int search_ref(FILE* fp, int64_t ref, int64_t target, size_t level, int* stack) } if (subref && (subref & 1) == 0) { if (subref == target) { - printf("Ref '0x%zx' found at [", target); + printf("Ref '0x%llx' found at [", (ULL)target); for (size_t j = 0; j < level + 1; j++) { if (j == 0) - printf("%d", stack[j]); + printf("%lld", (LL)stack[j]); else - printf(",%d", stack[j]); + printf(",%lld", (LL)stack[j]); } printf("]\n"); - return 1; + ret = 1; + break; + } + if (search_ref(fp, subref, target, level + 1, stack)) { + ret = 1; + break; } - if (search_ref(fp, subref, target, level + 1, stack)) - return 1; } } + free(buffer); } - return 0; + return ret; } -void dump_index(FILE* fp, int64_t ref, const char* arr) +static void dump_index(FILE* fp, int64_t ref, const char* arr) { char* p = (char*)arr; - unsigned idx = strtoll(arr, &p, 0); + ULL idx = strtoll(arr, &p, 0); NodeHeader header; get_header(&header, fp, ref); if (!header.has_refs) { - printf("Ref '0x%zx' does not point to an array with refs\n", ref); + printf("Ref '0x%llx' does not point to an array with refs\n", (ULL)ref); dump_header(fp, ref); exit(1); } if (idx >= header.size) { - printf("Index '%d' is out of bounds (size = %d)\n", idx, (int)header.size); + printf("Index '%lld' is out of bounds (size = %d)\n", idx, (int)header.size); dump_header(fp, ref); exit(1); } @@ -238,7 +246,7 @@ void dump_index(FILE* fp, int64_t ref, const char* arr) fread(&subref, byte_size, 1, fp); if (subref & 1) { - printf("Value '%ld' is not a subref\n", subref); + printf("Value '%lld' is not a subref\n", (LL)subref); exit(1); } while (isspace(*p)) @@ -248,13 +256,13 @@ void dump_index(FILE* fp, int64_t ref, const char* arr) dump_index(fp, subref, p); } else { - printf("looking up index %d at 0x%zx = 0x%zx\n", idx, (size_t)offset, subref); + printf("looking up index %lld at 0x%llx = 0x%llx\n", (LL)idx, (ULL)offset, (ULL)subref); size_t sz = dump_header(fp, subref); dump(fp, subref + 8, sz); } } -void usage() +static void usage() { printf("Usage: realm-dump [?][] []\n"); exit(1); @@ -304,7 +312,7 @@ int main(int argc, const char* argv[]) dump_index(fp, ref, array_str); } else if (find_ref) { - int stack[128]; + size_t stack[128]; if (!ref) ref = get_top_ref(fp); search_ref(fp, ref, find_ref, 0, stack); diff --git a/src/realm/exec/realm_encrypt.cpp b/src/realm/exec/realm_encrypt.cpp index 029ee7f4438..4a77e998700 100644 --- a/src/realm/exec/realm_encrypt.cpp +++ b/src/realm/exec/realm_encrypt.cpp @@ -5,8 +5,6 @@ using namespace realm; -constexpr size_t block_size = 4096; - int main(int argc, const char* argv[]) { if (argc > 1) { diff --git a/src/realm/exec/realm_enumerate.cpp b/src/realm/exec/realm_enumerate.cpp index 341abf5dcac..44f534c8454 100644 --- a/src/realm/exec/realm_enumerate.cpp +++ b/src/realm/exec/realm_enumerate.cpp @@ -106,8 +106,8 @@ int main(int argc, const char* argv[]) try { realm = realm::Realm::get_shared_realm(config); } - catch (const realm::RealmFileException& e) { - std::cout << "trying to open as a sync Realm\n" << std::endl; + catch (const realm::FileAccessError& e) { + std::cout << "trying to open as a sync Realm\n" << e.what() << "\n" << std::endl; config.force_sync_history = true; realm = realm::Realm::get_shared_realm(config); } diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp index e08b5b285b1..57468304888 100644 --- a/src/realm/exec/realm_trawler.cpp +++ b/src/realm/exec/realm_trawler.cpp @@ -537,7 +537,7 @@ class RealmFile { realm::SlabAlloc m_alloc; }; -std::string human_readable(uint64_t val) +static std::string human_readable(uint64_t val) { std::ostringstream out; out.precision(3); @@ -556,7 +556,7 @@ std::string human_readable(uint64_t val) return out.str(); } -uint64_t get_size(const std::vector& list) +static uint64_t get_size(const std::vector& list) { uint64_t sz = 0; std::for_each(list.begin(), list.end(), [&](const Entry& e) { @@ -629,7 +629,7 @@ void Table::print_columns(const Group& group) const if (col_key.is_set()) type_str += "{}"; if (col_key.is_dictionary()) { - auto key_type = realm::DataType(m_column_types.get_val(i) >> 16); + auto key_type = realm::DataType(int(m_column_types.get_val(i)) >> 16); type_str = std::string("{") + get_data_type_name(key_type) + ", " + type_str + "}"; } } @@ -675,7 +675,7 @@ void Node::init(realm::Allocator& alloc, uint64_t ref) std::vector path; -std::string print_path() +static std::string print_path() { std::string ret = "[" + std::to_string(path[0]); for (auto it = path.begin() + 1; it != path.end(); ++it) { @@ -685,7 +685,7 @@ std::string print_path() return ret + "]"; } -std::vector get_nodes(realm::Allocator& alloc, uint64_t ref) +static std::vector get_nodes(realm::Allocator& alloc, uint64_t ref) { std::vector nodes; if (ref != 0) { @@ -1089,7 +1089,7 @@ void RealmFile::changes() const } } -unsigned int hex_char_to_bin(char c) +static unsigned int hex_char_to_bin(char c) { if (c >= '0' && c <= '9') return c - '0'; @@ -1100,7 +1100,7 @@ unsigned int hex_char_to_bin(char c) throw std::invalid_argument("Illegal key (not a hex digit)"); } -unsigned int hex_to_bin(char first, char second) +static unsigned int hex_to_bin(char first, char second) { return (hex_char_to_bin(first) << 4) | hex_char_to_bin(second); } From a6955d569ec09b5b9aac570facd91b148f2e5dc5 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 5 Apr 2023 13:40:47 +0200 Subject: [PATCH 02/10] Resurrect csv imported tool --- src/realm/exec/CMakeLists.txt | 13 ++-- src/realm/exec/importer.cpp | 100 ++++++++++++++++++------------- src/realm/exec/importer_tool.cpp | 2 +- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 2796d873f5e..cb27f03ca0d 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -1,18 +1,17 @@ +add_executable(RealmImporter EXCLUDE_FROM_ALL importer_tool.cpp importer.cpp importer.hpp) +set_target_properties(RealmImporter PROPERTIES + OUTPUT_NAME "realm-importer" + DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} +) +target_link_libraries(RealmImporter Storage) if(NOT APPLE AND NOT ANDROID AND NOT CMAKE_SYSTEM_NAME MATCHES "^Windows") - # FIXME add_executable(RealmImporter importer_tool.cpp importer.cpp importer.hpp) - # set_target_properties(RealmImporter PROPERTIES - # OUTPUT_NAME "realm-importer" - # DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX}) - # target_link_libraries(RealmImporter realm) - add_executable(RealmDaemon realmd.cpp) set_target_properties(RealmDaemon PROPERTIES OUTPUT_NAME "realmd" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX}) target_link_libraries(RealmDaemon Storage) list(APPEND ExecTargetsToInstall RealmDaemon) - endif() add_executable(RealmTrawler EXCLUDE_FROM_ALL realm_trawler.cpp ) diff --git a/src/realm/exec/importer.cpp b/src/realm/exec/importer.cpp index 8acacd1f2d5..27078e740d4 100644 --- a/src/realm/exec/importer.cpp +++ b/src/realm/exec/importer.cpp @@ -65,13 +65,13 @@ const char* DataTypeToText(DataType t) void print_col_names(Table& table) { std::cout << "\n"; - for (size_t t = 0; t < table.get_column_count(); t++) { + for (auto t : table.get_column_keys()) { std::string s = std::string(table.get_column_name(t).data()); s = set_width(s, print_width); std::cout << s.c_str() << " "; } std::cout << "\n"; - for (size_t t = 0; t < table.get_column_count(); t++) { + for (auto t : table.get_column_keys()) { std::string s = "Type: " + std::string(DataTypeToText(table.get_column_type(t))); s = set_width(s, print_width); std::cout << s.c_str() << " "; @@ -83,23 +83,33 @@ void print_col_names(Table& table) // Prints row 'r' of a Realm table void print_row(Table& table, size_t r) { - for (size_t c = 0; c < table.get_column_count(); c++) { + auto&& obj = table.get_object(r); + for (auto k : table.get_column_keys()) { char buf[print_width]; - - if (table.get_column_type(c) == type_Bool) - sprintf(buf, "%s", table.get_bool(c, r) ? "true" : "false"); - if (table.get_column_type(c) == type_Double) - sprintf(buf, "%f", table.get_double(c, r)); - if (table.get_column_type(c) == type_Float) - sprintf(buf, "%f", table.get_float(c, r)); - if (table.get_column_type(c) == type_Int) - sprintf(buf, "%lld", static_cast(table.get_int(c, r))); - if (table.get_column_type(c) == type_String) { + auto type = table.get_column_type(k); + + switch (type) { + case type_Bool: + snprintf(buf, print_width, "%s", obj.get(k) ? "true" : "false"); + break; + case type_Double: + snprintf(buf, print_width, "%f", obj.get(k)); + break; + case type_Float: + snprintf(buf, print_width, "%f", obj.get(k)); + break; + case type_Int: + snprintf(buf, print_width, "%lld", static_cast(obj.get(k))); + break; + case type_String: #if defined(_MSC_VER) && _MSC_VER - _snprintf(buf, sizeof(buf), "%s", table.get_string(c, r).data()); + _snprintf(buf, sizeof(buf), "%s", obj.get(k).data()); #else - snprintf(buf, sizeof(buf), "%s", table.get_string(c, r).data()); + snprintf(buf, sizeof(buf), "%s", obj.get(k).data()); #endif + break; + default: + break; } std::string s = std::string(buf); s = set_width(s, print_width); @@ -528,10 +538,10 @@ size_t Importer::tokenize(std::vector>& payload, size_t std::string s = payload[payload.size() - 1][0]; if (s.length() > 100) s = s.substr(0, 100); - sprintf(buf, - "Wrong number of delimitors around line %lld (+|- 3) in csv file. First few characters " - "of line: %s", - static_cast(m_row - 1), s.c_str()); + snprintf(buf, 500, + "Wrong number of delimitors around line %lld (+|- 3) in csv file. First few characters " + "of line: %s", + static_cast(m_row - 1), s.c_str()); throw std::runtime_error(buf); } } @@ -615,7 +625,7 @@ size_t Importer::import_csv(FILE* file, Table& table, std::vector* imp // occurences by a string if (header[t] == "") { char buf[10]; - sprintf(buf, "Column%d", static_cast(t)); + snprintf(buf, 10, "Column%d", static_cast(t)); header[t] = buf; } } @@ -624,7 +634,7 @@ size_t Importer::import_csv(FILE* file, Table& table, std::vector* imp // Use "1", "2", "3", ... for column names for (size_t i = 0; i < scheme1.size(); i++) { char buf[10]; - sprintf(buf, "%d", static_cast(i)); + snprintf(buf, 10, "%d", static_cast(i)); header.push_back(buf); } } @@ -663,33 +673,40 @@ size_t Importer::import_csv(FILE* file, Table& table, std::vector* imp if (!Quiet && imported_rows % 123 == 0) std::cout << imported_rows << " rows\r"; - // Add empty row to Realm - REALM_ASSERT(false); // unimplemented - // table.add_empty_row(); - - // Add all fields to new row - for (size_t col = 0; col < scheme.size(); col++) { + FieldValues values; + auto keys = table.get_column_keys(); + size_t col = 0; + for (auto key = keys.begin(); key != keys.end(); ++key, ++col) { bool success = true; - if (scheme[col] == type_String) - table.set_string(col, imported_rows, StringData(payload[row][col])); - else if (scheme[col] == type_Int) - table.set_int(col, imported_rows, parse_integer(payload[row][col].c_str(), &success)); - else if (scheme[col] == type_Double) - table.set_double(col, imported_rows, parse_double(payload[row][col].c_str(), &success)); - else if (scheme[col] == type_Float) - table.set_float(col, imported_rows, parse_float(payload[row][col].c_str(), &success)); - else if (scheme[col] == type_Bool) - table.set_bool(col, imported_rows, parse_bool(payload[row][col].c_str(), &success)); - else - REALM_ASSERT(false); + switch (scheme[col]) { + case type_String: + values.insert(*key, StringData(payload[row][col])); + break; + case type_Int: + values.insert(*key, parse_integer(payload[row][col].c_str(), &success)); + break; + case type_Double: + values.insert(*key, parse_double(payload[row][col].c_str(), &success)); + break; + case type_Float: + values.insert(*key, parse_float(payload[row][col].c_str(), &success)); + break; + case type_Bool: + values.insert(*key, parse_bool(payload[row][col].c_str(), &success)); + break; + default: + REALM_ASSERT(false); + break; + } if (!success) { // Remove all columns so that user can call csv_import() on it again table.clear(); - for (size_t t = 0; t < table.get_column_count(); t++) - table.remove_column(0); + auto keys = table.get_column_keys(); + for (size_t i = keys.size(); i != 0;) + table.remove_column(keys[--i]); std::stringstream sstm; @@ -718,6 +735,7 @@ size_t Importer::import_csv(FILE* file, Table& table, std::vector* imp } } + (void)table.create_object(ObjKey(import_rows), values); if (!Quiet) { if (imported_rows < 10) diff --git a/src/realm/exec/importer_tool.cpp b/src/realm/exec/importer_tool.cpp index 5059ad5c9ba..f71a3c528ef 100644 --- a/src/realm/exec/importer_tool.cpp +++ b/src/realm/exec/importer_tool.cpp @@ -206,7 +206,7 @@ int main(int argc, char* argv[]) } } catch (const std::runtime_error& error) { - std::cerr << error.what(); + std::cerr << error.what() << std::endl; exit(-1); } From 179d9ced18ba7f2439eeb0a4cc918e20e8a14242 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 5 Apr 2023 13:41:52 +0200 Subject: [PATCH 03/10] Add build_all optional build target for realm tools --- src/realm/exec/CMakeLists.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index cb27f03ca0d..2cc8b2bd190 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -14,35 +14,35 @@ if(NOT APPLE AND NOT ANDROID AND NOT CMAKE_SYSTEM_NAME MATCHES "^Windows") list(APPEND ExecTargetsToInstall RealmDaemon) endif() -add_executable(RealmTrawler EXCLUDE_FROM_ALL realm_trawler.cpp ) +add_executable(RealmTrawler EXCLUDE_FROM_ALL realm_trawler.cpp) set_target_properties(RealmTrawler PROPERTIES OUTPUT_NAME "realm-trawler" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmTrawler Storage) -add_executable(RealmEnumerate EXCLUDE_FROM_ALL realm_enumerate.cpp ) +add_executable(RealmEnumerate EXCLUDE_FROM_ALL realm_enumerate.cpp) set_target_properties(RealmEnumerate PROPERTIES OUTPUT_NAME "realm-enumerate" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmEnumerate ObjectStore) -add_executable(RealmDecrypt EXCLUDE_FROM_ALL realm_decrypt.cpp ) +add_executable(RealmDecrypt EXCLUDE_FROM_ALL realm_decrypt.cpp) set_target_properties(RealmDecrypt PROPERTIES OUTPUT_NAME "realm-decrypt" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmDecrypt Storage) -add_executable(RealmEncrypt EXCLUDE_FROM_ALL realm_encrypt.cpp ) +add_executable(RealmEncrypt EXCLUDE_FROM_ALL realm_encrypt.cpp) set_target_properties(RealmEncrypt PROPERTIES OUTPUT_NAME "realm-encrypt" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmEncrypt Storage) -add_executable(RealmBrowser EXCLUDE_FROM_ALL realm_browser.cpp ) +add_executable(RealmBrowser EXCLUDE_FROM_ALL realm_browser.cpp) set_target_properties(RealmBrowser PROPERTIES OUTPUT_NAME "realm-browser-10" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} @@ -68,3 +68,8 @@ set_target_properties(RealmDump PROPERTIES install(TARGETS ${ExecTargetsToInstall} COMPONENT runtime DESTINATION bin) + +add_custom_target(build_all DEPENDS + RealmImporter RealmTrawler RealmEnumerate RealmDecrypt + RealmEncrypt RealmBrowser RealmDump) +add_dependencies(build_all ${ExecTargetsToInstall}) From f3e2d3f8cbc95de315da1efc9bea8b737c798301 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 5 Apr 2023 13:49:56 +0200 Subject: [PATCH 04/10] Add fileds separator flag to importer --- src/realm/exec/importer_tool.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/realm/exec/importer_tool.cpp b/src/realm/exec/importer_tool.cpp index f71a3c528ef..0cbea5b77de 100644 --- a/src/realm/exec/importer_tool.cpp +++ b/src/realm/exec/importer_tool.cpp @@ -59,12 +59,14 @@ const char* legend = " -q: Quiet, only print upon errors\n" " -f: Overwrite destination file if existing (default is to abort)\n" " -l: Name of the resulting table (default is 'table')\n" + " -p: Separator to use (default: ',')" "\n" "Examples:\n" " csv file.csv file.realm\n" " csv -a=200000 -e file.csv file.realm\n" " csv -t=ssdbi Name Email Height Gender Age file.csv -s=1 file.realm\n" - " csv -stdin file.realm < cat file.csv"; + " csv -stdin file.realm < cat file.csv\n" + " csv -p=$'\\t' file.tsv file.realm"; namespace { @@ -158,6 +160,11 @@ int main(int argc, char* argv[]) abort2(a >= argc - 4, "Too few arguments"); tablename = argv[++a]; } + else if (strncmp(argv[a], "-p", 2) == 0) { + std::cout << "separator: [" << argv[a][3] << "]" << std::endl; + abort2(strlen(argv[a]) != 4, "Set exactly one character for delimeter!"); + separator_flag = argv[a][3]; + } else { abort2(true, legend); } @@ -186,7 +193,7 @@ int main(int argc, char* argv[]) Importer importer; importer.Quiet = quiet_flag; - importer.Separator = ','; + importer.Separator = separator_flag; importer.Empty_as_string = empty_as_string_flag; try { From 6b543d89aee01c3b381700e0d9dae31b6e99a547 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Thu, 6 Apr 2023 20:58:28 +0200 Subject: [PATCH 05/10] Don't add empty dependencies --- src/realm/exec/CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 2cc8b2bd190..5f1abf3ff48 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -69,7 +69,10 @@ install(TARGETS ${ExecTargetsToInstall} COMPONENT runtime DESTINATION bin) -add_custom_target(build_all DEPENDS - RealmImporter RealmTrawler RealmEnumerate RealmDecrypt - RealmEncrypt RealmBrowser RealmDump) -add_dependencies(build_all ${ExecTargetsToInstall}) +set(OptionalToolsTargets RealmImporter RealmTrawler RealmEnumerate + RealmDecrypt RealmEncrypt RealmBrowser RealmDump) + +add_custom_target(build_all DEPENDS ${OptionalToolsTargets}) +if (ExecTargetsToInstall) + add_dependencies(build_all ${ExecTargetsToInstall}) +endif() From 6721f89d880b00b845e90fd0cd882fc921bba4b3 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Thu, 6 Apr 2023 20:59:15 +0200 Subject: [PATCH 06/10] Try to at least build tools by default --- src/realm/exec/CMakeLists.txt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 5f1abf3ff48..8f6f0f19917 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(RealmImporter EXCLUDE_FROM_ALL importer_tool.cpp importer.cpp importer.hpp) +add_executable(RealmImporter importer_tool.cpp importer.cpp importer.hpp) set_target_properties(RealmImporter PROPERTIES OUTPUT_NAME "realm-importer" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} @@ -14,35 +14,35 @@ if(NOT APPLE AND NOT ANDROID AND NOT CMAKE_SYSTEM_NAME MATCHES "^Windows") list(APPEND ExecTargetsToInstall RealmDaemon) endif() -add_executable(RealmTrawler EXCLUDE_FROM_ALL realm_trawler.cpp) +add_executable(RealmTrawler realm_trawler.cpp) set_target_properties(RealmTrawler PROPERTIES OUTPUT_NAME "realm-trawler" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmTrawler Storage) -add_executable(RealmEnumerate EXCLUDE_FROM_ALL realm_enumerate.cpp) +add_executable(RealmEnumerate realm_enumerate.cpp) set_target_properties(RealmEnumerate PROPERTIES OUTPUT_NAME "realm-enumerate" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmEnumerate ObjectStore) -add_executable(RealmDecrypt EXCLUDE_FROM_ALL realm_decrypt.cpp) +add_executable(RealmDecrypt realm_decrypt.cpp) set_target_properties(RealmDecrypt PROPERTIES OUTPUT_NAME "realm-decrypt" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmDecrypt Storage) -add_executable(RealmEncrypt EXCLUDE_FROM_ALL realm_encrypt.cpp) +add_executable(RealmEncrypt realm_encrypt.cpp) set_target_properties(RealmEncrypt PROPERTIES OUTPUT_NAME "realm-encrypt" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmEncrypt Storage) -add_executable(RealmBrowser EXCLUDE_FROM_ALL realm_browser.cpp) +add_executable(RealmBrowser realm_browser.cpp) set_target_properties(RealmBrowser PROPERTIES OUTPUT_NAME "realm-browser-10" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} @@ -59,7 +59,7 @@ target_link_libraries(Realm2JSON Storage QueryParser Sync) list(APPEND ExecTargetsToInstall Realm2JSON) endif() -add_executable(RealmDump EXCLUDE_FROM_ALL realm_dump.c) +add_executable(RealmDump realm_dump.c) set_target_properties(RealmDump PROPERTIES OUTPUT_NAME "realm-dump" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} @@ -76,3 +76,6 @@ add_custom_target(build_all DEPENDS ${OptionalToolsTargets}) if (ExecTargetsToInstall) add_dependencies(build_all ${ExecTargetsToInstall}) endif() + +message("Will try to build tools by default for: ${CMAKE_SYSTEM_NAME}...") +# set_target_properties(${OptionalToolsTargets} PROPERTIES EXCLUDE_FROM_ALL TRUE) From b72770906cb24e02b347e640021a6edec62f2ac0 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 10 May 2023 18:18:05 +0200 Subject: [PATCH 07/10] Only exclude the one with libuv dep --- src/realm/exec/CMakeLists.txt | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 8f6f0f19917..de6293d1ef1 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -27,6 +27,10 @@ set_target_properties(RealmEnumerate PROPERTIES DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmEnumerate ObjectStore) +# FIXME can be fixed for others, but requires link and install fixes for libuv target +if (NOT APPLE) + set_target_properties(RealmEnumerate PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() add_executable(RealmDecrypt realm_decrypt.cpp) set_target_properties(RealmDecrypt PROPERTIES @@ -68,14 +72,3 @@ set_target_properties(RealmDump PROPERTIES install(TARGETS ${ExecTargetsToInstall} COMPONENT runtime DESTINATION bin) - -set(OptionalToolsTargets RealmImporter RealmTrawler RealmEnumerate - RealmDecrypt RealmEncrypt RealmBrowser RealmDump) - -add_custom_target(build_all DEPENDS ${OptionalToolsTargets}) -if (ExecTargetsToInstall) - add_dependencies(build_all ${ExecTargetsToInstall}) -endif() - -message("Will try to build tools by default for: ${CMAKE_SYSTEM_NAME}...") -# set_target_properties(${OptionalToolsTargets} PROPERTIES EXCLUDE_FROM_ALL TRUE) From 29cd4057aa8a8e2317eb616d1503f4b43e6d4b0d Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 6 Sep 2023 21:52:59 +0200 Subject: [PATCH 08/10] Add change log entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1480c0e4ce4..11f866976e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ ----------- ### Internals -* None. +* Fix build of most internal tools, also build them by default. ([PR #6475](https://github.com/realm/realm-core/pull/6475)) ---------------------------------------------- From c8a796001c72c6f00c0b699402a54639b6bca942 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Wed, 6 Sep 2023 23:09:19 +0200 Subject: [PATCH 09/10] Exclude some more configurations --- src/realm/exec/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index de6293d1ef1..16bb966d868 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -20,6 +20,9 @@ set_target_properties(RealmTrawler PROPERTIES DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmTrawler Storage) +if (EMSCRIPTEN) + set_target_properties(RealmTrawler PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() add_executable(RealmEnumerate realm_enumerate.cpp) set_target_properties(RealmEnumerate PROPERTIES @@ -38,6 +41,9 @@ set_target_properties(RealmDecrypt PROPERTIES DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmDecrypt Storage) +if (NOT REALM_ENABLE_ENCRYPTION) + set_target_properties(RealmDecrypt PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() add_executable(RealmEncrypt realm_encrypt.cpp) set_target_properties(RealmEncrypt PROPERTIES From 8e900af9ef0afe5b8e21412213c2da1edfef8e21 Mon Sep 17 00:00:00 2001 From: Kirill Burtsev Date: Thu, 7 Sep 2023 14:08:16 +0200 Subject: [PATCH 10/10] Warning fixes for msvc arm64 and x86 builds --- src/external/IntelRDFPMathLib20U2/CMakeLists.txt | 5 ++++- src/external/s2/util/math/mathutil.h | 4 ++++ src/realm/exec/realm_decrypt.cpp | 2 +- src/realm/exec/realm_dump.c | 8 ++++---- src/realm/exec/realm_trawler.cpp | 16 ++++++++-------- test/object-store/CMakeLists.txt | 7 +++++++ 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/external/IntelRDFPMathLib20U2/CMakeLists.txt b/src/external/IntelRDFPMathLib20U2/CMakeLists.txt index b7ad7c17c49..182adfcb6d4 100644 --- a/src/external/IntelRDFPMathLib20U2/CMakeLists.txt +++ b/src/external/IntelRDFPMathLib20U2/CMakeLists.txt @@ -19,7 +19,10 @@ LIBRARY/src/bid_round.c add_library(Bid OBJECT ${BID_SOURCES}) if(MSVC) - target_compile_options(Bid PUBLIC /W1 /sdl-) + get_target_property(comp_opts Bid COMPILE_OPTIONS) + string(REGEX REPLACE "/W[1234]" "/W1" comp_opts "${comp_opts}") + set_target_properties(Bid PROPERTIES COMPILE_OPTIONS "${comp_opts}") + target_compile_options(Bid PUBLIC /sdl-) endif() # Note: checking for -Wfoo since unknown -Wno-foo flags are ignored. diff --git a/src/external/s2/util/math/mathutil.h b/src/external/s2/util/math/mathutil.h index 1fb93f63312..bd63e9cf23c 100644 --- a/src/external/s2/util/math/mathutil.h +++ b/src/external/s2/util/math/mathutil.h @@ -107,11 +107,13 @@ template struct MathLimits { #endif //UTIL_MATH_MATHLIMITS_H // ========================================================================= // +#if defined(__clang__) // Disable error about fabs causing truncation of value because // it takes a double instead of a long double (Clang 3.5) // See SERVER-15183 #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wabsolute-value" +#endif class MathUtil { public: @@ -695,6 +697,8 @@ bool MathUtil::WithinFractionOrMargin(const T x, const T y, } } +#if defined(__clang__) #pragma clang diagnostic pop +#endif #endif // UTIL_MATH_MATHUTIL_H__ diff --git a/src/realm/exec/realm_decrypt.cpp b/src/realm/exec/realm_decrypt.cpp index 7e08063850c..134bc0d1818 100644 --- a/src/realm/exec/realm_decrypt.cpp +++ b/src/realm/exec/realm_decrypt.cpp @@ -33,7 +33,7 @@ int main(int argc, const char* argv[]) util::File file; file.open(path); file.set_encryption_key(key); - auto size = file.get_size(); + auto size = (off_t)file.get_size(); decltype(size) pos = 0; util::AESCryptor cryptor(key_ptr); cryptor.set_file_size(size); diff --git a/src/realm/exec/realm_dump.c b/src/realm/exec/realm_dump.c index e45e84b5500..ccdbdabe692 100644 --- a/src/realm/exec/realm_dump.c +++ b/src/realm/exec/realm_dump.c @@ -72,7 +72,7 @@ static void dump_buffer(unsigned char* buffer, uint64_t addr, size_t sz) static void dump(FILE* fp, int64_t offset, size_t sz) { if (sz) { - do_seek(fp, offset, SEEK_SET); + do_seek(fp, (size_t)offset, SEEK_SET); unsigned char* buffer = malloc(sz); size_t actual = fread(buffer, 1, sz, fp); if (actual != sz) { @@ -93,7 +93,7 @@ static int get_header(NodeHeader* node_header, FILE* fp, int64_t offset) { memset(node_header, 0, sizeof(NodeHeader)); unsigned char header[8]; - do_seek(fp, offset, SEEK_SET); + do_seek(fp, (size_t)offset, SEEK_SET); fread(header, 1, 8, fp); if (strncmp((const char*)header, "AAAA", 4) != 0) { printf("Ref '0x%llx' does not point to an array\n", (ULL)offset); @@ -178,7 +178,7 @@ static int search_ref(FILE* fp, int64_t ref, int64_t target, size_t level, size_ assert(header.width >= 8); size_t byte_size = header.width / 8; char* buffer = malloc(byte_size * header.size); - do_seek(fp, ref + 8, SEEK_SET); + do_seek(fp, (size_t)(ref + 8), SEEK_SET); fread(buffer, byte_size * header.size, 1, fp); for (size_t i = 0; i < header.size; i++) { stack[level] = i; @@ -242,7 +242,7 @@ static void dump_index(FILE* fp, int64_t ref, const char* arr) assert(header.width >= 8); size_t byte_size = header.width / 8; int64_t offset = ref + 8 + byte_size * idx; - do_seek(fp, offset, SEEK_SET); + do_seek(fp, (size_t)offset, SEEK_SET); fread(&subref, byte_size, 1, fp); if (subref & 1) { diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp index 2e25fcebbe4..20dca4fe5ce 100644 --- a/src/realm/exec/realm_trawler.cpp +++ b/src/realm/exec/realm_trawler.cpp @@ -248,9 +248,9 @@ class Array : public Node { return str; } - size_t mem_usage(realm::Allocator& alloc) const + uint64_t mem_usage(realm::Allocator& alloc) const { - size_t mem = 0; + uint64_t mem = 0; _mem_usage(alloc, mem); return mem; } @@ -259,7 +259,7 @@ class Array : public Node { char* m_data; bool m_has_refs = false; - void _mem_usage(realm::Allocator& alloc, size_t& mem) const + void _mem_usage(realm::Allocator& alloc, uint64_t& mem) const { if (m_has_refs) { for (size_t i = 0; i < m_size; ++i) { @@ -336,11 +336,11 @@ class Table : public Array { } else { if (uint64_t key_ref = m_clusters.get_ref(0)) { - auto header = alloc.translate(key_ref); + auto header = alloc.translate(realm::to_ref(key_ref)); ret = realm::NodeHeader::get_size_from_header(header); } else { - ret = m_clusters.get_val(0); + ret = (size_t)m_clusters.get_val(0); } } } @@ -662,7 +662,7 @@ void Group::print_schema() const void Node::init(realm::Allocator& alloc, uint64_t ref) { m_ref = ref; - m_header = alloc.translate(ref); + m_header = alloc.translate(realm::to_ref(ref)); if (memcmp(m_header, &signature, 4)) { } @@ -895,8 +895,8 @@ void RealmFile::free_list_info() const std::map pinned_sizes; std::cout << "Free space:" << std::endl; auto free_list = m_group->get_free_list(); - size_t pinned_free_list_size = 0; - size_t total_free_list_size = 0; + uint64_t pinned_free_list_size = 0; + uint64_t total_free_list_size = 0; auto it = free_list.begin(); auto end = free_list.end(); while (it != end) { diff --git a/test/object-store/CMakeLists.txt b/test/object-store/CMakeLists.txt index 5391e54bb81..d6e6d1dbbb1 100644 --- a/test/object-store/CMakeLists.txt +++ b/test/object-store/CMakeLists.txt @@ -190,11 +190,18 @@ if(NOT APPLE AND NOT EMSCRIPTEN AND NOT WINDOWS_STORE) add_subdirectory(${libuv_SOURCE_DIR} ${libuv_BINARY_DIR} EXCLUDE_FROM_ALL) set(libuv_target uv_a) endif() + target_link_libraries(ObjectStoreTests ${libuv_target}) # FIXME: ObjectStore itself shouldn't care about this, but we need to refactor scheduler.cpp to make it happen target_compile_definitions(ObjectStore PUBLIC REALM_HAVE_UV=1) get_property(libuv_include_dir TARGET ${libuv_target} PROPERTY INCLUDE_DIRECTORIES) target_include_directories(ObjectStore PRIVATE ${libuv_include_dir}) + + if (MSVC) + get_target_property(comp_opts ${libuv_target} COMPILE_OPTIONS) + string(REGEX REPLACE "/W[1234]" "/W1" comp_opts "${comp_opts}") + set_target_properties(${libuv_target} PROPERTIES COMPILE_OPTIONS "${comp_opts}") + endif() endif() add_subdirectory(notifications-fuzzer)