Merge branch 'master' into kb/fix-tools-build

CHANGELOG.md

@@ -6,7 +6,7 @@
 
 ### Fixed
 * <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
-* None.
+* realm/sync/network/websocket_error.hpp was missing from the install package ([PR #6954](https://github.com/realm/realm-core/pull/6954), since v13.18.0).
 
 ### Breaking changes
 * None.
@@ -17,6 +17,7 @@
 -----------
 
 ### Internals
+* Add a CI job to validate that the headers in the installation package all build ([PR #6954](https://github.com/realm/realm-core/pull/6954)).
 * Fix build of most internal tools, also build them by default. ([PR #6475](https://github.com/realm/realm-core/pull/6475))
 
 ----------------------------------------------

evergreen/config.yml

@@ -3,8 +3,8 @@
 # * Pull Requests:
 #   * add the "for_pull_requests" tag to the task to run for pull requests
 # * Nightly Builds:
-#   * Add the '"for_nightly_tests"'' tag to the task for the "nightly_tests" alias to work
-#   * Add the 'activate: false' setting to the task under the build variant for each task to run during the nightly test runs
+#   * Add the `"for_nightly_tests"` tag to the task for the "nightly_tests" alias to work
+#   * Add the `allowed_requesters: [ "ad_hoc", "patch" ]` entry to the nightly task to only run for periodic tests
 # * Commit Builds:
 #   * All other tasks will be included in the set run when a PR is merged to master
 
@@ -290,7 +290,7 @@ functions:
         # Copy the baas_server log from the remote baas host if it exists
         if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then
           echo "No remote baas host or remote baas host definitions not found"
-          exit
+          exit 0
         fi
 
         BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}')
@@ -394,14 +394,14 @@ functions:
 
         if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then
           echo "No remote baas host or remote baas host definitions not found"
-          exit
+          exit 0
         fi
 
         BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}')
         export BAAS_HOST_NAME
 
         ssh_user="$(printf "ubuntu@%s" "$BAAS_HOST_NAME")"
-        ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=No -o ConnectTimeout=10 -i .baas_ssh_key"
+        ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i .baas_ssh_key"
 
         # Kill the remote baas server process, if the definition files exist
         echo "Terminating baas server on remote host..."
@@ -414,12 +414,15 @@ functions:
         set -o errexit
         set -o verbose
 
-        if [[ ! -d realm-core ]]; then
+        TOP_DIR=$(pwd)/realm-core
+        HANG_ANALYZER_PATH=$TOP_DIR/evergreen/hang_analyzer
+        REQUIREMENTS_PATH=$HANG_ANALYZER_PATH/requirements.txt
+
+        if [[ ! -d $TOP_DIR || ! -d $REQUIREMENTS_PATH ]]; then
           echo "No source directory exists. Not running hang analyzer"
           exit 1
         fi
 
-        TOP_DIR=$(pwd)/realm-core
         mkdir realm-core/hang_analyzer_workdir; cd realm-core/hang_analyzer_workdir
         ${python3|python3} -m venv venv
 
@@ -456,7 +459,6 @@ functions:
 
         python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1
 
-        REQUIREMENTS_PATH=$TOP_DIR/evergreen/hang_analyzer/requirements.txt
         if [ "Windows_NT" = "$OS" ]; then
           REQUIREMENTS_PATH=$(cygpath -w $REQUIREMENTS_PATH)
         fi
@@ -465,7 +467,6 @@ functions:
 
         echo "Going to run hang analyzer"
 
-        HANG_ANALYZER_PATH=$TOP_DIR/evergreen/hang_analyzer
         if [ "Windows_NT" = "$OS" ]; then
           HANG_ANALYZER_PATH=$(cygpath -w $HANG_ANALYZER_PATH)
         fi
@@ -649,6 +650,28 @@ tasks:
         xcrun swift build
         xcrun swift run ObjectStoreTests
 
+- name: validate-installed-headers
+  exec_timeout_secs: 1800
+  commands:
+  - func: "fetch source"
+  - func: "fetch binaries"
+  - command: shell.exec
+    params:
+      working_dir: realm-core
+      shell: /bin/bash
+      script: |-
+        #!/bin/bash
+        set -o errexit
+        set -o pipefail
+        set -o verbose
+
+        if [ -n "${xcode_developer_dir}" ]; then
+            export DEVELOPER_DIR="${xcode_developer_dir}"
+        fi
+
+        ./tools/build-cocoa.sh -bm
+        find core/include/realm -name '*.hpp' -print0 | xargs -0 -P "${max_jobs}" -n 1 clang++ -c -o /dev/null -Icore/include -std=c++17
+
 - name: test-on-exfat
   exec_timeout_secs: 2700
   commands:
@@ -664,6 +687,7 @@ tasks:
 - name: valgrind
   exec_timeout_secs: 14400
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - func: "fetch source"
   - func: "fetch binaries"
@@ -783,6 +807,7 @@ tasks:
 
 - name: long-running-core-tests
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - func: "run tests"
     # The long-running tests can take a really long time on Windows, so we give the test up to 4
@@ -1013,6 +1038,7 @@ tasks:
 
 - name: fuzzer
   tags: [ "for_nightly_tests" ]
+  allowed_requesters: [ "ad_hoc", "patch" ]
   commands:
   - command: shell.exec
     params:
@@ -1228,7 +1254,6 @@ buildvariants:
     distros:
     - ubuntu2004-large
   - name: long-running-tests
-    activate: false
   - name: bloaty
 
 - name: ubuntu2004-asan
@@ -1295,7 +1320,6 @@ buildvariants:
     enable_fuzzer: On
   tasks:
   - name: fuzzer-tests
-    activate: false
 
 - name: rhel70
   display_name: "RHEL 7 x86_64"
@@ -1323,7 +1347,6 @@ buildvariants:
     cmake_build_type: RelWithDebInfo
   tasks:
   - name: valgrind
-    activate: false
 
 - name: ubuntu2004-arm64
   display_name: "Ubuntu 20.04 ARM64"
@@ -1352,6 +1375,7 @@ buildvariants:
   tasks:
   - name: compile_test
   - name: swift-build-and-test
+  - name: validate-installed-headers
 
 - name: macos-encrypted
   display_name: "MacOS 11.0 x86_64 (Encryption enabled)"

src/realm/array_basic_tpl.hpp

@@ -21,9 +21,8 @@
 
 #include <algorithm>
 #include <limits>
-#include <stdexcept>
-#include <iomanip>
 
+#include <realm/array_basic.hpp>
 #include <realm/node.hpp>
 
 namespace realm {

src/realm/array_bool.hpp

@@ -20,6 +20,7 @@
 #define REALM_ARRAY_BOOL_HPP
 
 #include <realm/array.hpp>
+#include <realm/mixed.hpp>
 
 namespace realm {
 

src/realm/array_list.hpp

@@ -20,6 +20,7 @@
 #define REALM_ARRAY_LIST_HPP
 
 #include <realm/array.hpp>
+#include <realm/mixed.hpp>
 
 namespace realm {
 

src/realm/array_typed_link.hpp

@@ -19,9 +19,10 @@
 #ifndef REALM_ARRAY_TYPED_LINK_HPP
 #define REALM_ARRAY_TYPED_LINK_HPP
 
+#include <realm/array.hpp>
 #include <realm/data_type.hpp>
 #include <realm/keys.hpp>
-#include <realm/array.hpp>
+#include <realm/mixed.hpp>
 
 namespace realm {
 

src/realm/object-store/util/copyable_atomic.hpp

@@ -19,8 +19,9 @@
 #ifndef REALM_OS_COPYABLE_ATOMIC_HPP
 #define REALM_OS_COPYABLE_ATOMIC_HPP
 
-namespace realm {
-namespace util {
+#include <atomic>
+
+namespace realm::util {
 
 // std::atomic is not copyable because the resulting semantics are not useful
 // for many of the things atomics can be used for (in particular, anything
@@ -51,6 +52,5 @@ struct CopyableAtomic : std::atomic<T> {
     }
 };
 
-} // namespace util
-} // namespace realm
+} // namespace realm::util
 #endif // REALM_OS_COPYABLE_ATOMIC_HPP

src/realm/query.cpp

@@ -1398,9 +1398,12 @@ TableView Query::find_all(size_t limit) const
 
     TableView ret(*this, limit);
     if (m_ordering) {
+        // apply_descriptor_ordering will call do_sync
         ret.apply_descriptor_ordering(*m_ordering);
     }
-    ret.do_sync();
+    else {
+        ret.do_sync();
+    }
     return ret;
 }
 

src/realm/sort_descriptor.cpp

@@ -301,7 +301,29 @@ BaseDescriptor::Sorter SortDescriptor::sorter(Table const& table, const IndexPai
 
 void SortDescriptor::execute(IndexPairs& v, const Sorter& predicate, const BaseDescriptor* next) const
 {
-    std::sort(v.begin(), v.end(), std::ref(predicate));
+    size_t limit = size_t(-1);
+    if (next && next->get_type() == DescriptorType::Limit) {
+        limit = static_cast<const LimitDescriptor*>(next)->get_limit();
+    }
+    // Measurements shows that if limit is smaller than size / 16, then
+    // it is quicker to make a sorted insert into a smaller vector
+    if (limit < (v.size() >> 4)) {
+        IndexPairs buffer;
+        buffer.reserve(limit + 1);
+        for (auto& elem : v) {
+            auto it = std::lower_bound(buffer.begin(), buffer.end(), elem, std::ref(predicate));
+            buffer.insert(it, elem);
+            if (buffer.size() > limit) {
+                buffer.pop_back();
+            }
+        }
+        v.m_removed_by_limit += v.size() - limit;
+        v.erase(v.begin() + limit, v.end());
+        std::move(buffer.begin(), buffer.end(), v.begin());
+    }
+    else {
+        std::sort(v.begin(), v.end(), std::ref(predicate));
+    }
 
     // not doing this on the last step is an optimisation
     if (next) {

src/realm/sync/CMakeLists.txt

@@ -60,6 +60,7 @@ set(SYNC_NETWORK_INSTALL_HEADERS
     network/network.hpp
     network/network_ssl.hpp
     network/websocket.hpp
+    network/websocket_error.hpp
 )
 
 set(NOINST_HEADERS

src/realm/util/serializer.cpp

@@ -27,11 +27,10 @@
 #include <realm/table.hpp>
 #include <realm/timestamp.hpp>
 #include <realm/util/base64.hpp>
-#include <realm/table.hpp>
 
 #include <cctype>
 #include <cmath>
-
+#include <iomanip>
 
 namespace realm {
 

test/benchmark-common-tasks/main.cpp

@@ -1391,6 +1391,49 @@ struct BenchmarkSortIntDictionary : Benchmark {
     std::vector<size_t> m_indices;
 };
 
+struct BenchmarkSortThenLimit : Benchmark {
+    const char* name() const
+    {
+        return "SortThenLimit";
+    }
+
+    void before_all(DBRef group)
+    {
+        WriteTransaction tr(group);
+        TableRef t = tr.add_table(name());
+        m_col = t->add_column(type_Int, "first");
+
+        std::vector<int> values(10000);
+        std::iota(values.begin(), values.end(), 0);
+        std::shuffle(values.begin(), values.end(), std::mt19937(std::random_device()()));
+
+        for (auto i : values) {
+            t->create_object().set(m_col, i);
+        }
+        tr.commit();
+    }
+
+    void after_all(DBRef db)
+    {
+        WriteTransaction tr(db);
+        tr.get_group().remove_table(name());
+        tr.commit();
+    }
+
+    void operator()(DBRef db)
+    {
+        realm::ReadTransaction tr(db);
+        auto tv = tr.get_group().get_table(name())->where().find_all();
+        DescriptorOrdering ordering;
+        ordering.append_sort(SortDescriptor({{m_col}}));
+        ordering.append_limit(100);
+
+        tv.apply_descriptor_ordering(ordering);
+    }
+
+    ColKey m_col;
+};
+
 struct BenchmarkInsert : BenchmarkWithStringsTable {
     const char* name() const
     {
@@ -2249,7 +2292,6 @@ int benchmark_common_tasks_main()
 
 #define BENCH(B) run_benchmark<B>(results)
 #define BENCH2(B, mode) run_benchmark<B>(results, mode)
-
     BENCH2(BenchmarkEmptyCommit, true);
     BENCH2(BenchmarkEmptyCommit, false);
     BENCH2(BenchmarkNonInitiatorOpen, true);
@@ -2267,6 +2309,7 @@ int benchmark_common_tasks_main()
     BENCH(BenchmarkSortInt);
     BENCH(BenchmarkSortIntList);
     BENCH(BenchmarkSortIntDictionary);
+    BENCH(BenchmarkSortThenLimit);
 
     BENCH(BenchmarkUnorderedTableViewClear);
     BENCH(BenchmarkUnorderedTableViewClearIndexed);