Port the audit SDK from realm-cocoa-private

.clang-format

@@ -1,12 +1,5 @@
 ---
 BasedOnStyle: LLVM
-TabWidth: 4
-UseTab: Never
----
-Language:  JavaScript
-DisableFormat: true
----
-Language:        Cpp
 AccessModifierOffset: -4
 AlignAfterOpenBracket: Align
 AlignConsecutiveMacros: false

CHANGELOG.md

@@ -10,15 +10,18 @@
 * Expose Subscription properties on C API. ([#5454](https://github.com/realm/realm-core/pull/5454))
 * Erase Subscription by id for C API. ([#5475]https://github.com/realm/realm-core/issues/5475)
 * Erase and Find Subscription by Results for C API. ([#5470](https://github.com/realm/realm-core/issues/5470))
+* Expose Subscription properties on C-API ([#5454](https://github.com/realm/realm-core/pull/5454))
+* Move the implementation of the Audit API to the open-source repo and update it to work with MongoDB Realm.
 
 ### Fixed
 * C API client reset callbacks don't leak the `realm_t` parameter. ([#5464](https://github.com/realm/realm-core/pull/5464))
 * The sync client may have sent a corrupted upload cursor leading to a fatal error from the server due to an uninitialized variable. ([#5460](https://github.com/realm/realm-core/pull/5460), since v11.14.0)
 * The realm_async_open_task_start() in C API was not really useful as the received realm reference could not be transferred to another thread. ([#5465](https://github.com/realm/realm-core/pull/5465), since v11.5.0)
 * FLX sync would not correctly resume syncing if a bootstrap was interrupted ([#5466](https://github.com/realm/realm-core/pull/5466), since v11.8.0)
- 
+
 ### Breaking changes
-* Extra `realm_free_userdata_func_t` parameter added on some realm_config_set_... functions in the C API. The userdata will be freed when the config object is freed.
+* Extra `realm_free_userdata_func_t` parameter added on some realm_config_set... functions in the C API. The userdata will be freed when the config object is freed.
+* `realm::Realm::Config` has been renamed to `realm::RealmConfig`.
 
 ### Compatibility
 * Fileformat: Generates files with format v22. Reads and automatically upgrade from fileformat v5.

Package.swift

@@ -423,6 +423,7 @@ let headers: [String] = [
     "realm/obj.hpp",
     "realm/obj_list.hpp",
     "realm/object-store/audit.hpp",
+    "realm/object-store/audit_serializer.hpp",
     "realm/object-store/binding_callback_thread_observer.hpp",
     "realm/object-store/binding_context.hpp",
     "realm/object-store/c_api/conversion.hpp",
@@ -824,6 +825,7 @@ let package = Package(
                 "sync-1.x.realm",
                 "sync-metadata-v4.realm",
                 "sync-metadata-v5.realm",
+                "sync/flx_sync_harness.hpp",
                 "sync/session/session_util.hpp",
                 "sync/sync_test_utils.hpp",
                 "test_backup-olden-and-golden.realm",

src/realm/db.cpp

@@ -1357,7 +1357,7 @@ void Transaction::replicate(Transaction* dest, Replication& repl) const
 
 void Transaction::copy_to(TransactionRef dest) const
 {
-    impl::CopyReplication repl(dest);
+    _impl::CopyReplication repl(dest);
     replicate(dest.get(), repl);
 }
 

src/realm/db.hpp

@@ -633,6 +633,7 @@ class Transaction : public Group {
 
     // Live transactions state changes, often taking an observer functor:
     VersionID commit_and_continue_as_read(bool commit_to_disk = true) REQUIRES(!m_async_mutex);
+    void commit_and_continue_writing();
     template <class O>
     void rollback_and_continue_as_read(O* observer) REQUIRES(!m_async_mutex);
     void rollback_and_continue_as_read() REQUIRES(!m_async_mutex)
@@ -751,7 +752,6 @@ class Transaction : public Group {
     bool internal_advance_read(O* observer, VersionID target_version, _impl::History&, bool);
     void set_transact_stage(DB::TransactStage stage) noexcept;
     void do_end_read() noexcept REQUIRES(!m_async_mutex);
-    void commit_and_continue_writing();
     void initialize_replication();
 
     void replicate(Transaction* dest, Replication& repl) const;

src/realm/impl/copy_replication.cpp

@@ -16,12 +16,13 @@
  *
  **************************************************************************/
 
-#include "copy_replication.hpp"
+#include <realm/impl/copy_replication.hpp>
+
+#include <realm/dictionary.hpp>
 #include <realm/list.hpp>
 #include <realm/set.hpp>
-#include <realm/dictionary.hpp>
 
-namespace realm::impl {
+namespace realm::_impl {
 
 void CopyReplication::add_class(TableKey, StringData name, bool is_embedded)
 {
@@ -276,4 +277,4 @@ Mixed CopyReplication::handle_link(ColKey col_key, Mixed val, util::FunctionRef<
     return {};
 }
 
-} // namespace realm::impl
+} // namespace realm::_impl

src/realm/impl/copy_replication.hpp

@@ -22,7 +22,7 @@
 #include <realm/replication.hpp>
 #include <realm/db.hpp>
 
-namespace realm::impl {
+namespace realm::_impl {
 
 class CopyReplication : public Replication {
 public:
@@ -83,6 +83,6 @@ class CopyReplication : public Replication {
     std::map<const Table*, Table*> m_table_map;
 };
 
-} // namespace realm::impl
+} // namespace realm::_impl
 
 #endif /* REALM_COPY_REPLICATION_HPP */

src/realm/obj.hpp

@@ -159,6 +159,7 @@ class Obj {
     bool is_unresolved(ColKey col_key) const;
 
     size_t get_link_count(ColKey col_key) const;
+    TableRef get_target_table(ColKey col_key) const;
 
     bool is_null(ColKey col_key) const;
     bool is_null(StringData col_name) const
@@ -369,7 +370,6 @@ class Obj {
     ColKey get_column_key(StringData col_name) const;
     ColKey get_primary_key_column() const;
     TableKey get_table_key() const;
-    TableRef get_target_table(ColKey col_key) const;
     TableRef get_target_table(ObjLink link) const;
     const Spec& get_spec() const;
 

src/realm/object-store/CMakeLists.txt

@@ -2,18 +2,18 @@ add_subdirectory(c_api)
 
 set(SOURCES
     binding_callback_thread_observer.cpp
-    collection_notifications.cpp
-    index_set.cpp
     collection.cpp
+    collection_notifications.cpp
     dictionary.cpp
+    index_set.cpp
     list.cpp
     object.cpp
     object_changeset.cpp
     object_schema.cpp
     object_store.cpp
     results.cpp
-    set.cpp
     schema.cpp
+    set.cpp
     shared_realm.cpp
     thread_safe_reference.cpp
 
@@ -35,14 +35,16 @@ set(SOURCES
     util/uv/scheduler.hpp)
 
 set(HEADERS
+    audit.hpp
+    audit_serializer.hpp
     binding_callback_thread_observer.hpp
     binding_context.hpp
+    collection.hpp
     collection_notifications.hpp
+    dictionary.hpp
     feature_checks.hpp
     index_set.hpp
     keypath_helpers.hpp
-    collection.hpp
-    dictionary.hpp
     list.hpp
     object.hpp
     object_accessor.hpp
@@ -51,8 +53,8 @@ set(HEADERS
     object_store.hpp
     property.hpp
     results.hpp
-    set.hpp
     schema.hpp
+    set.hpp
     shared_realm.hpp
     thread_safe_reference.hpp
 
@@ -134,12 +136,12 @@ if(REALM_ENABLE_SYNC)
         util/bson/bson.cpp
         util/bson/regular_expression.cpp)
     if(APPLE)
-
         list(APPEND HEADERS
             sync/impl/apple/network_reachability_observer.hpp
             sync/impl/apple/system_configuration.hpp)
 
         list(APPEND SOURCES
+            audit.mm
             sync/impl/apple/network_reachability_observer.cpp
             sync/impl/apple/system_configuration.cpp)
     endif()

src/realm/object-store/audit.hpp

@@ -1,6 +1,6 @@
 ////////////////////////////////////////////////////////////////////////////
 //
-// Copyright 2018 Realm Inc.
+// Copyright 2022 Realm Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,20 +16,109 @@
 //
 ////////////////////////////////////////////////////////////////////////////
 
+#ifndef REALM_AUDIT_HPP
+#define REALM_AUDIT_HPP
+
+#include <realm/util/functional.hpp>
+#include <realm/util/optional.hpp>
+#include <realm/util/terminate.hpp>
+
+#include <memory>
+#include <string>
+#include <vector>
+
 namespace realm {
-class Table;
+class DB;
+class AuditObjectSerializer;
+class Obj;
+class SyncUser;
 class TableView;
-template <typename>
-class BasicRowExpr;
-using RowExpr = BasicRowExpr<Table>;
+class Timestamp;
+struct ColKey;
+struct RealmConfig;
+struct SyncError;
 struct VersionID;
+namespace util {
+class Logger;
+}
+
+struct AuditConfig {
+    // User to open the audit Realms with. If null, the user used to open the
+    // Realm being audited is used
+    std::shared_ptr<SyncUser> audit_user;
+    // Prefix added to the start of generated partition keys for audit Realms.
+    std::string partition_value_prefix = "audit";
+    // Object serializer instance for converting objects to JSON payloads. If
+    // null, a default implementation is used.
+    std::shared_ptr<AuditObjectSerializer> serializer;
+    // Logger for audit events. If null, the sync logger from the Realm under
+    // audit is used.
+    std::shared_ptr<util::Logger> logger;
+    // Error handler which is called if fatal sync errors occur on the sync
+    // Realm. If null, an error is logged then abort() is called.
+    std::function<void(SyncError)> sync_error_handler;
+    // Metadata to attach to each audit event. Each key used must be a property
+    // in the server-side schema for AuditEvent. This is not validated and will
+    // result in a sync error if violated.
+    std::vector<std::pair<std::string, std::string>> metadata;
+};
 
 class AuditInterface {
 public:
-    virtual ~AuditInterface() {}
+    virtual ~AuditInterface() = default;
+
+    // Internal interface for recording auditable events. SDKs may need to call
+    // record_read() if they do not go through Object; record_query() and
+    // record_write() should be handled automatically
+    virtual void record_query(VersionID, const TableView&) = 0;
+    virtual void record_read(VersionID, const Obj& obj, const Obj& parent, ColKey col) = 0;
+    virtual void record_write(VersionID old_version, VersionID new_version) = 0;
+
+    // -- Audit functionality which should be exposed in the SDK
+
+    // Update the metadata attached to subsequence audit events. Does not effect
+    // the current audit scope if called while a scope is active.
+    virtual void update_metadata(std::vector<std::pair<std::string, std::string>> new_metadata) = 0;
 
-    virtual void record_query(realm::VersionID, realm::TableView const&) = 0;
-    virtual void record_read(realm::VersionID, realm::RowExpr) = 0;
-    virtual void record_write(realm::VersionID, realm::VersionID) = 0;
+    // Begin an audit scope. The given `name` is stored in the activity field
+    // of each generated event.
+    virtual void begin_scope(std::string_view name) = 0;
+    // End the current scope and asynchronously save it to disk. The optional
+    // completion function is called once it has been committed (or an error
+    // ocurred while trying to do so).
+    virtual void end_scope(util::UniqueFunction<void(std::exception_ptr)>&& completion = nullptr) = 0;
+    // Record a custom audit event. Does not use the scope (and does not need to be inside a scope).
+    virtual void record_event(std::string_view activity, util::Optional<std::string> event_type,
+                              util::Optional<std::string> data,
+                              util::UniqueFunction<void(std::exception_ptr)>&& completion) = 0;
+
+    // -- Test helper functionality
+
+    // Wait for all scopes to be written to disk. Does not wait for them to be
+    // uploaded to the server.
+    virtual void wait_for_completion() = 0;
+    // Wait for there to be no more data to upload. This is not a precise check;
+    // if more scopes are created while this is waiting they may or may not be
+    // included in the wait.
+    virtual void wait_for_uploads() = 0;
 };
+
+std::shared_ptr<AuditInterface> make_audit_context(std::shared_ptr<DB>, RealmConfig const& parent_config);
+
+// Hooks for testing. Do not use outside of tests.
+namespace audit_test_hooks {
+void set_maximum_shard_size(int64_t max_size);
+// Not thread-safe, so this must be called at a point when no audit contexts exist.
+void set_clock(util::UniqueFunction<Timestamp()>&&);
+} // namespace audit_test_hooks
+
+#if !REALM_PLATFORM_APPLE
+inline std::shared_ptr<AuditInterface> make_audit_context(std::shared_ptr<DB>, RealmConfig const&)
+{
+    REALM_TERMINATE("Audit not supported on this platform");
+}
+#endif
+
 } // namespace realm
+
+#endif // REALM_AUDIT_HPP