diff --git a/C/Cpp_include/c4Base.hh b/C/Cpp_include/c4Base.hh index 2f838fbe8..05161acf1 100644 --- a/C/Cpp_include/c4Base.hh +++ b/C/Cpp_include/c4Base.hh @@ -72,11 +72,6 @@ namespace litecore { class PublicKey; } // namespace crypto - namespace REST { - class Listener; - class RESTListener; - } // namespace REST - namespace websocket { class WebSocket; } diff --git a/C/Cpp_include/c4Collection.hh b/C/Cpp_include/c4Collection.hh index 10009f3d4..f333940cc 100644 --- a/C/Cpp_include/c4Collection.hh +++ b/C/Cpp_include/c4Collection.hh @@ -55,6 +55,7 @@ struct C4Collection virtual uint64_t getDocumentCount() const = 0; virtual C4SequenceNumber getLastSequence() const = 0; + virtual uint64_t getPurgeCount() const = 0; C4ExtraInfo& extraInfo() noexcept { return _extraInfo; } @@ -93,7 +94,8 @@ struct C4Collection /// Same as the C4Database method, but the query will refer to this collection by default. Retained newQuery(C4QueryLanguage language, slice queryExpr, int* outErrorPos) const; - virtual void createIndex(slice name, slice indexSpec, C4QueryLanguage indexLanguage, C4IndexType indexType, + /// Returns true if it created or replaced the index, false if it already exists. + virtual bool createIndex(slice name, slice indexSpec, C4QueryLanguage indexLanguage, C4IndexType indexType, const C4IndexOptions* C4NULLABLE indexOptions = nullptr) = 0; virtual Retained getIndex(slice name) = 0; diff --git a/C/Cpp_include/c4Database.hh b/C/Cpp_include/c4Database.hh index dfea19b55..ce8910044 100644 --- a/C/Cpp_include/c4Database.hh +++ b/C/Cpp_include/c4Database.hh @@ -56,10 +56,10 @@ struct C4Database /** Attempts to discover and verify the named extension in the provided path */ static void enableExtension(slice name, slice path); - static bool exists(slice name, slice inDirectory); - static void copyNamed(slice sourcePath, slice destinationName, const Config&); - static bool deleteNamed(slice name, slice inDirectory); - static bool deleteAtPath(slice path); + static bool exists(slice name, slice inDirectory); + static void copyNamed(slice sourcePath, slice destinationName, const Config&); + [[nodiscard]] static bool deleteNamed(slice name, slice inDirectory); + [[nodiscard]] static bool deleteAtPath(slice path); static Retained openNamed(slice name, const Config&); @@ -184,6 +184,8 @@ struct C4Database db->endTransaction(false); } + bool isActive() const noexcept { return _db != nullptr; } + ~Transaction() { if ( _db ) _db->endTransaction(false); } diff --git a/C/Cpp_include/c4DocEnumerator.hh b/C/Cpp_include/c4DocEnumerator.hh index 48841f828..d134fb311 100644 --- a/C/Cpp_include/c4DocEnumerator.hh +++ b/C/Cpp_include/c4DocEnumerator.hh @@ -35,6 +35,13 @@ struct C4DocEnumerator explicit C4DocEnumerator(C4Collection* collection, const C4EnumeratorOptions& options = kC4DefaultEnumeratorOptions); + /// Creates an enumerator on a collection, beginning at `startKey`. + /// (This means that if the order is descending, `startKey` will be the maximum key.) + /// If `startKey` is null, it's ignored and all documents are returned. + /// You must first call \ref next to step to the first document. + explicit C4DocEnumerator(C4Collection* collection, slice startKey, + const C4EnumeratorOptions& options = kC4DefaultEnumeratorOptions); + /// Creates an enumerator on a collection, ordered by sequence. /// You must first call \ref next to step to the first document. explicit C4DocEnumerator(C4Collection* collection, C4SequenceNumber since, diff --git a/C/Cpp_include/c4Listener.hh b/C/Cpp_include/c4Listener.hh index 0e35a19a0..b7f5c4887 100644 --- a/C/Cpp_include/c4Listener.hh +++ b/C/Cpp_include/c4Listener.hh @@ -13,6 +13,7 @@ #pragma once #include "c4Base.hh" #include "c4ListenerTypes.h" +#include "fleece/FLBase.h" #include "fleece/InstanceCounted.hh" #include @@ -24,41 +25,76 @@ C4_ASSUME_NONNULL_BEGIN // the dynamic library only exports the C API. // ************************************************************************ +namespace litecore::REST { + class HTTPListener; +} +/** A lightweight server that shares databases over the network for replication. + @note This class is not ref-counted. Instances must be explicitly deleted/destructed. */ struct C4Listener final : public fleece::InstanceCounted , C4Base { - static C4ListenerAPIs availableAPIs(); - - explicit C4Listener(C4ListenerConfig config); - - ~C4Listener() override; - - bool shareDB(slice name, C4Database* db); - + /// Constructor. Starts the listener (asynchronously) but does not share any databases. + explicit C4Listener(C4ListenerConfig const& config); + + ~C4Listener() noexcept override; + + /// Stops the listener. If you don't call this, the destructor will do it for you. + C4Error stop() noexcept; + + /// Shares a database, and its default collection. + /// @param name The URI name (first path component) in the HTTP API. + /// If `nullslice`, the C4Database's name will be used (possibly URL-escaped). + /// The name may not include '/', '.', control characters, or non-ASCII characters. + /// @param db The database to share. On success this instance is now managed by the Listener + /// and should not be used again by the caller. + /// @param dbConfig Optional configuration for this database. Overrides the C4ListenerConfig. + /// @returns True on success, false if the name is already in use. + [[nodiscard]] bool shareDB(slice name, C4Database* db, + C4ListenerDatabaseConfig const* C4NULLABLE dbConfig = nullptr); + + /// Stops sharing a database. `db` need not be the exact instance that was registered; + /// any instance on the same database file will work. bool unshareDB(C4Database* db); - bool shareCollection(slice name, C4Collection* coll); - - bool unshareCollection(slice name, C4Collection* coll); - + /// Adds a collection to be shared. + /// @note A database's default collection is automatically shared. + /// @param name The URI name the database is registered by. + /// @param collection The collection instance to share. + /// @returns True on success, false if `name` is not registered. */ + [[nodiscard]] bool shareCollection(slice name, C4Collection* collection); + + /// Stops sharing a collection. + /// @note Call this after \ref registerDatabase if you don't want to share the default collection. + /// @param name The URI name the database is registered by. + /// @param collection The collection instance. + /// @returns True on success, false if the database name or collection is not registered. */ + bool unshareCollection(slice name, C4Collection* collection); + + /// The TCP port number for incoming connections. [[nodiscard]] uint16_t port() const; + /// Returns first the number of connections, and second the number of active connections. [[nodiscard]] std::pair connectionStatus() const; - std::vector URLs(C4Database* C4NULLABLE db, C4ListenerAPIs api) const; + /// Returns the URL(s) of a database being shared, or of the root. + /// The URLs will differ only in their hostname -- there will be one for each IP address or known + /// hostname of the computer, or of the network interface. + [[nodiscard]] std::vector URLs(C4Database* C4NULLABLE db) const; - static std::string URLNameFromPath(slice path); + /// A convenience that, given a filesystem path to a database, returns the database name + /// for use in an HTTP URI path. + [[nodiscard]] static std::string URLNameFromPath(slice path); C4Listener(const C4Listener&) = delete; + // internal use only + C4Listener(C4ListenerConfig const& config, Retained impl); + private: - // For some reason, MSVC on Jenkins will not compile this with noexcept (everything else will) - C4Listener(C4Listener&&); // NOLINT(performance-noexcept-move-constructor) + C4Listener(C4Listener&&) noexcept; - Retained _impl; - C4ListenerHTTPAuthCallback C4NULLABLE _httpAuthCallback; - void* C4NULLABLE _callbackContext; + Retained _impl; }; C4_ASSUME_NONNULL_END diff --git a/C/Cpp_include/c4Query.hh b/C/Cpp_include/c4Query.hh index bfae48e6b..2c2439664 100644 --- a/C/Cpp_include/c4Query.hh +++ b/C/Cpp_include/c4Query.hh @@ -18,6 +18,7 @@ #include #include #include +#include #include C4_ASSUME_NONNULL_BEGIN @@ -45,11 +46,12 @@ struct C4Query final int* C4NULLABLE outErrorPos); unsigned columnCount() const noexcept; - slice columnTitle(unsigned col) const; + slice columnTitle(unsigned col) const LIFETIMEBOUND; alloc_slice explain() const; - alloc_slice parameters() const noexcept; - void setParameters(slice parameters); + const std::set& parameterNames() const noexcept LIFETIMEBOUND; + alloc_slice parameters() const noexcept; + void setParameters(slice parameters); alloc_slice fullTextMatched(const C4FullTextMatch&); @@ -62,8 +64,8 @@ struct C4Query final [[nodiscard]] int64_t rowCount() const; void seek(int64_t rowIndex); - [[nodiscard]] FLArrayIterator columns() const; - [[nodiscard]] FLValue column(unsigned i) const; + [[nodiscard]] FLArrayIterator columns() const LIFETIMEBOUND; + [[nodiscard]] FLValue column(unsigned i) const LIFETIMEBOUND; [[nodiscard]] unsigned fullTextMatchCount() const; [[nodiscard]] C4FullTextMatch fullTextMatch(unsigned i) const; diff --git a/C/c4Base.cc b/C/c4Base.cc index 16ed83335..e0e4b0930 100644 --- a/C/c4Base.cc +++ b/C/c4Base.cc @@ -43,8 +43,8 @@ using namespace litecore; extern "C" { -CBL_CORE_API std::atomic_int gC4ExpectExceptions; -bool C4ExpectingExceptions(); +std::atomic_int gC4ExpectExceptions; +bool C4ExpectingExceptions(); bool C4ExpectingExceptions() { return gC4ExpectExceptions > 0; } // LCOV_EXCL_LINE } @@ -174,11 +174,11 @@ C4StringResult c4log_binaryFilePath(void) C4API { } // NOLINTBEGIN(misc-misplaced-const,cppcoreguidelines-interfaces-global-init) -CBL_CORE_API C4LogDomain const kC4DefaultLog = (C4LogDomain)&kC4Cpp_DefaultLog; -CBL_CORE_API C4LogDomain const kC4DatabaseLog = (C4LogDomain)&DBLog; -CBL_CORE_API C4LogDomain const kC4QueryLog = (C4LogDomain)&QueryLog; -CBL_CORE_API C4LogDomain const kC4SyncLog = (C4LogDomain)&SyncLog; -CBL_CORE_API C4LogDomain const kC4WebSocketLog = (C4LogDomain)&websocket::WSLogDomain; +C4LogDomain const kC4DefaultLog = (C4LogDomain)&kC4Cpp_DefaultLog; +C4LogDomain const kC4DatabaseLog = (C4LogDomain)&DBLog; +C4LogDomain const kC4QueryLog = (C4LogDomain)&QueryLog; +C4LogDomain const kC4SyncLog = (C4LogDomain)&SyncLog; +C4LogDomain const kC4WebSocketLog = (C4LogDomain)&websocket::WSLogDomain; // NOLINTEND(misc-misplaced-const,cppcoreguidelines-interfaces-global-init) diff --git a/C/c4BlobStore.cc b/C/c4BlobStore.cc index b13e9945a..6043f70d1 100644 --- a/C/c4BlobStore.cc +++ b/C/c4BlobStore.cc @@ -94,8 +94,7 @@ C4BlobStore::C4BlobStore(slice dirPath, C4DatabaseFlags flags, const C4Encryptio FilePath dir(_dirPath, ""); if ( dir.exists() ) { dir.mustExistAsDir(); - } else { - if ( !(flags & kC4DB_Create) ) error::_throw(error::NotFound); + } else if ( !(flags & kC4DB_ReadOnly) ) { dir.mkdir(); } } diff --git a/C/c4Certificate.cc b/C/c4Certificate.cc index 8d0f37750..5b9abee91 100644 --- a/C/c4Certificate.cc +++ b/C/c4Certificate.cc @@ -39,7 +39,7 @@ using namespace litecore::crypto; #ifdef COUCHBASE_ENTERPRISE -CBL_CORE_API const C4CertIssuerParameters kDefaultCertIssuerParameters = { +const C4CertIssuerParameters kDefaultCertIssuerParameters = { CertSigningRequest::kOneYear, C4STR("1"), -1, false, true, true, true}; diff --git a/C/c4Database.cc b/C/c4Database.cc index f07294e34..11a72ea40 100644 --- a/C/c4Database.cc +++ b/C/c4Database.cc @@ -36,9 +36,9 @@ using namespace fleece; using namespace litecore; -CBL_CORE_API const char* const kC4DatabaseFilenameExtension = ".cblite2"; +const char* const kC4DatabaseFilenameExtension = ".cblite2"; -CBL_CORE_API C4StorageEngine const kC4SQLiteStorageEngine = "SQLite"; +C4StorageEngine const kC4SQLiteStorageEngine = "SQLite"; C4EncryptionKey C4EncryptionKeyFromPassword(slice password, C4EncryptionAlgorithm alg) { C4EncryptionKey key; diff --git a/C/c4DocEnumerator.cc b/C/c4DocEnumerator.cc index 4b52b4f97..7c1c2d80b 100644 --- a/C/c4DocEnumerator.cc +++ b/C/c4DocEnumerator.cc @@ -24,34 +24,36 @@ using namespace litecore; #pragma mark - DOC ENUMERATION: -CBL_CORE_API const C4EnumeratorOptions kC4DefaultEnumeratorOptions = {kC4IncludeNonConflicted | kC4IncludeBodies}; +const C4EnumeratorOptions kC4DefaultEnumeratorOptions = {kC4IncludeNonConflicted | kC4IncludeBodies}; + +static RecordEnumerator::Options recordOptions(const C4EnumeratorOptions& c4options, slice startKey) { + RecordEnumerator::Options options; + if ( c4options.flags & kC4Descending ) options.sortOption = kDescending; + else if ( c4options.flags & kC4Unsorted ) + options.sortOption = kUnsorted; + options.includeDeleted = (c4options.flags & kC4IncludeDeleted) != 0; + options.onlyConflicts = (c4options.flags & kC4IncludeNonConflicted) == 0; + if ( (c4options.flags & kC4IncludeBodies) == 0 ) options.contentOption = kMetaOnly; + else + options.contentOption = kEntireBody; + options.startKey = startKey; + return options; +} + +static RecordEnumerator::Options recordOptions(const C4EnumeratorOptions& c4options, C4SequenceNumber since) { + auto options = recordOptions(c4options, nullslice); + options.minSequence = since + 1; + return options; +} class C4DocEnumerator::Impl : public RecordEnumerator - , public fleece::InstanceCounted { + , public InstanceCounted { public: - Impl(C4Collection* collection, sequence_t since, const C4EnumeratorOptions& options) - : RecordEnumerator(asInternal(collection)->keyStore(), since, recordOptions(options)) + Impl(C4Collection* collection, const C4EnumeratorOptions& c4Options, const RecordEnumerator::Options& options) + : RecordEnumerator(asInternal(collection)->keyStore(), options) , _collection(asInternal(collection)) - , _options(options) {} - - Impl(C4Collection* collection, const C4EnumeratorOptions& options) - : RecordEnumerator(asInternal(collection)->keyStore(), recordOptions(options)) - , _collection(asInternal(collection)) - , _options(options) {} - - static RecordEnumerator::Options recordOptions(const C4EnumeratorOptions& c4options) { - RecordEnumerator::Options options; - if ( c4options.flags & kC4Descending ) options.sortOption = kDescending; - else if ( c4options.flags & kC4Unsorted ) - options.sortOption = kUnsorted; - options.includeDeleted = (c4options.flags & kC4IncludeDeleted) != 0; - options.onlyConflicts = (c4options.flags & kC4IncludeNonConflicted) == 0; - if ( (c4options.flags & kC4IncludeBodies) == 0 ) options.contentOption = kMetaOnly; - else - options.contentOption = kEntireBody; - return options; - } + , _c4Options(c4Options) {} Retained getDoc() { if ( !hasRecord() ) return nullptr; @@ -62,7 +64,8 @@ class C4DocEnumerator::Impl if ( !this->hasRecord() ) return false; revid vers(record().version()); - if ( (_options.flags & kC4IncludeRevHistory) && vers.isVersion() ) _docRevID = vers.asVersionVector().asASCII(); + if ( (_c4Options.flags & kC4IncludeRevHistory) && vers.isVersion() ) + _docRevID = vers.asVersionVector().asASCII(); else _docRevID = vers.expanded(); @@ -78,15 +81,18 @@ class C4DocEnumerator::Impl private: litecore::CollectionImpl* _collection; - C4EnumeratorOptions const _options; + C4EnumeratorOptions const _c4Options; alloc_slice _docRevID; }; -C4DocEnumerator::C4DocEnumerator(C4Collection* collection, C4SequenceNumber since, const C4EnumeratorOptions& options) - : _impl(new Impl(collection, since, options)) {} - C4DocEnumerator::C4DocEnumerator(C4Collection* collection, const C4EnumeratorOptions& options) - : _impl(new Impl(collection, options)) {} + : C4DocEnumerator(collection, nullslice, options) {} + +C4DocEnumerator::C4DocEnumerator(C4Collection* collection, slice startKey, const C4EnumeratorOptions& options) + : _impl(new Impl(collection, options, recordOptions(options, startKey))) {} + +C4DocEnumerator::C4DocEnumerator(C4Collection* collection, C4SequenceNumber since, const C4EnumeratorOptions& options) + : _impl(new Impl(collection, options, recordOptions(options, since))) {} #ifndef C4_STRICT_COLLECTION_API C4DocEnumerator::C4DocEnumerator(C4Database* database, const C4EnumeratorOptions& options) diff --git a/C/c4Document.cc b/C/c4Document.cc index afce76923..2914d4a49 100644 --- a/C/c4Document.cc +++ b/C/c4Document.cc @@ -208,30 +208,35 @@ Retained C4Document::update(slice revBody, C4RevisionFlags revFlags) // Sanity checks a document update request before writing to the database. bool C4Document::checkNewRev(slice parentRevID, C4RevisionFlags rqFlags, bool allowConflict, C4Error* outError) noexcept { - int code = 0; - if ( parentRevID ) { - // Updating an existing revision; make sure it exists and is a leaf: - if ( !exists() ) code = kC4ErrorNotFound; - else if ( !selectRevision(parentRevID, false) ) - code = allowConflict ? kC4ErrorNotFound : kC4ErrorConflict; - else if ( !allowConflict && !(_selected.flags & kRevLeaf) ) - code = kC4ErrorConflict; - } else { - // No parent revision given: - if ( rqFlags & kRevDeleted ) { - // Didn't specify a revision to delete: NotFound or a Conflict, depending - code = ((_flags & kDocExists) ? kC4ErrorConflict : kC4ErrorNotFound); - } else if ( (_flags & kDocExists) && !(_selected.flags & kRevDeleted) ) { - // If doc exists, current rev must be a deletion or there will be a conflict: - code = kC4ErrorConflict; + try { + int code = 0; + if ( parentRevID ) { + // Updating an existing revision; make sure it exists and is a leaf: + if ( !exists() ) code = kC4ErrorNotFound; + else if ( !selectRevision(parentRevID, false) ) + code = allowConflict ? kC4ErrorNotFound : kC4ErrorConflict; + else if ( !allowConflict && !(_selected.flags & kRevLeaf) ) + code = kC4ErrorConflict; + } else { + // No parent revision given: + if ( rqFlags & kRevDeleted ) { + // Didn't specify a revision to delete: NotFound or a Conflict, depending + code = ((_flags & kDocExists) ? kC4ErrorConflict : kC4ErrorNotFound); + } else if ( (_flags & kDocExists) && !(_selected.flags & kRevDeleted) ) { + // If doc exists, current rev must be a deletion or there will be a conflict: + code = kC4ErrorConflict; + } } - } - if ( code ) { - c4error_return(LiteCoreDomain, code, nullslice, outError); + if ( code ) { + c4error_return(LiteCoreDomain, code, nullslice, outError); + return false; + } + return true; + } catch ( ... ) { + if ( outError ) *outError = C4Error::fromCurrentException(); return false; } - return true; } #pragma mark - CONFLICTS: diff --git a/C/c4Query.cc b/C/c4Query.cc index d054640af..42371bdf7 100644 --- a/C/c4Query.cc +++ b/C/c4Query.cc @@ -26,7 +26,7 @@ using namespace fleece::impl; using namespace litecore; -CBL_CORE_API const C4QueryOptions kC4DefaultQueryOptions = {}; +const C4QueryOptions kC4DefaultQueryOptions = {}; C4Query::C4Query(C4Collection* coll, C4QueryLanguage language, slice queryExpression) : _database(asInternal(coll)->dbImpl()) @@ -64,6 +64,8 @@ alloc_slice C4Query::fullTextMatched(const C4FullTextMatch& term) { return _query->getMatchedText((Query::FullTextTerm&)term); } +const set& C4Query::parameterNames() const noexcept { return _query->parameterNames(); } + alloc_slice C4Query::parameters() const noexcept { LOCK(_mutex); return _parameters; diff --git a/C/c4_ee.exp b/C/c4_ee.exp index d71c64ef7..18aa8625e 100644 --- a/C/c4_ee.exp +++ b/C/c4_ee.exp @@ -274,7 +274,6 @@ __FLBuf_Release -_c4listener_availableAPIs _c4listener_start _c4listener_free _c4listener_shareDB diff --git a/C/include/c4Listener.h b/C/include/c4Listener.h index 4333e6448..8e08f19fe 100644 --- a/C/include/c4Listener.h +++ b/C/include/c4Listener.h @@ -20,86 +20,90 @@ C4API_BEGIN_DECLS /** \defgroup Listener Network Listener: REST API and Sync Server @{ */ -/** Returns flags for the available APIs in this build (REST, sync, or both.) */ -CBL_CORE_API C4ListenerAPIs c4listener_availableAPIs(void) C4API; - /** Creates and starts a new listener. Caller must release it when done. */ NODISCARD CBL_CORE_API C4Listener* C4NULLABLE c4listener_start(const C4ListenerConfig* config, C4Error* C4NULLABLE error) C4API; -/** Makes a database available from the network. - \note This function is thread-safe. - @param listener The listener that should share the database. - @param name The URI name to share it under, i.e. the path component in the URL. - If this is left null, a name will be chosen based as though you had called - \ref c4db_URINameFromPath. - @param db The database to share. - @param outError On failure, the error info is stored here if non-NULL. - @return True on success, false if the name is invalid as a URI component. */ +/** Makes a database available from the network, and its default collection. + This function is equivalent to `c4listener_shareDBWithConfig`, with `config` being `NULL`. */ NODISCARD CBL_CORE_API bool c4listener_shareDB(C4Listener* listener, C4String name, C4Database* db, C4Error* C4NULLABLE outError) C4API; -/** Makes a previously-shared database unavailable. - \note This function is thread-safe. */ +/** Makes a database available from the network, and its default collection. + @note The caller must use a lock for the C4Database when this function is called. + @param listener The listener that should share the database. + @param name The URI name to share it under, i.e. the path component in the URL. + If this is left null, a name will be chosen by calling \ref c4db_URINameFromPath. + The name may not include '/', '.', control characters, or non-ASCII characters. + @param db The database to share. + @param config Per-database configuration overriding the `C4ListenerConfig`, or `NULL`. + @param outError On failure, the error info is stored here if non-NULL. + @return True on success, false if the name is already in use or invalid as a URI component. */ +NODISCARD CBL_CORE_API bool c4listener_shareDBWithConfig(C4Listener* listener, C4String name, C4Database* db, + const C4ListenerDatabaseConfig* config, + C4Error* C4NULLABLE outError) C4API; + +/** Makes a previously-shared database unavailable. + @note `db` need not be the same instance that was registered, merely on the same file. + @note The caller must use a lock for the C4Database when this function is called. */ NODISCARD CBL_CORE_API bool c4listener_unshareDB(C4Listener* listener, C4Database* db, C4Error* C4NULLABLE outError) C4API; /** Specifies a collection to be used in a P2P listener context. NOTE: A database - must have been previously shared under the same name, or this operation will fail. - \note This function is thread-safe. - @param listener The listener that should share the collection. - @param name The URI name to share it under, this must match the name of an already - shared DB. - @param collection The collection to share. - @param outError On failure, the error info is stored here if non-NULL. */ + must have been previously shared under the same name, or this operation will fail. + @note The caller must use a lock for the C4Collection when this function is called. + @param listener The listener that should share the collection. + @param name The URI name to share it under, this must match the name of an already + shared DB. + @param collection The collection to share. + @param outError On failure, the error info is stored here if non-NULL. */ NODISCARD CBL_CORE_API bool c4listener_shareCollection(C4Listener* listener, C4String name, C4Collection* collection, C4Error* C4NULLABLE outError) C4API; /** Makes a previously-shared collection unavailable. - \note This function is thread-safe. */ + @note The caller must use a lock for the C4Collection when this function is called. */ NODISCARD CBL_CORE_API bool c4listener_unshareCollection(C4Listener* listener, C4String name, C4Collection* collection, C4Error* C4NULLABLE outError) C4API; -/** Returns the URL(s) of a database being shared, or of the root, separated by "\n" bytes. - The URLs will differ only in their hostname -- there will be one for each IP address or known - hostname of the computer, or of the network interface. - - WARNING: Link-local IPv6 addresses are included in this list. However, due to IPv6 specification - rules, a scope ID is also required to connect to these addresses. So if the address starts with fe80:: - you will need to take care on the other side to also incorporate the scope of of the client network interface - into the URL when connecting (in short, it's probably best to avoid these but they are there if - you would like to try) - \note This function is thread-safe. - @param listener The active listener. - @param db A database being shared, or NULL to get the listener's root URL(s). - @param api The API variant for which the URLs should be retrieved. If the listener is not running in the given mode, - or more than one mode is given, an error is returned - @param err The error information, if any - @return Fleece array of or more URL strings, or null if an error occurred. - Caller is responsible for releasing the result. */ +/** Returns the URL(s) of a database being shared, or of the root. + The URLs will differ only in their hostname -- there will be one for each IP address or known + hostname of the computer, or of the network interface. + + WARNING: Link-local IPv6 addresses are included in this list. However, due to IPv6 specification + rules, a scope ID is also required to connect to these addresses. So if the address starts with fe80:: + you will need to take care on the other side to also incorporate the scope of of the client network interface + into the URL when connecting (in short, it's probably best to avoid these but they are there if + you would like to try) + @note The caller must use a lock for the C4Database when this function is called. + @note The caller is responsible for releasing the returned Fleece array. + @param listener The active listener. + @param db A database being shared, or NULL to get the listener's root URL(s). + @param err The error information, if any + @return Fleece array of or more URL strings, or null if an error occurred. + Caller is responsible for releasing the result. */ NODISCARD CBL_CORE_API FLMutableArray c4listener_getURLs(const C4Listener* listener, C4Database* C4NULLABLE db, - C4ListenerAPIs api, C4Error* C4NULLABLE err) C4API; + C4Error* C4NULLABLE err) C4API; /** Returns the port number the listener is accepting connections on. - This is useful if you didn't specify a port in the config (`port`=0), so you can find out which - port the kernel picked. - \note This function is thread-safe. */ + This is useful if you didn't specify a port in the config (`port`=0), so you can find out which + port the kernel picked. + \note This function is thread-safe. */ CBL_CORE_API uint16_t c4listener_getPort(const C4Listener* listener) C4API; /** Returns the number of client connections, and how many of those are currently active. - Either parameter can be NULL if you don't care about it. - \note This function is thread-safe. */ + Either parameter can be NULL if you don't care about it. + \note This function is thread-safe. */ CBL_CORE_API void c4listener_getConnectionStatus(const C4Listener* listener, unsigned* C4NULLABLE connectionCount, unsigned* C4NULLABLE activeConnectionCount) C4API; /** A convenience that, given a filesystem path to a database, returns the database name - for use in an HTTP URI path. - - The directory portion of the path and the ".cblite2" extension are removed. - - Any leading "_" is replaced with a "-". - - Any control characters or slashes are replaced with "-". - @param path The filesystem path of a database. - @return A name that can be used as a URI path component, or NULL if the path is not a valid - database path (does not end with ".cblite2".) */ + for use in an HTTP URI path. + - The directory portion of the path and the ".cblite2" extension are removed. + - Any leading "_" is replaced with a "-". + - Any control characters or slashes are replaced with "-". + @param path The filesystem path of a database. + @return A name that can be used as a URI path component, or NULL if the path is not a valid + database path (does not end with ".cblite2".) */ CBL_CORE_API C4StringResult c4db_URINameFromPath(C4String path) C4API; diff --git a/C/include/c4ListenerTypes.h b/C/include/c4ListenerTypes.h index 4ed1464c1..47d9d9f93 100644 --- a/C/include/c4ListenerTypes.h +++ b/C/include/c4ListenerTypes.h @@ -20,13 +20,6 @@ C4API_BEGIN_DECLS @{ */ -/** Flags indicating which network API(s) to serve. */ -typedef C4_OPTIONS(unsigned, C4ListenerAPIs){ - kC4RESTAPI = 0x01, ///< CouchDB-like REST API - kC4SyncAPI = 0x02 ///< Replication server -}; - - /** Different ways to provide TLS private keys. */ typedef C4_ENUM(unsigned, C4PrivateKeyRepresentation){ kC4PrivateKeyFromCert, ///< Key in secure storage, associated with certificate @@ -52,7 +45,7 @@ typedef bool (*C4ListenerHTTPAuthCallback)(C4Listener* listener, C4Slice authHea /** TLS configuration for C4Listener. */ typedef struct C4TLSConfig { C4PrivateKeyRepresentation privateKeyRepresentation; ///< Interpretation of `privateKey` - C4KeyPair* key; ///< A key pair that contains the private key + C4KeyPair* C4NULLABLE key; ///< A key pair that contains the private key C4Cert* certificate; ///< X.509 certificate data bool requireClientCerts; ///< True to require clients to authenticate with a cert C4Cert* C4NULLABLE rootClientCerts; ///< Root CA certs to trust when verifying client cert @@ -64,25 +57,25 @@ typedef struct C4TLSConfig { typedef struct C4ListenerConfig { uint16_t port; ///< TCP port to listen on C4String networkInterface; ///< name or address of interface to listen on; else all - C4ListenerAPIs apis; ///< Which API(s) to enable C4TLSConfig* C4NULLABLE tlsConfig; ///< TLS configuration, or NULL for no TLS + C4String serverName; ///< Name for "Server:" response header (optional) + C4String serverVersion; ///< Version for "Server:" response header (optional) C4ListenerHTTPAuthCallback C4NULLABLE httpAuthCallback; ///< Callback for HTTP auth void* C4NULLABLE callbackContext; ///< Client value passed to HTTP auth callback - // For REST listeners only: - C4String directory; ///< Directory where newly-PUT databases will be created - bool allowCreateDBs; ///< If true, "PUT /db" is allowed - bool allowDeleteDBs; ///< If true, "DELETE /db" is allowed - bool allowCreateCollections; ///< If true, "PUT /db.scope.coll" is allowed - bool allowDeleteCollections; ///< If true, "DELETE /db.scope.coll" is allowed - - // For sync listeners only: bool allowPush; ///< Allow peers to push changes to local db bool allowPull; ///< Allow peers to pull changes from local db bool enableDeltaSync; ///< Enable document-deltas optimization } C4ListenerConfig; +/** Per-database configuration for a C4Listener. */ +typedef struct C4ListenerDatabaseConfig { + bool allowPush; ///< Allow peers to push changes to local db + bool allowPull; ///< Allow peers to pull changes from local db + bool enableDeltaSync; ///< Enable document-deltas optimization +} C4ListenerDatabaseConfig; + /** @} */ C4API_END_DECLS diff --git a/C/include/c4ReplicatorTypes.h b/C/include/c4ReplicatorTypes.h index f91c01aa2..fd4ac9321 100644 --- a/C/include/c4ReplicatorTypes.h +++ b/C/include/c4ReplicatorTypes.h @@ -236,7 +236,7 @@ typedef struct C4ReplicatorParameters { "onlySelfSignedServer" ///< Only accept self signed server certs (for P2P, bool) // HTTP options: -#define kC4ReplicatorOptionExtraHeaders "headers" ///< Extra HTTP headers (string[]) +#define kC4ReplicatorOptionExtraHeaders "headers" ///< Extra HTTP headers (Dict) #define kC4ReplicatorOptionCookies "cookies" ///< HTTP Cookie header value (string) #define kC4ReplicatorOptionAuthentication "auth" ///< Auth settings (Dict); see [1] #define kC4ReplicatorOptionProxyServer "proxy" ///< Proxy settings (Dict); see [3]] @@ -251,16 +251,17 @@ typedef struct C4ReplicatorParameters { #define kC4ReplicatorCompressionLevel "BLIPCompressionLevel" ///< Data compression level, 0..9 // [1]: Auth dictionary keys: -#define kC4ReplicatorAuthType "type" ///< Auth type; see [2] (string) -#define kC4ReplicatorAuthUserName "username" ///< User name for basic auth (string) -#define kC4ReplicatorAuthPassword "password" ///< Password for basic auth (string) -#define kC4ReplicatorAuthEnableChallengeAuth \ - "challengeAuth" ///< Use challenge auth instead of preemptive auth for basic auth, default is false (bool); \ - ///< Implemented by BuiltInWebSocket. +#define kC4ReplicatorAuthType "type" ///< Auth type; see [2] (string) +#define kC4ReplicatorAuthUserName "username" ///< User name for basic auth (string) +#define kC4ReplicatorAuthPassword "password" ///< Password for basic auth (string) #define kC4ReplicatorAuthClientCert "clientCert" ///< TLS client certificate (value platform-dependent) #define kC4ReplicatorAuthClientCertKey "clientCertKey" ///< Client cert's private key (data) #define kC4ReplicatorAuthToken "token" ///< Session cookie or auth token (string) +/// Use challenge auth instead of preemptive auth for basic auth, default is false (bool); +/// Implemented by BuiltInWebSocket. +#define kC4ReplicatorAuthEnableChallengeAuth "challengeAuth" + // [2]: auth.type values: #define kC4AuthTypeBasic "Basic" ///< HTTP Basic (the default) #define kC4AuthTypeSession "Session" ///< SG session cookie diff --git a/C/tests/CMakeLists.txt b/C/tests/CMakeLists.txt index 6688f22a9..6e7b233fb 100644 --- a/C/tests/CMakeLists.txt +++ b/C/tests/CMakeLists.txt @@ -117,6 +117,8 @@ target_include_directories( ${TOP}vendor/fleece/API ${TOP}vendor/fleece/Fleece/Support ${TOP}C + ${TOP}C/include + ${TOP}C/Cpp_include ${TOP}Crypto ${TOP}Replicator ${TOP}Replicator/tests diff --git a/C/tests/c4Test.cc b/C/tests/c4Test.cc index a078824f7..7a8a90aa6 100644 --- a/C/tests/c4Test.cc +++ b/C/tests/c4Test.cc @@ -78,11 +78,22 @@ ostream& operator<<(ostream& out, C4Error error) { } ERROR_INFO::~ERROR_INFO() { - if ( _error->code && OnMainThread() ) UNSCOPED_INFO(*_error); + if ( _error->code && OnMainThread() ) { + if ( OnMainThread() ) UNSCOPED_INFO(*_error); + else + std::cerr << "ERROR_INFO: " << *_error << '\n'; + } } WITH_ERROR::~WITH_ERROR() { - if ( _error->code ) std::cerr << "with error: " << *_error << '\n'; + if ( _error->code ) { + // Unfortunately it's too late to use UNSCOPED_INFO; since WITH_ERROR is used inside the + // CHECK/REQUIRE macro, by the time it's destructed Catch has already registered the error. + // But we can tell Catch to warn about it, which will show up below its failure message. + if ( OnMainThread() ) WARN(*_error); + else + std::cerr << "WITH_ERROR: " << *_error << '\n'; + } } void CheckError(C4Error error, C4ErrorDomain expectedDomain, int expectedCode, const char* expectedMessage) { diff --git a/C/tests/cmake/platform_linux.cmake b/C/tests/cmake/platform_linux.cmake index 82d1276e4..ab1a53802 100644 --- a/C/tests/cmake/platform_linux.cmake +++ b/C/tests/cmake/platform_linux.cmake @@ -1,8 +1,8 @@ function(setup_build) target_sources( C4Tests PRIVATE - ${TOP}Crypto/mbedUtils.cc - ${TOP}LiteCore/Unix/strlcat.c + ${TOP}Crypto/mbedUtils.cc + ${TOP}LiteCore/Unix/strlcat.c ) target_link_libraries( diff --git a/CMakeLists.txt b/CMakeLists.txt index 31e35c3df..d788c61b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,9 +37,7 @@ Platform logic is largely separated into the cmake/platform_*.cmake files. Plat - platform_win_desktop ]]# -cmake_minimum_required (VERSION 3.9) -cmake_policy(VERSION 3.9) -cmake_policy(SET CMP0057 NEW) +cmake_minimum_required (VERSION 3.20.3) # Mac/apple setup -- must appear before the first "project()" line" set(CMAKE_OSX_DEPLOYMENT_TARGET "12.0") @@ -96,6 +94,8 @@ option(DISABLE_LTO_BUILD "Disable build with Link-time optimization" OFF) option(LITECORE_BUILD_TESTS "Builds C4Tests and CppTests" ON) option(LITECORE_WARNINGS_HARDCORE "Enables tons of warnings and makes them errors (Clang only)" OFF) option(LITECORE_SANITIZE "Enables address and undefined-behavior sanitizers (Clang only)" OFF) +option(LITECORE_BUILD_SHARED "Enables building the LiteCore shared library" ON) +option(LITECORE_BUILD_STATIC "Enables building the LiteCore static library" ON) set(LITECORE_PREBUILT_LIB "" CACHE STRING "If set, C4Tests will use the prebuilt LiteCore instead of building it from source") @@ -209,7 +209,7 @@ target_compile_definitions( set_litecore_source(RESULT ALL_SRC_FILES) add_library(LiteCoreObjects OBJECT ${ALL_SRC_FILES}) -add_library(LiteCoreUnitTesting OBJECT ${ALL_SRC_FILES}) +add_library(LiteCoreUnitTesting OBJECT EXCLUDE_FROM_ALL ${ALL_SRC_FILES}) set(LiteCoreObjectsDefines LITECORE_IMPL LITECORE_CPP_API=1 @@ -276,11 +276,6 @@ target_include_directories( ${LiteCoreObjectsIncludes} ) -add_library(LiteCore SHARED $) - -add_library(LiteCoreStatic STATIC EXCLUDE_FROM_ALL $) -target_link_libraries(LiteCoreStatic PRIVATE LiteCoreObjects) - # Library flags defined in platform_linux set( LITECORE_LIBRARIES_PRIVATE @@ -298,12 +293,36 @@ if(BUILD_ENTERPRISE) ) endif() -target_include_directories( - LiteCore INTERFACE - C/include - C/Cpp_include -) -target_link_libraries(LiteCore PRIVATE ${LITECORE_LIBRARIES_PRIVATE}) +if(LITECORE_BUILD_SHARED) + add_library(LiteCore SHARED $) + target_include_directories( + LiteCore INTERFACE + C/include + C/Cpp_include + ) + target_link_libraries(LiteCore PRIVATE ${LITECORE_LIBRARIES_PRIVATE}) + + install ( + TARGETS LiteCore + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + OPTIONAL + ) + + file(GLOB LITECORE_HEADERS ${PROJECT_SOURCE_DIR}/C/include/*.h ${PROJECT_SOURCE_DIR}/C/Cpp_include/*.hh) + file(GLOB FLEECE_HEADERS ${PROJECT_SOURCE_DIR}/vendor/fleece/API/fleece/*.h ${PROJECT_SOURCE_DIR}/vendor/fleece/API/fleece/*.hh) + + install(FILES ${LITECORE_HEADERS} DESTINATION include) + install(FILES ${FLEECE_HEADERS} DESTINATION include/fleece) +endif() + +if(LITECORE_BUILD_STATIC) + add_library(LiteCoreStatic STATIC EXCLUDE_FROM_ALL $) + target_link_libraries(LiteCoreStatic PRIVATE LiteCoreObjects) +endif() + + target_link_libraries( LiteCoreObjects INTERFACE SQLite3_UnicodeSN @@ -330,20 +349,6 @@ if(USE_COUCHBASE_SQLITE) ) endif() -install ( - TARGETS LiteCore - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - OPTIONAL -) - -file(GLOB LITECORE_HEADERS ${PROJECT_SOURCE_DIR}/C/include/*.h ${PROJECT_SOURCE_DIR}/C/Cpp_include/*.hh) -file(GLOB FLEECE_HEADERS ${PROJECT_SOURCE_DIR}/vendor/fleece/API/fleece/*.h ${PROJECT_SOURCE_DIR}/vendor/fleece/API/fleece/*.hh) - -install(FILES ${LITECORE_HEADERS} DESTINATION include) -install(FILES ${FLEECE_HEADERS} DESTINATION include/fleece) - ### Support Libraries (Add functionality, but add nothing to official API) add_subdirectory(REST EXCLUDE_FROM_ALL) @@ -375,6 +380,7 @@ target_include_directories( C/include C/Cpp_include Crypto + LiteCore/Database LiteCore/Support Networking Networking/BLIP/ diff --git a/Crypto/Certificate.cc b/Crypto/Certificate.cc index 2c7e9635b..1ac26aa34 100644 --- a/Crypto/Certificate.cc +++ b/Crypto/Certificate.cc @@ -524,7 +524,9 @@ namespace litecore::crypto { Identity::Identity(Cert* cert_, PrivateKey* key_) : cert(cert_), privateKey(key_) { // Make sure the private and public keys match: - Assert(mbedtls_pk_check_pair(cert->subjectPublicKey()->context(), privateKey->context()) == 0); + if ( int err = mbedtls_pk_check_pair(cert->subjectPublicKey()->context(), privateKey->context()) ) { + throwMbedTLSError(err); + } } } // namespace litecore::crypto diff --git a/Crypto/PublicKey+Apple.mm b/Crypto/PublicKey+Apple.mm index 29752b3b6..cacf09270 100644 --- a/Crypto/PublicKey+Apple.mm +++ b/Crypto/PublicKey+Apple.mm @@ -14,6 +14,7 @@ #include "Certificate.hh" #include "PublicKey.hh" #include "TLSContext.hh" +#include "c4ExceptionUtils.hh" #include "c4Private.h" #include "Error.hh" #include "Logging.hh" @@ -277,9 +278,13 @@ virtual int _sign(int/*mbedtls_md_type_t*/ mbedDigestAlgorithm, // Create the signature: NSData* data = inputData.uncopiedNSData(); CFErrorRef error; - NSData* sigData = CFBridgingRelease( SecKeyCreateSignature(_privateKeyRef, - digestAlgorithm, - (CFDataRef)data, &error) ); + NSData* sigData; + { + ExpectingExceptions x; + sigData = CFBridgingRelease( SecKeyCreateSignature(_privateKeyRef, + digestAlgorithm, + (CFDataRef)data, &error) ); + } if (!sigData) { warnCFError(error, "SecKeyCreateSignature"); return MBEDTLS_ERR_RSA_PRIVATE_FAILED; diff --git a/Jenkinsfile b/Jenkinsfile index 323ee00f9..2d15aa8d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,8 +53,8 @@ pipeline { agent { label 's61113u16 (litecore)' } environment { BRANCH = "${BRANCH_NAME}" - CC = "gcc-10" - CXX = "g++-10" + CC = "gcc-11" + CXX = "g++-11" } steps { sh 'jenkins/jenkins_unix.sh' diff --git a/LiteCore/Database/CollectionImpl.hh b/LiteCore/Database/CollectionImpl.hh index 6692eebb9..df3170f83 100644 --- a/LiteCore/Database/CollectionImpl.hh +++ b/LiteCore/Database/CollectionImpl.hh @@ -90,6 +90,8 @@ namespace litecore { C4SequenceNumber getLastSequence() const override { return keyStore().lastSequence(); } + uint64_t getPurgeCount() const override { return keyStore().purgeCount(); } + DatabaseImpl* dbImpl() { return asInternal(getDatabase()); } const DatabaseImpl* dbImpl() const { return asInternal(getDatabase()); } @@ -393,7 +395,7 @@ namespace litecore { #pragma mark - INDEXES: - void createIndex(slice indexName, slice indexSpec, C4QueryLanguage indexLanguage, C4IndexType indexType, + bool createIndex(slice indexName, slice indexSpec, C4QueryLanguage indexLanguage, C4IndexType indexType, const C4IndexOptions* indexOptions = nullptr) override { IndexSpec::Options options; switch ( indexType ) { @@ -478,8 +480,8 @@ namespace litecore { error::_throw(error::InvalidParameter, "Invalid index type"); break; } - keyStore().createIndex(indexName, indexSpec, (QueryLanguage)indexLanguage, (IndexSpec::Type)indexType, - options); + return keyStore().createIndex(indexName, indexSpec, (QueryLanguage)indexLanguage, + (IndexSpec::Type)indexType, options); } Retained getIndex(slice name) override { return C4Index::getIndex(this, name); } diff --git a/LiteCore/Database/DatabaseImpl.cc b/LiteCore/Database/DatabaseImpl.cc index 4f554d277..b5d79173c 100644 --- a/LiteCore/Database/DatabaseImpl.cc +++ b/LiteCore/Database/DatabaseImpl.cc @@ -311,7 +311,7 @@ namespace litecore { // Split path into a separate variable to workaround GCC 8 constructor resolution issue auto path = alloc_slice(filePath().subdirectoryNamed(dirname)); auto flags = _config.flags; - if ( force ) { flags |= kC4DB_Create; } + if ( !(_config.flags & kC4DB_ReadOnly) || force ) { flags |= kC4DB_Create; } return make_unique(path, flags, encryptionKey); } diff --git a/LiteCore/Query/Query.hh b/LiteCore/Query/Query.hh index e3b1ce1e9..780920632 100644 --- a/LiteCore/Query/Query.hh +++ b/LiteCore/Query/Query.hh @@ -15,6 +15,7 @@ #include "Error.hh" #include "Logging.hh" #include +#include #include #include @@ -54,7 +55,9 @@ namespace litecore { virtual unsigned columnCount() const noexcept = 0; - virtual const std::vector& columnTitles() const noexcept = 0; + virtual const std::vector& columnTitles() const noexcept LIFETIMEBOUND = 0; + + virtual const std::set& parameterNames() const noexcept LIFETIMEBOUND = 0; virtual alloc_slice getMatchedText(const FullTextTerm&) = 0; @@ -140,7 +143,7 @@ namespace litecore { virtual bool hasFullText() const { return false; } - virtual const FullTextTerms& fullTextTerms() { return _fullTextTerms; } + virtual const FullTextTerms& fullTextTerms() LIFETIMEBOUND { return _fullTextTerms; } /** If the query results have changed since I was created, returns a new enumerator that will return the new results. Otherwise returns null. */ diff --git a/LiteCore/Query/SQLiteQuery.cc b/LiteCore/Query/SQLiteQuery.cc index 2c8f38216..7cc876859 100644 --- a/LiteCore/Query/SQLiteQuery.cc +++ b/LiteCore/Query/SQLiteQuery.cc @@ -167,6 +167,8 @@ namespace litecore { const vector& columnTitles() const noexcept override { return _columnTitles; } + const set& parameterNames() const noexcept override { return _parameters; } + string explain() override { stringstream result; // https://www.sqlite.org/eqp.html diff --git a/LiteCore/Storage/BothKeyStore.cc b/LiteCore/Storage/BothKeyStore.cc index d20bc0ee2..6c9a567dd 100644 --- a/LiteCore/Storage/BothKeyStore.cc +++ b/LiteCore/Storage/BothKeyStore.cc @@ -123,11 +123,10 @@ namespace litecore { // always returning the lowest-sorting record (basically a merge-sort.) class BothEnumeratorImpl final : public RecordEnumerator::Impl { public: - BothEnumeratorImpl(bool bySequence, sequence_t since, RecordEnumerator::Options options, KeyStore* liveStore, - KeyStore* deadStore) - : _liveImpl(liveStore->newEnumeratorImpl(bySequence, since, options)) - , _deadImpl(deadStore->newEnumeratorImpl(bySequence, since, options)) - , _bySequence(bySequence) + BothEnumeratorImpl(RecordEnumerator::Options options, KeyStore* liveStore, KeyStore* deadStore) + : _liveImpl(liveStore->newEnumeratorImpl(options)) + , _deadImpl(deadStore->newEnumeratorImpl(options)) + , _bySequence(options.minSequence > 0_seq) , _descending(options.sortOption == kDescending) {} bool next() override { @@ -181,17 +180,13 @@ namespace litecore { // indexes in `onlyConflicts` mode. class BothUnorderedEnumeratorImpl final : public RecordEnumerator::Impl { public: - BothUnorderedEnumeratorImpl(sequence_t since, RecordEnumerator::Options options, KeyStore* liveStore, - KeyStore* deadStore) - : _impl(liveStore->newEnumeratorImpl(false, since, options)) - , _deadStore(deadStore) - , _since(since) - , _options(options) {} + BothUnorderedEnumeratorImpl(RecordEnumerator::Options const& options, KeyStore* liveStore, KeyStore* deadStore) + : _impl(liveStore->newEnumeratorImpl(options)), _deadStore(deadStore), _options(options) {} bool next() override { bool ok = _impl->next(); if ( !ok && _deadStore != nullptr ) { - _impl = unique_ptr(_deadStore->newEnumeratorImpl(false, _since, _options)); + _impl = unique_ptr(_deadStore->newEnumeratorImpl(_options)); _deadStore = nullptr; ok = _impl->next(); } @@ -207,26 +202,25 @@ namespace litecore { private: unique_ptr _impl; // Current enumerator KeyStore* _deadStore; // The deleted store, before I switch to it - sequence_t _since; // Starting sequence RecordEnumerator::Options _options; // Enumerator options }; - RecordEnumerator::Impl* BothKeyStore::newEnumeratorImpl(bool bySequence, sequence_t since, - RecordEnumerator::Options options) { + RecordEnumerator::Impl* BothKeyStore::newEnumeratorImpl(RecordEnumerator::Options const& options) { bool isDefaultStore = (name() == DataFile::kDefaultKeyStoreName); if ( options.includeDeleted ) { if ( options.sortOption == kUnsorted ) - return new BothUnorderedEnumeratorImpl(since, options, _liveStore.get(), _deadStore.get()); + return new BothUnorderedEnumeratorImpl(options, _liveStore.get(), _deadStore.get()); else - return new BothEnumeratorImpl(bySequence, since, options, _liveStore.get(), _deadStore.get()); + return new BothEnumeratorImpl(options, _liveStore.get(), _deadStore.get()); } else { + auto optionsCopy = options; if ( !isDefaultStore ) { // For non default store, liveStore contains only live records. By assigning // includeDeleted to true, we won't apply flag filter to filter out the deleted. // For default store, however, liveStore may have deleted records. - options.includeDeleted = true; // no need for enum to filter out deleted docs + optionsCopy.includeDeleted = true; // no need for enum to filter out deleted docs } - return _liveStore->newEnumeratorImpl(bySequence, since, options); + return _liveStore->newEnumeratorImpl(optionsCopy); } } diff --git a/LiteCore/Storage/BothKeyStore.hh b/LiteCore/Storage/BothKeyStore.hh index 20b3ee27b..b7f2f6100 100644 --- a/LiteCore/Storage/BothKeyStore.hh +++ b/LiteCore/Storage/BothKeyStore.hh @@ -113,8 +113,7 @@ namespace litecore { _deadStore->close(); } - RecordEnumerator::Impl* newEnumeratorImpl(bool bySequence, sequence_t since, - RecordEnumerator::Options) override; + RecordEnumerator::Impl* newEnumeratorImpl(RecordEnumerator::Options const&) override; private: std::unique_ptr _liveStore; diff --git a/LiteCore/Storage/DataFile.cc b/LiteCore/Storage/DataFile.cc index c2aed626e..05830b9f6 100644 --- a/LiteCore/Storage/DataFile.cc +++ b/LiteCore/Storage/DataFile.cc @@ -130,6 +130,8 @@ namespace litecore { DataFile::DataFile(const FilePath& path, Delegate* delegate, const DataFile::Options* options) : Logging(DBLog), _delegate(delegate), _path(path), _options(options ? *options : Options::defaults) { + if ( _options.create && !_options.writeable ) // SQLite will complain + error::_throw(error::InvalidParameter, "invalid database flags: create and read-only"); // Do this last so I'm fully constructed before other threads can see me (#425) _shared = Shared::forPath(path, this); } @@ -390,6 +392,8 @@ namespace litecore { ExclusiveTransaction::ExclusiveTransaction(DataFile* db) : ExclusiveTransaction(db, true) {} ExclusiveTransaction::ExclusiveTransaction(DataFile* db, bool active) : _db(*db), _active(false) { + if ( active && !_db.options().writeable ) + error::_throw(error::NotWriteable, "Transaction on read-only database"); _db.beginTransactionScope(this); if ( active ) { _db._logVerbose("begin transaction"); diff --git a/LiteCore/Storage/KeyStore.hh b/LiteCore/Storage/KeyStore.hh index 5d21f96f3..92cf7f33d 100644 --- a/LiteCore/Storage/KeyStore.hh +++ b/LiteCore/Storage/KeyStore.hh @@ -219,8 +219,7 @@ namespace litecore { virtual void close() {} - virtual RecordEnumerator::Impl* newEnumeratorImpl(bool bySequence, sequence_t since, - RecordEnumerator::Options) = 0; + virtual RecordEnumerator::Impl* newEnumeratorImpl(RecordEnumerator::Options const&) = 0; DataFile& _db; // The DataFile I'm contained in const std::string _name; // My name diff --git a/LiteCore/Storage/RecordEnumerator.cc b/LiteCore/Storage/RecordEnumerator.cc index 5d47860a5..c6aabde75 100644 --- a/LiteCore/Storage/RecordEnumerator.cc +++ b/LiteCore/Storage/RecordEnumerator.cc @@ -24,18 +24,16 @@ namespace litecore { // By-key constructor - RecordEnumerator::RecordEnumerator(KeyStore& store, Options options) : _store(&store) { - LogVerbose(QueryLog, "RecordEnumerator %p: (%s, %d%d%d %d)", this, store.name().c_str(), options.includeDeleted, - options.onlyConflicts, options.onlyBlobs, options.sortOption); - _impl.reset(_store->newEnumeratorImpl(false, 0_seq, options)); - } - - // By-sequence constructor - RecordEnumerator::RecordEnumerator(KeyStore& store, sequence_t since, Options options) : _store(&store) { - LogVerbose(QueryLog, "RecordEnumerator %p: (%s, #%llu..., %d%d%d %d)", this, store.name().c_str(), - (unsigned long long)since, options.includeDeleted, options.onlyConflicts, options.onlyBlobs, - options.sortOption); - _impl.reset(_store->newEnumeratorImpl(true, since, options)); + RecordEnumerator::RecordEnumerator(KeyStore& store, Options const& options) : _store(&store) { + if ( options.minSequence != 0_seq ) { + LogVerbose(QueryLog, "RecordEnumerator %p: (%s, #%llu..., %d%d%d %d)", this, store.name().c_str(), + (unsigned long long)options.minSequence, options.includeDeleted, options.onlyConflicts, + options.onlyBlobs, options.sortOption); + } else { + LogVerbose(QueryLog, "RecordEnumerator %p: (%s, %d%d%d %d)", this, store.name().c_str(), + options.includeDeleted, options.onlyConflicts, options.onlyBlobs, options.sortOption); + } + _impl.reset(_store->newEnumeratorImpl(options)); } void RecordEnumerator::close() noexcept { diff --git a/LiteCore/Storage/RecordEnumerator.hh b/LiteCore/Storage/RecordEnumerator.hh index 10a5f8585..d816eee8f 100644 --- a/LiteCore/Storage/RecordEnumerator.hh +++ b/LiteCore/Storage/RecordEnumerator.hh @@ -39,12 +39,13 @@ namespace litecore { bool onlyConflicts = false; ///< Only include records with conflicts SortOption sortOption = kAscending; ///< Sort order, or unsorted ContentOption contentOption = kEntireBody; ///< Load record bodies? - - Options() {} + sequence_t minSequence{}; ///< Minimum sequence; if nonzero, orders by sequence + alloc_slice startKey; ///< Min key, or max key if descending }; - explicit RecordEnumerator(KeyStore&, Options options = Options()); - RecordEnumerator(KeyStore&, sequence_t since, Options options = Options()); + explicit RecordEnumerator(KeyStore& ks) : RecordEnumerator(ks, Options{}) {} + + RecordEnumerator(KeyStore&, Options const&); RecordEnumerator(RecordEnumerator&& e) noexcept { *this = std::move(e); } diff --git a/LiteCore/Storage/SQLiteEnumerator.cc b/LiteCore/Storage/SQLiteEnumerator.cc index bca8310b7..21c8d4d9e 100644 --- a/LiteCore/Storage/SQLiteEnumerator.cc +++ b/LiteCore/Storage/SQLiteEnumerator.cc @@ -48,8 +48,8 @@ namespace litecore { ContentOption _content; }; - RecordEnumerator::Impl* SQLiteKeyStore::newEnumeratorImpl(bool bySequence, sequence_t since, - RecordEnumerator::Options options) { + RecordEnumerator::Impl* SQLiteKeyStore::newEnumeratorImpl(RecordEnumerator::Options const& options) { + bool bySequence = (options.minSequence > 0_seq); if ( _db.options().writeable ) { if ( bySequence ) createSequenceIndex(); if ( options.onlyConflicts ) createConflictsIndex(); @@ -64,12 +64,14 @@ namespace litecore { sql << (mayHaveExpiration() ? ", expiration" : ", 0"); sql << " FROM " << quotedTableName(); - bool writeAnd = false; + bool writeAnd = true; if ( bySequence ) { - sql << " WHERE sequence > ?"; - writeAnd = true; + sql << " WHERE sequence >= ?"; + } else if ( options.startKey ) { + sql << ((options.sortOption != kDescending) ? " WHERE key >= ?" : " WHERE key <= ?"); } else { if ( !options.includeDeleted || options.onlyBlobs || options.onlyConflicts ) sql << " WHERE "; + writeAnd = false; } auto writeFlagTest = [&](DocumentFlags flag, const char* test) { @@ -103,7 +105,9 @@ namespace litecore { } - if ( bySequence ) stmt->bind(1, (long long)since); + if ( bySequence ) stmt->bind(1, (long long)options.minSequence); + else if ( options.startKey ) + stmt->bind(1, (const char*)options.startKey.buf, (int)options.startKey.size); return new SQLiteEnumerator(stmt, options.contentOption); } diff --git a/LiteCore/Storage/SQLiteKeyStore.hh b/LiteCore/Storage/SQLiteKeyStore.hh index 5c8dec232..ec5e72271 100644 --- a/LiteCore/Storage/SQLiteKeyStore.hh +++ b/LiteCore/Storage/SQLiteKeyStore.hh @@ -94,8 +94,7 @@ namespace litecore { protected: bool mayHaveExpiration() override; - RecordEnumerator::Impl* newEnumeratorImpl(bool bySequence, sequence_t since, - RecordEnumerator::Options) override; + RecordEnumerator::Impl* newEnumeratorImpl(RecordEnumerator::Options const&) override; std::unique_ptr compile(const char* sql) const; SQLite::Statement& compileCached(const std::string& sqlTemplate) const; diff --git a/LiteCore/Support/DatabasePool.cc b/LiteCore/Support/DatabasePool.cc new file mode 100644 index 000000000..8190c4e4b --- /dev/null +++ b/LiteCore/Support/DatabasePool.cc @@ -0,0 +1,325 @@ +// +// DatabasePool.cc +// +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#include "DatabasePool.hh" +#include "DatabaseImpl.hh" // for asInternal, dataFile +#include "Delimiter.hh" +#include "FilePath.hh" +#include "Logging.hh" +#include "c4Collection.hh" +#include "c4ExceptionUtils.hh" +#include + +namespace litecore { + using namespace std; + using namespace fleece; + + static constexpr size_t kDefaultReadOnlyCapacity = 4; + + static constexpr auto kTimeout = 10s; + + static string nameOf(C4Database* db) { return asInternal(db)->dataFile()->loggingName(); } + + DatabasePool::DatabasePool(slice name, C4DatabaseConfig2 const& config) + : Logging(DBLog) + , _dbName(name) + , _dbConfig(config) + , _dbDir(_dbConfig.parentDirectory) + , _readOnly(config.flags | kC4DB_ReadOnly, kDefaultReadOnlyCapacity) + , _readWrite(config.flags & ~kC4DB_ReadOnly, (_dbConfig.flags & kC4DB_ReadOnly) ? 0 : 1) { + _dbConfig.parentDirectory = _dbDir; + } + + DatabasePool::DatabasePool(C4Database* main) : DatabasePool(main->getName(), main->getConfiguration()) { + logInfo("initial database is %s", nameOf(main).c_str()); + _dbTag = _c4db_getDatabaseTag(main); + Cache& cache = writeable() ? _readWrite : _readOnly; + cache.entries[0].db = std::move(main); + cache.created++; + cache.available++; + } + + DatabasePool::~DatabasePool() { close(); } + + string DatabasePool::loggingIdentifier() const { return _dbName; } + + void DatabasePool::close() { + unique_lock lock(_mutex); + if ( !_closed ) { + logInfo("Closing pool..."); + _closed = true; + _cond.notify_all(); // unblocks borrow() calls so they can throw NotOpen + } + _closeUnused(_readOnly); + _closeUnused(_readWrite); + + if ( auto remaining = _readOnly.created + _readWrite.created ) { + logInfo("Waiting for %u borrowed dbs to be returned...", remaining); + auto timeout = std::chrono::system_clock::now() + kTimeout; + bool ok = _cond.wait_until(lock, timeout, [&] { return _readOnly.created + _readWrite.created == 0; }); + if ( !ok ) error::_throw(error::Busy, "Timed out closing DatabasePool"); + Assert(_readOnly.created + _readWrite.created == 0); + } + logInfo("...all databases closed!"); + } + + FilePath DatabasePool::databasePath() const { return FilePath{_dbDir, _dbName + kC4DatabaseFilenameExtension}; } + + unsigned DatabasePool::capacity() const noexcept { + unique_lock lock(_mutex); + return _readOnly.capacity + _readWrite.capacity; + } + + void DatabasePool::setCapacity(unsigned newCapacity) { + Assert(newCapacity <= kMaxCapacity); + unique_lock lock(_mutex); + if ( _closed ) error::_throw(error::NotOpen, "DatabasePool is closed"); + Assert(newCapacity >= 1 + _readWrite.capacity, "capacity too small"); + _readOnly.capacity = newCapacity - _readWrite.capacity; + Assert(_readOnly.capacity >= 1); + + // Toss out any excess available RO databases: + for ( auto& entry : _readOnly.entries ) { + if ( _readOnly.created > _readOnly.capacity && entry.db && entry.borrowCount == 0 ) { + closeDB(std::move(entry.db)); + --_readOnly.available; + --_readOnly.created; + } + } + } + + bool DatabasePool::sameAs(C4Database* db) const noexcept { + return db->getName() == _dbName && db->getConfiguration().parentDirectory == _dbConfig.parentDirectory; + } + + void DatabasePool::onOpen(std::function init, bool callNow) { + unique_lock lock(_mutex); + _initializer = std::move(init); + if ( callNow && _initializer ) { + for ( auto& entry : _readOnly.entries ) + if ( entry.db ) _initializer(entry.db); + for ( auto& entry : _readWrite.entries ) + if ( entry.db ) _initializer(entry.db); + } + } + + unsigned DatabasePool::openCount() const noexcept { + unique_lock lock(_mutex); + return _readOnly.created + _readWrite.created; + } + + unsigned DatabasePool::borrowedCount() const noexcept { + unique_lock lock(_mutex); + return _readOnly.borrowedCount() + _readWrite.borrowedCount(); + } + + void DatabasePool::closeUnused() { + unique_lock lock(_mutex); + _closeUnused(_readOnly); + _closeUnused(_readWrite); + } + + void DatabasePool::_closeUnused(Cache& cache) { + for ( auto& entry : cache.entries ) { + if ( entry.borrowCount == 0 && entry.db ) { + closeDB(std::move(entry.db)); + --cache.created; + --cache.available; + } + } + } + + // Allocates and opens a new C4Database instance. + Retained DatabasePool::newDB(Cache& cache) { + auto config = _dbConfig; + config.flags = cache.flags; + Retained db = C4Database::openNamed(_dbName, config); + if ( _dbTag >= 0 ) _c4db_setDatabaseTag(db, C4DatabaseTag(_dbTag)); + if ( _initializer ) { + try { + _initializer(db); + } catch ( ... ) { + closeDB(db); + throw; + } + } + logInfo("created %s", nameOf(db).c_str()); + return db; + } + + void DatabasePool::closeDB(Retained db) noexcept { + logInfo("closing %s", nameOf(db).c_str()); + try { + db->close(); + } + catchAndWarn(); + } + + BorrowedDatabase DatabasePool::borrow(Cache& cache, bool or_wait) { + auto tid = std::this_thread::get_id(); + + unique_lock lock(_mutex); + while ( true ) { + if ( _closed ) error::_throw(error::NotOpen, "DatabasePool is closed"); + + auto borrow = [&](Cache::Entry& entry) -> BorrowedDatabase { + DebugAssert(entry.db); + DebugAssert(entry.borrower == tid); + ++entry.borrowCount; + return BorrowedDatabase(entry.db, this); + }; + + if ( !cache.writeable() && cache.borrowedCount() > 0 ) { + // A thread can borrow the same read-only database multiple times: + for ( auto& entry : cache.entries ) { + if ( entry.borrower == tid ) { + DebugAssert(entry.borrowCount > 0); + return borrow(entry); + } + } + } + if ( cache.available > 0 ) { + // Look for an available database: + for ( auto& entry : cache.entries ) { + if ( entry.db && entry.borrowCount == 0 ) { + DebugAssert(entry.borrower == thread::id{}); + entry.borrower = tid; + --cache.available; + return borrow(entry); + } + } + } + if ( cache.borrowedCount() < cache.capacity ) { + // Open a new C4Database if cache is not yet at capacity: + for ( auto& entry : cache.entries ) { + if ( entry.db == nullptr ) { + DebugAssert(entry.borrowCount == 0); + entry.db = newDB(cache); + ++cache.created; + entry.borrower = tid; + return borrow(entry); + } + } + } + + // Couldn't borrow a database: + if ( cache.capacity == 0 ) { + Assert(cache.writeable()); + error::_throw(error::NotWriteable, "Database is read-only"); + } + + if ( !or_wait ) return BorrowedDatabase(nullptr, this); + + // Nothing available, so wait and retry + auto timeout = std::chrono::system_clock::now() + kTimeout; + if ( _cond.wait_until(lock, timeout) == std::cv_status::timeout ) borrowFailed(cache); + } + } + + __cold void DatabasePool::borrowFailed(Cache& cache) { + // Try to identify the source of the deadlock: + stringstream out; + out << "Thread " << std::this_thread::get_id() << " timed out waiting on DatabasePool::borrow [" + << (&cache == &_readWrite ? "writeable" : "read-only") << "]. Borrowers are "; + delimiter comma; + for ( auto& entry : cache.entries ) { + if ( entry.borrowCount ) out << comma << entry.borrower; + } + error::_throw(error::Busy, "%s", out.str().c_str()); + } + + BorrowedDatabase DatabasePool::borrow() { return borrow(_readOnly, true); } + + BorrowedDatabase DatabasePool::tryBorrow() { return borrow(_readOnly, false); } + + BorrowedDatabase DatabasePool::borrowWriteable() { return borrow(_readWrite, true); } + + BorrowedDatabase DatabasePool::tryBorrowWriteable() { return borrow(_readWrite, false); } + + // Called by BorrowedDatabase's destructor and its reset method. + void DatabasePool::returnDatabase(fleece::Retained db) { + Assert(db && !db->isInTransaction()); + unique_lock lock(_mutex); + + Cache& cache = (db->getConfiguration().flags & kC4DB_ReadOnly) ? _readOnly : _readWrite; + Assert(cache.borrowedCount() > 0); + + for ( auto& entry : cache.entries ) { + if ( entry.db == db ) { + auto tid = this_thread::get_id(); + if ( entry.borrower != tid ) + Warn("DatabasePool::returnDatabase: Calling thread is not the same that borrowed db"); + Assert(entry.borrowCount > 0); + if ( --entry.borrowCount == 0 ) { + entry.borrower = {}; + if ( cache.created > cache.capacity || _closed ) { + // Toss out a DB if capacity was lowered after it was checked out, or I'm closed: + closeDB(std::move(db)); + entry.db = nullptr; + --cache.created; + } else { + ++cache.available; + } + } + + _cond.notify_all(); // wake up waiting `borrow` and `close` calls + return; + } + } + error::_throw(error::AssertionFailed, "DatabasePool::returnDatabase: db does not belong to pool"); + } + +#pragma mark - CACHE: + + DatabasePool::Cache::Cache(C4DatabaseFlags flags_, unsigned capacity_) + : flags(flags_ & ~kC4DB_Create), capacity(capacity_) { + Assert(capacity <= kMaxCapacity); + } + + // Retained DatabasePool::Cache::pop() { + // Retained db; + // if ( !available.empty() ) { + // db = std::move(available.back()); + // available.pop_back(); + // } + // return db; + // } + + +#pragma mark - BORROWED DATABASE: + + BorrowedDatabase& BorrowedDatabase::operator=(BorrowedDatabase&& b) noexcept { + _return(); + _db = std::move(b._db); + _pool = std::move(b._pool); + return *this; + } + + void BorrowedDatabase::reset() { + _return(); + _db = nullptr; + _pool = nullptr; + } + + void BorrowedDatabase::_return() { + if ( _db && _pool ) _pool->returnDatabase(std::move(_db)); + } + + BorrowedCollection::BorrowedCollection(BorrowedDatabase&& bdb, C4CollectionSpec const& spec) + : _bdb(std::move(bdb)), _collection(_bdb ? _bdb->getCollection(spec) : nullptr) { + if ( _bdb && !_collection ) error::_throw(error::NotFound, "no such collection"); + } + + BorrowedCollection::BorrowedCollection() noexcept = default; + BorrowedCollection::~BorrowedCollection() = default; + +} // namespace litecore diff --git a/LiteCore/Support/DatabasePool.hh b/LiteCore/Support/DatabasePool.hh new file mode 100644 index 000000000..8937f59db --- /dev/null +++ b/LiteCore/Support/DatabasePool.hh @@ -0,0 +1,312 @@ +// +// DatabasePool.hh +// +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#pragma once +#include "c4Database.hh" +#include "Error.hh" +#include "Logging.hh" +#include "fleece/RefCounted.hh" +#include +#include +#include +#include +#include + +C4_ASSUME_NONNULL_BEGIN + +namespace litecore { + class FilePath; + class BorrowedDatabase; + + /** A concurrent pool of C4Database instances on a single file. + A thread wanting to use the database can temporarily "borrow" an instance, wrapped in a + `BorrowedDatabase` smart-pointer. The database is returned to the pool when the + `BorrwedDatabase` object exits scope. There's also a `BorrowedCollection`. + + The databases in the pool are opened read-only, except for one writeable instance. + (Since SQLite allows concurrent readers but only a single writer, this ensures that no two + borrowed databases will block each other.) Therefore, if there is a possibility you need + to write to the database, you must call `borrowWriteable` or else you'll get database- + locked errors. + + If you try to borrow but all matching databases are checked out, the method blocks until + one is returned; but after waiting ten seconds it will throw a `Busy` exception. + If you don't want to block, call one of the `tryBorrow` methods, which return an empty/null + `BorrowedDatabase` instead of blocking. + + @warning Watch out for nested writeable borrows! If you borrow a writeable database, + then call another function that also borrows a writeable database from the same pool, + the second/nested call will deadlock since there's only one writeable database and it's in use. + This will time out after ten seconds and throw a `Busy` exception. + (This isn't a problem with read-only databases; a nested borrow on the same thread will + just return the same instance again. This is safe because both users are on the same thread + and there's no mutable state like transactions.) */ + + class DatabasePool + : public fleece::RefCounted + , public Logging { + public: + /// Constructs a pool that will manage multiple instances of the given database file. + /// If the `kC4DB_ReadOnly` flag is set, no writeable instances will be provided. + /// @note This method does not attempt to open a database. If the database can't be opened, + /// you won't get an exception until you try to borrow it. + explicit DatabasePool(fleece::slice name, C4DatabaseConfig2 const&); + + /// Constructs a pool that will manage multiple instances of the given database file. + /// If this database was opened read-only, then no writeable instances will be provided. + /// @warning The C4Database is now owned by the pool and shouldn't be used directly. + explicit DatabasePool(C4Database*); + + /// Closes all databases, waiting until all borrowed ones have been returned. + /// No more databases can be borrowed after this method begins. + void close(); + + /// Closes all databases the pool has opened that aren't currently in use. + /// (The pool can still re-open more databases on demand, up to its capacity.) + void closeUnused(); + + /// The database configuration. + C4DatabaseConfig2 const& getConfiguration() const { return _dbConfig; } + + /// The filesystem path of the database. + FilePath databasePath() const; + + /// True if it's possible to get a writeable database. + bool writeable() const noexcept { return _readWrite.capacity > 0; } + + /// The maximum number of databases the pool will create. + /// Defaults to 5, or 4 if not writeable. + unsigned capacity() const noexcept; + + /// Sets the maximum number of databases the pool will create, including any writeable one. + /// Minimum value is 2 (otherwise why are you using a pool at all?) + void setCapacity(unsigned capacity); + + /// True if this pool manages the same file as this database. + bool sameAs(C4Database* db) const noexcept; + + /// Registers a function that will be called just after a new `C4Database` is opened. + /// Or if `nullptr` is passed, it clears any already-set function. + /// The function may make connection-level changes. + /// If `callNow` is true, the function will be immediately be called on the databases + /// already in the pool, _except_ for ones currently borrowed. + void onOpen(std::function, bool callNow = true); + + /// The number of databases open, both borrowed and available. + unsigned openCount() const noexcept; + + /// The number of databases currently borrowed. Ranges from 0 up to `capacity`. + unsigned borrowedCount() const noexcept; + + /// Returns a smart-pointer to a **read-only** C4Database a client can use. + /// When the `BorrowedDatabase` goes out of scope, the database is returned to the pool. + /// You must not use the C4Database reference after that! + /// @note If all read-only databases are checked out, waits until one is returned. + /// @throws litecore::error `NotOpen` if `close` has been called. + /// @throws litecore::error `Timeout` after waiting ten seonds. + BorrowedDatabase borrow(); + + /// Same as `borrow`, except that if no databases are available to borrow it returns an + /// empty `BorrowedDatabase` instead of waiting. + /// You must check the returned object to see if the C4Database it's holding is `nullptr`. + BorrowedDatabase tryBorrow(); + + /// Returns a smart-pointer to a **writeable** database a client can use. + /// (There is only one of these per pool, since LiteCore only supports one writer at a time.) + /// When the `BorrowedDatabase` goes out of scope, the database is returned to the pool. + /// You must not use the C4Database reference after that! + /// @note If the writeable database is checcked out, waits until it's returned. + /// @throws litecore::error `NotWriteable` if `writeable()` is false. + /// @throws litecore::error `NotOpen` if `close` has been called. + /// @throws litecore::error `Timeout` after waiting ten seonds. + BorrowedDatabase borrowWriteable(); + + /// Same as `borrowWriteable`, except that if no databases are available to borrow it returns an + /// empty `BorrowedDatabase` instead of waiting. + /// You must check the returned object to see if the C4Database it's holding is `nullptr`. + /// @throws litecore::error `NotWriteable` if `writeable()` is false. + /// @throws litecore::error `NotOpen` if `close` has been called. + BorrowedDatabase tryBorrowWriteable(); + + class Transaction; + + /// Creates a Transaction. Equivalent to `DatabasePool::Transaction(*pool)`. + inline Transaction transaction(); + + /// Calls the function/lambda in a transaction, passing it the `C4Database*`. + /// The transaction automatically commits after `fn` returns, or aborts if it throws. + inline auto inTransaction(auto fn); + + protected: + ~DatabasePool() override; + std::string loggingIdentifier() const override; + + private: + friend class BorrowedDatabase; + + static constexpr size_t kMaxCapacity = 8; + + /** A cache of available db instances with the same access, either read-only or read-write. */ + struct Cache { + Cache(C4DatabaseFlags, unsigned capacity); + + struct Entry { + Retained db; ///< Database (may be nullptr) + unsigned borrowCount = 0; ///< Number of borrows + std::thread::id borrower; ///< Thread that borrowed it, if any + }; + + C4DatabaseFlags const flags; ///< Flags for opening dbs + unsigned capacity; ///< Max number of open dbs + unsigned created = 0; ///< Number of open dbs + unsigned available = 0; ///< Number of un-borrowed open dbs + std::array entries; ///< Tracks each db and its borrowers + + bool writeable() const noexcept { return (flags & kC4DB_ReadOnly) == 0; } + + unsigned borrowedCount() const noexcept { return created - available; } + }; + + DatabasePool(DatabasePool&&) = delete; + DatabasePool& operator=(DatabasePool&&) = delete; + BorrowedDatabase borrow(Cache& cache, bool orWait); + [[noreturn]] void borrowFailed(Cache&); + + fleece::Retained newDB(Cache&); + void closeDB(Retained) noexcept; + void returnDatabase(fleece::Retained); + void _closeUnused(Cache&); + void _closeAll(Cache&); + + std::string const _dbName; // Name of database + C4DatabaseConfig2 _dbConfig; // Database config + fleece::alloc_slice _dbDir; // Parent directory of database + mutable std::mutex _mutex; // Thread-safety + mutable std::condition_variable _cond; // Used for waiting for a db + std::function _initializer; // Init fn called on each new database + Cache _readOnly; // Manages read-only databases + Cache _readWrite; // Manages writeable databases + int _dbTag = -1; // C4DatabaseTag + bool _closed = false; // Set by `close` + }; + + /** An RAII wrapper around a C4Database "borrowed" from a DatabasePool. + When it exits scope, the database is returned to the pool. + @note A `BorrowedDatabase`s lifetime should be kept short, and limited to a single thread. */ + class BorrowedDatabase { + public: + /// Constructs an empty BorrowedDatabase. (Use `operator bool` to test for emptiness.) + BorrowedDatabase() noexcept = default; + + /// Borrows a (read-only) database from a pool. Equivalent to calling `pool.borrow()`. + explicit BorrowedDatabase(DatabasePool* pool) : BorrowedDatabase(pool->borrow()) {} + + /// "Borrows" a database without a pool -- simply retains the db and acts as a smart pointer + /// to it. This allows you to use `BorrowedDatabase` with or without a `DatabasePool`. + explicit BorrowedDatabase(C4Database* db) noexcept : _db(db) {} + + BorrowedDatabase(BorrowedDatabase&& b) noexcept = default; + + BorrowedDatabase& operator=(BorrowedDatabase&& b) noexcept; + + ~BorrowedDatabase() { _return(); } + + /// Checks whether I am non-empty. + /// @note It's illegal to dereference an empty instance. The only way to create such + /// an instance is with the default constructor, or `DatabasePool::tryBorrow`. + explicit operator bool() const noexcept { return _db != nullptr; } + + C4Database* get() const noexcept LIFETIMEBOUND { + DebugAssert(_db); + return _db; + } + + C4Database* operator->() const noexcept LIFETIMEBOUND { return get(); } + + operator C4Database* C4NONNULL() const noexcept LIFETIMEBOUND { return get(); } + + /// Returns the database to the pool, leaving me empty. + void reset(); + + protected: + friend class DatabasePool; + BorrowedDatabase(BorrowedDatabase const&) = delete; + BorrowedDatabase& operator=(BorrowedDatabase const&) = delete; + + // used by DatabasePool::borrow methods + BorrowedDatabase(fleece::Retained db, DatabasePool* pool) : _db(std::move(db)), _pool(pool) {} + + private: + void _return(); + + fleece::Retained _db; + fleece::Retained _pool; + }; + + /** An RAII wrapper around a collection of a database "borrowed" from a `DatabasePool`. + When it exits scope, its database is returned to the pool. */ + class BorrowedCollection { + public: + /// Constructs an empty BorrowedCollection. (Use `operator bool` to test for emptiness.) + BorrowedCollection() noexcept; + + /// Constructor. + /// @throws `error::NotFound` if there is a database but no such collection in it. + BorrowedCollection(BorrowedDatabase&& bdb, C4CollectionSpec const& spec); + + BorrowedCollection(BorrowedCollection&& b) noexcept = default; + BorrowedCollection& operator=(BorrowedCollection&& b) noexcept = default; + ~BorrowedCollection(); + + /// Checks whether I am non-empty, i.e. I have a a collection. + explicit operator bool() const noexcept { return _collection != nullptr; } + + C4Collection* get() const noexcept LIFETIMEBOUND { + DebugAssert(_collection); + return _collection; + } + + C4Collection* operator->() const noexcept LIFETIMEBOUND { return get(); } + + operator C4Collection* C4NONNULL() const noexcept LIFETIMEBOUND { return get(); } + + private: + BorrowedDatabase _bdb; + Retained _collection{}; + }; + + /** Subclass of C4Database::Transaction : a transaction on a borrowed (writeable) database. + Remember to call `commit`! + @note Using this avoids the footgun `C4Database::Transaction t(pool.borrowWriteable());` + which unintentionally returns the database to the pool immediately! */ + class DatabasePool::Transaction + : private BorrowedDatabase + , public C4Database::Transaction { + public: + explicit Transaction(DatabasePool& pool) + : BorrowedDatabase(pool.borrowWriteable()), C4Database::Transaction(db()) {} + + C4Database* db() const noexcept LIFETIMEBOUND { return get(); } + }; + + inline DatabasePool::Transaction DatabasePool::transaction() { return Transaction(*this); } + + inline auto DatabasePool::inTransaction(auto fn) { + Transaction txn(*this); + fn(txn.db()); + if ( txn.isActive() ) txn.commit(); + } + + +} // namespace litecore + +C4_ASSUME_NONNULL_END diff --git a/LiteCore/Support/StringUtil.cc b/LiteCore/Support/StringUtil.cc index 9f8740165..4493c1e08 100644 --- a/LiteCore/Support/StringUtil.cc +++ b/LiteCore/Support/StringUtil.cc @@ -71,10 +71,24 @@ namespace litecore { string::size_type pos, next; for ( pos = 0; pos < end; pos = next + separator.size() ) { next = str.find(separator, pos); - if ( next == string::npos ) break; + if ( next == string::npos ) next = end; callback(str.substr(pos, next - pos)); } - callback(str.substr(pos)); + } + + std::vector split(std::string_view str, std::string_view separator) { + vector strings; + split(str, separator, [&](string_view s) { strings.push_back(s); }); + return strings; + } + + std::pair split2(std::string_view str, std::string_view separator) { + string_view rest; + if ( auto pos = str.find(separator); pos != string::npos ) { + rest = str.substr(pos + 1); + str = str.substr(0, pos); + } + return {str, rest}; } stringstream& join(stringstream& s, const std::vector& strings, const char* separator) { @@ -118,6 +132,12 @@ namespace litecore { return replaced; } + std::string_view trimWhitespace(std::string_view str) { + while ( !str.empty() && isspace(str[0]) ) str.remove_prefix(1); + while ( !str.empty() && isspace(str[str.size() - 1]) ) str.remove_suffix(1); + return str; + } + bool hasPrefix(string_view str, string_view prefix) noexcept { return str.size() >= prefix.size() && memcmp(str.data(), prefix.data(), prefix.size()) == 0; } diff --git a/LiteCore/Support/StringUtil.hh b/LiteCore/Support/StringUtil.hh index 0b3a890bc..037e8a483 100644 --- a/LiteCore/Support/StringUtil.hh +++ b/LiteCore/Support/StringUtil.hh @@ -53,8 +53,17 @@ namespace litecore { /** Like vsprintf(), but returns a std::string */ std::string vstringprintf(const char* fmt NONNULL, va_list) __printflike(1, 0); + /** Splits the string at occurrences of `separator` and calls the callback for each piece. + There may be empty pieces, if the separator occurs at the start or end or twice in a row. */ void split(std::string_view str, std::string_view separator, fleece::function_ref callback); + /** Splits the string at occurrences of `separator` and returns all the pieces.*/ + std::vector split(std::string_view str, std::string_view separator); + + /** Splits the string at the _first occurrence_ of `separator` and returns the pieces before and after. + If the separator is not found, returns the original string and an empty string. */ + std::pair split2(std::string_view str, std::string_view separator); + /** Returns the strings in the vector concatenated together, with the separator (if non-null) between them. */ std::string join(const std::vector&, const char* separator = nullptr); @@ -75,6 +84,9 @@ namespace litecore { /** Replaces all occurrences of `oldStr` with `newStr`. Returns true if anything was replaced. */ bool replace(std::string& str, std::string_view oldStr, std::string_view newStr); + /** Returns a substring of a string without any leading or trailing ASCII whitespace. */ + std::string_view trimWhitespace(std::string_view); + /** Returns true if `str` begins with the string `prefix`. */ bool hasPrefix(std::string_view str, std::string_view prefix) noexcept; diff --git a/LiteCore/tests/CMakeLists.txt b/LiteCore/tests/CMakeLists.txt index 78c5badc1..b338a9578 100644 --- a/LiteCore/tests/CMakeLists.txt +++ b/LiteCore/tests/CMakeLists.txt @@ -73,7 +73,6 @@ add_executable( VectorQueryTest.cc LazyVectorQueryTest.cc VersionVectorTest.cc - ${TOP}REST/tests/RESTListenerTest.cc ${TOP}REST/tests/RESTClientTest.cc ${TOP}REST/tests/SyncListenerTest.cc ${TOP}vendor/fleece/Tests/API_ValueTests.cc diff --git a/LiteCore/tests/LiteCoreTest.cc b/LiteCore/tests/LiteCoreTest.cc index 645c79393..3bf876c96 100644 --- a/LiteCore/tests/LiteCoreTest.cc +++ b/LiteCore/tests/LiteCoreTest.cc @@ -43,9 +43,6 @@ string TestFixture::sFixturesDir = "../LiteCore/tests/data/"; string TestFixture::sFixturesDir = "LiteCore/tests/data/"; #endif - -FilePath TestFixture::sTempDir = GetTempDirectory(); - string stringWithFormat(const char* format, ...) { va_list args; va_start(args, format); @@ -62,6 +59,7 @@ void ExpectException(litecore::error::Domain domain, int code, const char* what, } catch ( std::runtime_error& x ) { Log("... caught exception %s", x.what()); error err = error::convertRuntimeError(x).standardized(); + INFO("ExpectException caught " << err.domain << '/' << err.code << ": " << err.what()); CHECK(err.domain == domain); CHECK(err.code == code); if ( what ) CHECK(string_view(err.what()) == string_view(what)); diff --git a/LiteCore/tests/LogEncoderTest.cc b/LiteCore/tests/LogEncoderTest.cc index 8fad922b0..a7061f730 100644 --- a/LiteCore/tests/LogEncoderTest.cc +++ b/LiteCore/tests/LogEncoderTest.cc @@ -30,6 +30,10 @@ using namespace std; constexpr size_t kFolderBufSize = 64; +// This is moved here so that it can be shared between cbl-logtest +// and CppTests +FilePath TestFixture::sTempDir = GetTempDirectory(); + class LogObject : public Logging { public: explicit LogObject(const std::string& identifier) : Logging(DBLog), _identifier(identifier) {} diff --git a/Networking/HTTP/HTTPLogic.cc b/Networking/HTTP/HTTPLogic.cc index 2966ba05b..32420001e 100644 --- a/Networking/HTTP/HTTPLogic.cc +++ b/Networking/HTTP/HTTPLogic.cc @@ -224,7 +224,11 @@ namespace litecore::net { while ( true ) { slice line = responseData.readToDelimiter("\r\n"_sl); if ( !line ) return false; - if ( line.size == 0 ) break; // empty line + if ( line.size == 0 ) break; // empty line denotes end; exit + for ( uint8_t byte : line ) { + if ( (byte < ' ' && byte != '\t') || byte == 0x7F ) // no control characters + return false; + } const uint8_t* colon = line.findByte(':'); if ( !colon ) return false; slice name(line.buf, colon); diff --git a/Networking/HTTP/HTTPLogic.hh b/Networking/HTTP/HTTPLogic.hh index cfdb24577..07e62cfd6 100644 --- a/Networking/HTTP/HTTPLogic.hh +++ b/Networking/HTTP/HTTPLogic.hh @@ -162,9 +162,10 @@ namespace litecore::net { /// (before the blank line). static std::string formatHTTP(slice http); - /** Utility function to parse HTTP headers. Reads header lines from HTTP data until - it reaches an empty line (CRLFCRLF). On return, \ref httpData will point to any - data remaining after the empty line. */ + /// Utility function to parse HTTP headers. Reads header lines from HTTP data until + /// it reaches an empty line (CRLFCRLF). On return, \ref httpData will point to any + /// data remaining after the empty line. + /// @returns True on success, false on parse error. static bool parseHeaders(fleece::slice_istream& httpData, websocket::Headers&); /// Given a "Sec-WebSocket-Key" header value, returns the "Sec-WebSocket-Accept" value. diff --git a/Networking/HTTP/HTTPTypes.cc b/Networking/HTTP/HTTPTypes.cc index eef07a87a..34a5e367e 100644 --- a/Networking/HTTP/HTTPTypes.cc +++ b/Networking/HTTP/HTTPTypes.cc @@ -61,6 +61,50 @@ namespace litecore::net { return Method::None; } + HTTPStatus StatusFromError(C4Error err) { + if ( err.code == 0 ) return HTTPStatus::OK; + HTTPStatus status = HTTPStatus::ServerError; + // TODO: Add more mappings, and make these table-driven + switch ( err.domain ) { + case LiteCoreDomain: + switch ( err.code ) { + case kC4ErrorInvalidParameter: + case kC4ErrorBadRevisionID: + status = HTTPStatus::BadRequest; + break; + case kC4ErrorNotADatabaseFile: + case kC4ErrorCrypto: + status = HTTPStatus::Unauthorized; + break; + case kC4ErrorNotWriteable: + status = HTTPStatus::Forbidden; + break; + case kC4ErrorNotFound: + status = HTTPStatus::NotFound; + break; + case kC4ErrorConflict: + status = HTTPStatus::Conflict; + break; + case kC4ErrorUnimplemented: + case kC4ErrorUnsupported: + status = HTTPStatus::NotImplemented; + break; + case kC4ErrorRemoteError: + status = HTTPStatus::GatewayError; + break; + case kC4ErrorBusy: + status = HTTPStatus::Locked; + break; + } + break; + case WebSocketDomain: + if ( err.code < 1000 ) status = HTTPStatus(err.code); + default: + break; + } + return status; + } + ProxySpec::ProxySpec(const C4Address& addr) { if ( slice(addr.scheme).caseEquivalent("http"_sl) ) type = ProxyType::HTTP; if ( slice(addr.scheme).caseEquivalent("https"_sl) ) type = ProxyType::HTTPS; diff --git a/Networking/HTTP/HTTPTypes.hh b/Networking/HTTP/HTTPTypes.hh index f92b8f8f1..6b495199b 100644 --- a/Networking/HTTP/HTTPTypes.hh +++ b/Networking/HTTP/HTTPTypes.hh @@ -31,6 +31,7 @@ namespace litecore::net { OK = 200, Created = 201, + Accepted = 202, NoContent = 204, MovedPermanently = 301, @@ -40,18 +41,19 @@ namespace litecore::net { UseProxy = 305, TemporaryRedirect = 307, - BadRequest = 400, - Unauthorized = 401, - Forbidden = 403, - NotFound = 404, - MethodNotAllowed = 405, - NotAcceptable = 406, - ProxyAuthRequired = 407, - Conflict = 409, - Gone = 410, - PreconditionFailed = 412, - UnprocessableEntity = 422, - Locked = 423, + BadRequest = 400, + Unauthorized = 401, + Forbidden = 403, + NotFound = 404, + MethodNotAllowed = 405, + NotAcceptable = 406, + ProxyAuthRequired = 407, + Conflict = 409, + Gone = 410, + PreconditionFailed = 412, + UnsupportedMediaType = 415, + UnprocessableEntity = 422, + Locked = 423, ServerError = 500, NotImplemented = 501, @@ -62,6 +64,8 @@ namespace litecore::net { const char* StatusMessage(HTTPStatus); + HTTPStatus StatusFromError(C4Error); + /// HTTP methods. These do NOT have consecutive values, rather they're powers of two /// so they can be used as bit-masks. enum Method : unsigned { diff --git a/Networking/HTTP/Headers.cc b/Networking/HTTP/Headers.cc index ac44d9324..5b6916331 100644 --- a/Networking/HTTP/Headers.cc +++ b/Networking/HTTP/Headers.cc @@ -11,42 +11,24 @@ // #include "Headers.hh" +#include "StringUtil.hh" #include "fleece/Fleece.hh" #include "fleece/Expert.hh" #include "slice_stream.hh" #include -#include #include "betterassert.hh" namespace litecore::websocket { + using namespace std; using namespace fleece; - Headers::Headers(const fleece::alloc_slice& encoded) : _backingStore(encoded) { + Headers::Headers(const fleece::alloc_slice& encoded) : _backingStore{encoded} { readFrom(ValueFromData(encoded).asDict()); } Headers::Headers(Dict dict) { readFrom(dict); } - Headers::Headers(const Headers& other) { *this = other; } - - Headers::Headers(Headers&& other) noexcept - : _map(std::move(other._map)) - , _backingStore(std::move(other._backingStore)) - , _writer(std::move(other._writer)) {} - - Headers& Headers::operator=(const Headers& other) { - clear(); - if ( other._writer.length() == 0 ) { - _map = other._map; - _backingStore = other._backingStore; - } else { - setBackingStore(other._backingStore); - for ( auto& entry : other._map ) add(entry.first, entry.second); - } - return *this; - } - void Headers::readFrom(Dict dict) { for ( Dict::iterator i(dict); i; ++i ) { slice key = i.keyString(); @@ -59,20 +41,17 @@ namespace litecore::websocket { } } - void Headers::setBackingStore(alloc_slice backingStore) { - assert(_map.empty()); - _backingStore = std::move(backingStore); - } - void Headers::clear() { _map.clear(); - _backingStore = nullslice; - _writer.reset(); + _backingStore.clear(); } slice Headers::store(slice s) { - if ( _backingStore.containsAddressRange(s) ) return s; - return {_writer.write(s), s.size}; + for ( auto& stored : _backingStore ) { + if ( stored.containsAddressRange(s) ) return s; + } + _backingStore.emplace_back(s); + return _backingStore.back(); } void Headers::add(slice name, slice value) { @@ -80,6 +59,12 @@ namespace litecore::websocket { if ( value ) _map.insert({store(name), store(value)}); } + void Headers::set(slice name, slice value) { + assert(name); + _map.erase(name); + add(name, value); + } + slice Headers::get(slice name) const { auto i = _map.find(name); if ( i == _map.end() ) return nullslice; @@ -94,6 +79,15 @@ namespace litecore::websocket { return n; } + std::string Headers::getAll(slice name) const { + string all; + forEach(name, [&all](slice value) { + if ( !all.empty() ) all += ','; + all += string_view(value); + }); + return all; + } + void Headers::forEach(fleece::function_ref callback) const { for ( const auto& i : _map ) callback(i.first, i.second); } diff --git a/Networking/HTTP/Headers.hh b/Networking/HTTP/Headers.hh index e0ec78e1d..eaa015a2c 100644 --- a/Networking/HTTP/Headers.hh +++ b/Networking/HTTP/Headers.hh @@ -12,9 +12,9 @@ #pragma once #include "fleece/slice.hh" -#include "Writer.hh" #include "fleece/function_ref.hh" #include +#include namespace fleece { class Dict; @@ -30,42 +30,52 @@ namespace litecore::websocket { using slice = fleece::slice; using alloc_slice = fleece::alloc_slice; + /** Creates an empty instance. */ Headers() = default; - /** Reconstitute from Fleece data. */ + /** Instantiate from a Fleece Dict whose keys are header names and values are either + strings or arrays of strings. */ + explicit Headers(fleece::Dict); + + /** Reconstitute from an encoded Fleece Dict. */ explicit Headers(const alloc_slice& encoded); + /** Reconstitute from an encoded Fleece Dict. */ explicit Headers(slice encoded) : Headers(alloc_slice(encoded)) {} - explicit Headers(fleece::Dict); - - Headers(const Headers&); - Headers(Headers&&) noexcept; - Headers& operator=(const Headers&); + Headers(const Headers&) = default; + Headers& operator=(const Headers&) = default; + Headers(Headers&&) noexcept = default; + Headers& operator=(Headers&&) noexcept = default; + /** Removes all headers. */ void clear(); + /** True if there are no headers. */ [[nodiscard]] bool empty() const { return _map.empty(); } - /** Keep a reference to this alloc_slice; any keys/values that are added that point - within the backing store won't cause any allocation. */ - void setBackingStore(alloc_slice); - /** Adds a header. If a header with that name already exists, it adds a second. */ void add(slice name, slice value); + /** Sets the value of a header. If headers with that name exist, they're replaced. */ + void set(slice name, slice value); + /** Returns the value of a header with that name.*/ [[nodiscard]] slice get(slice name) const; + /** Returns a header parsed as an integer. If missing, returns `defaultValue` */ [[nodiscard]] int64_t getInt(slice name, int64_t defaultValue = 0) const; - /** Returns the value of a header with that name.*/ - slice operator[](slice name) const { return get(name); } + /** Returns the value of a header with that name. */ + [[nodiscard]] slice operator[](slice name) const { return get(name); } + + /** Returns all header values with the given name, separated by commas. */ + [[nodiscard]] std::string getAll(slice name) const; - /** Calls the function once for each header, in ASCII order.*/ + /** Calls the function once for each header/value pair, in ASCII order.*/ void forEach(fleece::function_ref callback) const; - /** Calls the function once for each value with the given name.*/ + /** Calls the function once for each header with the given name.*/ void forEach(slice name, fleece::function_ref callback) const; /** Encodes the headers as a Fleece dictionary. Each key is a header name, and its @@ -78,12 +88,11 @@ namespace litecore::websocket { class HeaderCmp { public: - bool operator()(fleece::slice a, fleece::slice b) const noexcept { return a.caseEquivalentCompare(b) < 0; } + bool operator()(slice a, slice b) const noexcept { return a.caseEquivalentCompare(b) < 0; } }; std::multimap _map; - alloc_slice _backingStore; - fleece::Writer _writer; + std::vector _backingStore; // Owns the data that _map points to }; diff --git a/Networking/TCPSocket.cc b/Networking/TCPSocket.cc index b51346aef..8d8941acc 100644 --- a/Networking/TCPSocket.cc +++ b/Networking/TCPSocket.cc @@ -75,10 +75,7 @@ namespace litecore::net { bool TCPSocket::wrapTLS(slice hostname) { if ( !_tlsContext ) _tlsContext = new TLSContext(_isClient ? TLSContext::Client : TLSContext::Server); - string hostnameStr(hostname); - auto oldSocket = std::move(_socket); - return setSocket(_tlsContext->_context->wrap_socket( - std::move(oldSocket), (_isClient ? tls_context::CLIENT : tls_context::SERVER), hostnameStr)); + return setSocket(_tlsContext->wrapSocket(std::move(_socket), string(hostname))); } bool TCPSocket::connected() const { return _socket && !_socket->is_shutdown(); } @@ -387,7 +384,7 @@ namespace litecore::net { slice_istream reader(line); chunkLength = (size_t)reader.readHex(); if ( !reader.eof() ) { - setError(WebSocketDomain, kCodeProtocolError, "Invalid chunked response data"); + setError(WebSocketDomain, 400, "Invalid chunked response data"); return nullslice; } @@ -399,7 +396,7 @@ namespace litecore::net { char crlf[2]; if ( readExactly(crlf, 2) < 2 ) return nullslice; if ( crlf[0] != '\r' || crlf[1] != '\n' ) { - setError(WebSocketDomain, kCodeProtocolError, "Invalid chunked response data"); + setError(WebSocketDomain, 400, "Invalid chunked response data"); return nullslice; } } while ( chunkLength > 0 ); @@ -422,17 +419,23 @@ namespace litecore::net { //TODO: There may be more response headers after the chunks } else { body.reset(); - setError(NetworkDomain, kNetErrUnknown, "Unsupported HTTP Transfer-Encoding"); + setError(WebSocketDomain, 501, "Unsupported HTTP Transfer-Encoding"); // Other transfer encodings are "gzip", "deflate" } - } else if ( auto conn = headers["Connection"]; conn.caseEquivalent("close") ) { - // Connection:Close mode -- read till EOF: - body = readToEOF(); + } else if ( auto conn = headers["Connection"] ) { + if ( conn.caseEquivalent("close") ) { + // Connection:Close mode -- read till EOF: + body = readToEOF(); + } else { + body.reset(); + setError(WebSocketDomain, 501, "Unsupported 'Connection' response header"); + } } else { body.reset(); - setError(WebSocketDomain, kCodeProtocolError, "Unsupported 'Connection' response header"); + setError(WebSocketDomain, 400, + "Response has neither 'Content-Length', 'Transfer-Encoding' nor 'Connection: close'"); } return !!body; @@ -607,14 +610,21 @@ namespace litecore::net { int err = _socket->last_error(); Assert(err != 0); if ( err > 0 ) { - err = socketToPosixErrCode(err); - string errStr = error::_what(error::POSIX, err); - LogWarn(WSLog, "%s got POSIX error %d \"%s\"", (_isClient ? "ClientSocket" : "ResponderSocket"), err, - errStr.c_str()); + err = socketToPosixErrCode(err); + C4Error error; if ( err == EWOULDBLOCK ) // Occurs in blocking mode when I/O times out - setError(NetworkDomain, kC4NetErrTimeout); + error = C4Error{NetworkDomain, kC4NetErrTimeout}; else - setError(POSIXDomain, err); + error = C4Error{POSIXDomain, err}; + if ( error != _error ) { + _error = error; + LogLevel level = LogLevel::Warning; + // As a server, it's normal for the client to close their socket, so don't warn. + if ( !_isClient && err == ECONNRESET ) level = LogLevel::Info; + string errStr = error::_what(error::POSIX, err); + WSLog.log(level, "%s got POSIX error %d \"%s\"", (_isClient ? "ClientSocket" : "ResponderSocket"), err, + errStr.c_str()); + } } else { // Negative errors are assumed to be from mbedTLS. char msgbuf[100]; diff --git a/Networking/TLSContext.cc b/Networking/TLSContext.cc index be0b6793b..b91b5e505 100644 --- a/Networking/TLSContext.cc +++ b/Networking/TLSContext.cc @@ -41,11 +41,13 @@ namespace litecore::net { else if ( logLevel == LogLevel::Debug ) mbedLogLevel = 4; _context->set_logger(mbedLogLevel, [=](int level, const char* filename, int line, const char* message) { - static const LogLevel kLogLevels[] = {LogLevel::Error, LogLevel::Error, LogLevel::Verbose, - LogLevel::Verbose, LogLevel::Debug}; - size_t len = strlen(message); - if ( message[len - 1] == '\n' ) --len; - TLSLogDomain.log(kLogLevels[level], "mbedTLS(%s): %.*s", (role == Client ? "C" : "S"), int(len), message); + // mbedTLS logging callback: + static const LogLevel kLogLevels[] = {LogLevel::Info, LogLevel::Info, LogLevel::Verbose, LogLevel::Verbose, + LogLevel::Debug}; + string_view str(message); + if ( str.ends_with('\n') ) str = str.substr(0, str.size() - 1); + TLSLogDomain.log(kLogLevels[level], "mbedTLS(%s): %.*s", (role == Client ? "C" : "S"), int(str.size()), + str.data()); }); } @@ -123,6 +125,12 @@ namespace litecore::net { _context->set_identity(string(certData), string(keyData)); } + unique_ptr TLSContext::wrapSocket(unique_ptr socket, const string& peer_name) { + // (This method should _not_ lock the mutex, because the entire TLS handshake runs + // synchronously during the `wrap_socket` call. + return _context->wrap_socket(std::move(socket), tls_context::role_t(_role), peer_name); + } + void TLSContext::resetRootCertFinder() { #ifdef ROOT_CERT_LOOKUP_AVAILABLE _context->set_root_cert_locator( diff --git a/Networking/TLSContext.hh b/Networking/TLSContext.hh index d15c1b0ee..9b391a2ed 100644 --- a/Networking/TLSContext.hh +++ b/Networking/TLSContext.hh @@ -15,10 +15,13 @@ #include "fleece/slice.hh" #include #include +#include namespace sockpp { class mbedtls_context; -} + class stream_socket; + class tls_socket; +} // namespace sockpp namespace litecore { class LogDomain; @@ -83,6 +86,11 @@ namespace litecore::net { void setIdentity(crypto::Identity* NONNULL); void setIdentity(fleece::slice certData, fleece::slice privateKeyData); + /// Performs the TLS handshake, then returns a wrapper socket that can be used for I/O. + /// Be sure to check the returned socket's error status to see if the handshake failed. + std::unique_ptr wrapSocket(std::unique_ptr, + const std::string& peer_name); + protected: ~TLSContext() override; static bool findSigningRootCert(const std::string& certStr, std::string& rootStr); @@ -94,8 +102,6 @@ namespace litecore::net { fleece::Retained _identity; role_t _role; bool _onlySelfSigned{false}; - - friend class TCPSocket; }; } // namespace litecore::net diff --git a/Networking/WebSockets/BuiltInWebSocket.cc b/Networking/WebSockets/BuiltInWebSocket.cc index fc6002a3a..f0768c241 100644 --- a/Networking/WebSockets/BuiltInWebSocket.cc +++ b/Networking/WebSockets/BuiltInWebSocket.cc @@ -15,6 +15,7 @@ #include "HTTPLogic.hh" #include "Certificate.hh" #include "CookieStore.hh" +#include "DBAccess.hh" #include "c4Database.hh" #include "c4ReplicatorTypes.h" #include "c4Socket+Internal.hh" @@ -332,7 +333,7 @@ namespace litecore::websocket { void BuiltInWebSocket::setCookie(const Address& addr, slice cookieHeader) { bool acceptParentDomain = options()[kC4ReplicatorOptionAcceptParentDomainCookies].asBool(); - _database->useLocked()->setCookie(cookieHeader, addr.hostname(), addr.path(), acceptParentDomain); + _database->useWriteable()->setCookie(cookieHeader, addr.hostname(), addr.path(), acceptParentDomain); } #pragma mark - I/O: @@ -465,6 +466,7 @@ namespace litecore::websocket { status.reason = kPOSIXError; else if ( err.domain == NetworkDomain ) status.reason = kNetworkError; + logError("closeWithError %s", err.description().c_str()); onClose(status); } _selfRetain = nullptr; // allow myself to be freed now diff --git a/Networking/WebSockets/BuiltInWebSocket.hh b/Networking/WebSockets/BuiltInWebSocket.hh index b1f1b5fd5..512e395af 100644 --- a/Networking/WebSockets/BuiltInWebSocket.hh +++ b/Networking/WebSockets/BuiltInWebSocket.hh @@ -14,7 +14,6 @@ #include "WebSocketImpl.hh" #include "TCPSocket.hh" #include "HTTPLogic.hh" -#include "DBAccess.hh" #include #include #include @@ -34,6 +33,10 @@ namespace litecore::crypto { struct Identity; } +namespace litecore::repl { + class DBAccess; +} + namespace litecore::websocket { /** WebSocket implementation using TCPSocket. */ diff --git a/REST/CMakeLists.txt b/REST/CMakeLists.txt index d83889729..bad37a040 100644 --- a/REST/CMakeLists.txt +++ b/REST/CMakeLists.txt @@ -22,19 +22,16 @@ set(CMAKE_C_STANDARD 11) set( ALL_SRC_FILES - c4Listener+RESTFactory.cc c4Listener.cc - Listener.cc + c4Listener_CAPI.cc + CertRequest.cc + DatabaseRegistry.cc + HTTPListener.cc netUtils.cc Request.cc Response.cc - RESTListener+Handlers.cc - RESTListener+Replicate.cc - RESTListener.cc - REST_CAPI.cc Server.cc EE/RESTSyncListener_stub.cc - CertRequest.cc ) ### STATIC LIBRARY: diff --git a/REST/DatabaseRegistry.cc b/REST/DatabaseRegistry.cc new file mode 100644 index 000000000..89e8702c2 --- /dev/null +++ b/REST/DatabaseRegistry.cc @@ -0,0 +1,197 @@ +// +// DatabaseRegistry.cc +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#include "DatabaseRegistry.hh" +#include "DatabasePool.hh" +#include "c4Database.hh" +#include "c4ListenerInternal.hh" +#include "c4Private.h" // for _c4db_setDatabaseTag +#include "Error.hh" +#include "slice_stream.hh" + +#ifdef COUCHBASE_ENTERPRISE + +using namespace std; +using namespace fleece; + +namespace litecore::REST { + + string DatabaseRegistry::makeKeyspace(string_view dbName, C4CollectionSpec const& coll) { + string keyspace(dbName); + bool hasScope = (coll.scope && coll.scope != kC4DefaultScopeID); + if ( hasScope ) { + keyspace += "."; + keyspace += slice(coll.scope); + } + if ( hasScope || (coll.name && coll.name != kC4DefaultCollectionName) ) { + keyspace += "."; + keyspace += slice(coll.name ? coll.name : kC4DefaultCollectionName); + } + return keyspace; + } + + pair DatabaseRegistry::parseKeyspace(slice keyspace) { + slice_istream in(keyspace); + slice dbName = in.readToDelimiter("."); + if ( !dbName ) return {keyspace, kC4DefaultCollectionSpec}; + C4CollectionSpec spec = {}; + spec.name = in.readToDelimiterOrEnd("."); + if ( in.size > 0 ) { + spec.scope = spec.name; + spec.name = in; + } else { + spec.scope = kC4DefaultScopeID; + } + return {dbName, spec}; + } + + static bool isCharValidInDBName(char c) { + // '.' is a keyspace collection delimiter, '/' is a path separator. + return c >= ' ' && c < 0x7F && c != '.' && c != '/'; + } + + string DatabaseRegistry::databaseNameFromPath(const FilePath& path) { + string name = path.fileOrDirName(); + auto split = FilePath::splitExtension(name); + if ( split.second != kC4DatabaseFilenameExtension ) + error::_throw(error::InvalidParameter, "Not a database path"); + name = split.first; + + // Make the name legal as a URI component in the REST API. + // It shouldn't be empty, nor start with an underscore, nor contain reserved characters. + if ( name.empty() ) name = "db"; + else if ( name[0] == '_' ) + name[0] = '-'; + for ( char& c : name ) + if ( !isCharValidInDBName(c) ) c = '-'; + return name; + } + + bool DatabaseRegistry::isValidDatabaseName(const string& name) { + if ( name.empty() || name.size() > 240 || name[0] == '_' ) return false; + return std::all_of(name.begin(), name.end(), isCharValidInDBName); + } + + bool DatabaseRegistry::registerDatabase(C4Database* db, optional name, + C4ListenerDatabaseConfig const& dbConfig) { + if ( !name ) { + alloc_slice path(db->getPath()); + name = databaseNameFromPath(FilePath(string(path))); + } else if ( !isValidDatabaseName(*name) ) { + error::_throw(error::InvalidParameter, "Invalid name for sharing a database"); + } + lock_guard lock(_mutex); + if ( _databases.contains(*name) ) return false; + + auto pool = make_retained(db); + pool->onOpen([](C4Database* db) { _c4db_setDatabaseTag(db, DatabaseTag_RESTListener); }); + _databases.emplace(*name, DBShare{.pool = std::move(pool), + .keySpaces = {makeKeyspace(*name, kC4DefaultCollectionSpec)}, + .config = dbConfig}); + return true; + } + + bool DatabaseRegistry::unregisterDatabase(const std::string& name) { + lock_guard lock(_mutex); + auto i = _databases.find(name); + if ( i == _databases.end() ) return false; + _databases.erase(i); + return true; + } + + bool DatabaseRegistry::unregisterDatabase(C4Database* db) { + lock_guard lock(_mutex); + for ( auto i = _databases.begin(); i != _databases.end(); ++i ) { + if ( i->second.pool->sameAs(db) ) { + _databases.erase(i); + return true; + } + } + return false; + } + + DatabaseRegistry::DBShare* DatabaseRegistry::_getShare(std::string const& name) { + auto i = _databases.find(name); + return i != _databases.end() ? &i->second : nullptr; + } + + DatabaseRegistry::DBShare const* DatabaseRegistry::_getShare(std::string const& name) const { + return const_cast(this)->_getShare(name); + } + + optional DatabaseRegistry::getShare(std::string const& name) const { + lock_guard lock(_mutex); + optional result; + if ( auto share = _getShare(name) ) result.emplace(*share); + return result; + } + + bool DatabaseRegistry::registerCollection(const string& name, C4CollectionSpec const& collection) { + lock_guard lock(_mutex); + auto share = _getShare(name); + if ( !share ) return false; + share->keySpaces.insert(makeKeyspace(name, collection)); + return true; + } + + bool DatabaseRegistry::unregisterCollection(const string& name, C4CollectionSpec const& collection) { + lock_guard lock(_mutex); + auto share = _getShare(name); + return share && share->keySpaces.erase(makeKeyspace(name, collection)) > 0; + } + + BorrowedDatabase DatabaseRegistry::borrowDatabaseNamed(const string& name, bool writeable) const { + lock_guard lock(_mutex); + if ( auto share = _getShare(name) ) return writeable ? share->pool->borrowWriteable() : share->pool->borrow(); + else + return {}; + } + + BorrowedCollection DatabaseRegistry::borrowCollection(const string& keyspace, bool writeable) const { + auto [dbName, spec] = parseKeyspace(keyspace); + lock_guard lock(_mutex); + auto share = _getShare(string(dbName)); + if ( !share ) return {}; + if ( !share->keySpaces.contains(keyspace) ) { + // The input keyspace might not be in normalized form, i.e. might be `db._default`. + // Construct a normalized keyspace string and retry: + string normalized = makeKeyspace(dbName, spec); + if ( !share->keySpaces.contains(normalized) ) return {}; + } + return BorrowedCollection(writeable ? share->pool->borrowWriteable() : share->pool->borrow(), spec); + } + + optional DatabaseRegistry::nameOfDatabase(C4Database* db) const { + lock_guard lock(_mutex); + for ( auto& [aName, share] : _databases ) + if ( share.pool->sameAs(db) ) return aName; + return nullopt; + } + + vector DatabaseRegistry::databaseNames() const { + lock_guard lock(_mutex); + vector names; + names.reserve(_databases.size()); + for ( auto& d : _databases ) names.push_back(d.first); + return names; + } + + void DatabaseRegistry::closeDatabases() { + lock_guard lock(_mutex); + c4log(ListenerLog, kC4LogInfo, "Closing databases"); + for ( auto& d : _databases ) d.second.pool->close(); + _databases.clear(); + } + +} // namespace litecore::REST + +#endif diff --git a/REST/DatabaseRegistry.hh b/REST/DatabaseRegistry.hh new file mode 100644 index 000000000..5c620b30d --- /dev/null +++ b/REST/DatabaseRegistry.hh @@ -0,0 +1,128 @@ +// +// DatabaseRegistry.hh +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#pragma once +#include "c4Database.hh" +#include "c4ListenerTypes.h" +#include "FilePath.hh" +#include +#include +#include +#include +#include + +#ifdef COUCHBASE_ENTERPRISE + +C4_ASSUME_NONNULL_BEGIN + +namespace litecore { + class DatabasePool; + class BorrowedCollection; + class BorrowedDatabase; +} // namespace litecore + +namespace litecore::REST { + + /** Tracks the databases and collections shared by a HTTPListener. */ + class DatabaseRegistry { + public: + /// Creates a "keyspace" string from a database name and collection spec. + /// This is the database name, scope and collection name separated by ".". + /// If the scope is default, it's omitted. + /// If the scope and collection name are default, the keyspace is just the database name. + static std::string makeKeyspace(std::string_view dbName, C4CollectionSpec const&); + + /// Takes apart a "keyspace" string into a database name and collection spec. + static std::pair parseKeyspace(fleece::slice ks LIFETIMEBOUND); + + /** Determines whether a database name is valid for use as a URI path component. + It must be nonempty, no more than 240 bytes long, not start with an underscore, + and contain no control characters. */ + static bool isValidDatabaseName(const std::string&); + + /// Given a filesystem path to a database, returns the database name. + /// (This takes the last path component and removes the ".cblite2" extension.) + /// Returns an empty string if the path is not a database, or if the name would not + /// be valid according to isValidDatabaseName(). + static std::string databaseNameFromPath(const FilePath&); + + /// Makes a database visible via the REST API. + /// By default, only its default collection is served. Call `registerCollection` to add others. + /// @param db The database to share. On success this instance is now managed by the Listener + /// and should not be used again by the caller. + /// @param name The URI name (first path component) in the HTTP API. + /// If not given, the C4Database's name will be used (possibly URL-escaped). + /// @param dbConfig Configuration for this database. Overrides the C4ListenerConfig. + /// @returns True on success, false if the name is already in use. + bool registerDatabase(C4Database* db, std::optional name, + C4ListenerDatabaseConfig const& dbConfig); + + /// Unregisters a database by its registered URI name. + bool unregisterDatabase(const std::string& name); + + /// Unregisters a database. `db` need not be the exact instance that was registered; + /// any instance on the same database file will work. + bool unregisterDatabase(C4Database* db); + + /// Adds a collection to be shared. + /// @note A database's default collection is automatically shared. + /// @param name The URI name the database is registered by. + /// @param collection The C4CollectionSpec identifying the collection. + /// @returns True on success, false if the name is not registered. */ + bool registerCollection(const std::string& name, C4CollectionSpec const& collection); + + /// Unregisters a collection. + /// @note You can use this after `registerDatabase` to unregister the default collection. + /// @param name The URI name the database is registered by. + /// @param collection The C4CollectionSpec identifying the collection. + /// @returns True on success, false if the database name or collection is not registered. */ + bool unregisterCollection(const std::string& name, C4CollectionSpec const& collection); + + /// Returns the name a database is registered under. + /// `db` need not be the exact instance that was registered; + /// any instance on the same database file will work. */ + std::optional nameOfDatabase(C4Database*) const; + + /// Returns all registered database names. + std::vector databaseNames() const; + + /** Struct representing a shared database. */ + struct DBShare { + fleece::Retained pool; /// Pool of C4Database instances + std::set keySpaces; /// Shared collections + C4ListenerDatabaseConfig config; /// Configuration + }; + + /// Returns a copy of the sharing info for a database. + std::optional getShare(std::string const& name) const; + + /// Returns a temporary C4Database instance, by the shared name. + BorrowedDatabase borrowDatabaseNamed(const std::string& name, bool writeable) const; + + /// Returns a temporary C4Collection instance, by the shared db name and keyspace. + BorrowedCollection borrowCollection(const std::string& keyspace, bool writeable) const; + + void closeDatabases(); + + private: + DBShare* C4NULLABLE _getShare(std::string const& name); + DBShare const* C4NULLABLE _getShare(std::string const& name) const; + + mutable std::mutex _mutex; + std::map _databases; + }; + +} // namespace litecore::REST + +C4_ASSUME_NONNULL_END + +#endif diff --git a/REST/EE/RESTSyncListener_stub.cc b/REST/EE/RESTSyncListener_stub.cc index 7b716f0c2..c2f26ef9c 100644 --- a/REST/EE/RESTSyncListener_stub.cc +++ b/REST/EE/RESTSyncListener_stub.cc @@ -14,5 +14,5 @@ // NOTE: RESTSyncListener.cc is not in this repo, and is not open source. // It is part of Couchbase Lite Enterprise Edition (EE), which can be licensed in binary form // from Couchbase. -# include "../../../couchbase-lite-core-EE/Listener/RESTSyncListener.cc" +# include "../../../couchbase-lite-core-EE/Listener/SyncListener.cc" #endif diff --git a/REST/HTTPListener.cc b/REST/HTTPListener.cc new file mode 100644 index 000000000..ddf703806 --- /dev/null +++ b/REST/HTTPListener.cc @@ -0,0 +1,263 @@ +// +// HTTPListener.cc +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#include "HTTPListener.hh" +#include "DatabasePool.hh" +#include "c4Certificate.hh" +#include "c4Database.hh" +#include "c4ListenerInternal.hh" +#include "Address.hh" +#include "Headers.hh" +#include "netUtils.hh" +#include "Replicator.hh" +#include "Request.hh" +#include "TCPSocket.hh" +#include "TLSContext.hh" +#include "Certificate.hh" +#include "PublicKey.hh" +#include "Error.hh" +#include "StringUtil.hh" +#include "fleece/Fleece.hh" +#include "slice_stream.hh" + +#ifdef COUCHBASE_ENTERPRISE + +using namespace std; +using namespace fleece; +using namespace litecore::crypto; + +namespace litecore::REST { + using namespace net; + + HTTPListener::HTTPListener(const C4ListenerConfig& config) + : _config(config) + , _serverName(config.serverName ? slice(config.serverName) : "CouchbaseLite"_sl) + , _serverVersion(config.serverVersion ? slice(config.serverVersion) : alloc_slice(c4_getVersion())) + , _server(new Server(*this)) { + _server->start(config.port, config.networkInterface, createTLSContext(config.tlsConfig).get()); + } + + HTTPListener::~HTTPListener() { stop(); } + + void HTTPListener::stop() { + if ( _server ) { + _server->stop(); + stopTasks(); + _registry.closeDatabases(); + _server = nullptr; + } + } + + vector
HTTPListener::addresses(C4Database* dbOrNull, bool webSocketScheme) const { + optional dbNameStr; + slice dbName; + if ( dbOrNull ) { + dbNameStr = _registry.nameOfDatabase(dbOrNull); + if ( dbNameStr ) dbName = *dbNameStr; + } + + string scheme = webSocketScheme ? "ws" : "http"; + if ( _identity ) scheme += 's'; + + uint16_t port = _server->port(); + vector
addresses; + for ( auto& host : _server->addresses() ) addresses.emplace_back(scheme, host, port, dbName); + return addresses; + } + + Retained HTTPListener::loadTLSIdentity(const C4TLSConfig* config) { + if ( !config ) return nullptr; + Retained cert = config->certificate->assertSignedCert(); + Retained privateKey; + switch ( config->privateKeyRepresentation ) { + case kC4PrivateKeyFromKey: + privateKey = config->key->getPrivateKey(); + break; + case kC4PrivateKeyFromCert: +# ifdef PERSISTENT_PRIVATE_KEY_AVAILABLE + privateKey = cert->loadPrivateKey(); + if ( !privateKey ) + error::_throw(error::CryptoError, + "No persistent private key found matching certificate public key"); + break; +# else + error::_throw(error::Unimplemented, "kC4PrivateKeyFromCert not implemented"); +# endif + } + return new Identity(cert, privateKey); + } + + Retained HTTPListener::createTLSContext(const C4TLSConfig* tlsConfig) { + if ( !tlsConfig ) return nullptr; + _identity = loadTLSIdentity(tlsConfig); + + auto tlsContext = retained(new TLSContext(TLSContext::Server)); + tlsContext->setIdentity(_identity); + if ( tlsConfig->requireClientCerts ) tlsContext->requirePeerCert(true); + if ( tlsConfig->rootClientCerts ) tlsContext->setRootCerts(tlsConfig->rootClientCerts->assertSignedCert()); + if ( auto callback = tlsConfig->certAuthCallback; callback ) { + auto context = tlsConfig->tlsCallbackContext; + tlsContext->setCertAuthCallback([callback, this, context](slice certData) { + return callback((C4Listener*)this, certData, context); + }); + } + return tlsContext; + } + +# pragma mark - CONNECTIONS: + + int HTTPListener::connectionCount() { return _server->connectionCount(); } + + bool HTTPListener::registerDatabase(C4Database* db, optional name, + C4ListenerDatabaseConfig const* dbConfigP) { + C4ListenerDatabaseConfig dbConfig; + if ( !dbConfigP ) { + dbConfig = {.allowPush = _config.allowPush, + .allowPull = _config.allowPull, + .enableDeltaSync = _config.enableDeltaSync}; + dbConfigP = &dbConfig; + } + return _registry.registerDatabase(db, name, *dbConfigP); + } + + bool HTTPListener::unregisterDatabase(C4Database* db) { return _registry.unregisterDatabase(db); } + + bool HTTPListener::registerCollection(const std::string& name, C4CollectionSpec const& collection) { + return _registry.registerCollection(name, collection); + } + + bool HTTPListener::unregisterCollection(const std::string& name, C4CollectionSpec const& collection) { + return _registry.unregisterCollection(name, collection); + } + + void HTTPListener::handleConnection(std::unique_ptr socket) { + // Parse HTTP request: + Request rq(socket.get()); + if ( C4Error err = rq.socketError() ) { + string peer = socket->peerAddress(); + if ( err == C4Error{NetworkDomain, kC4NetErrConnectionReset} ) { + c4log(ListenerLog, kC4LogInfo, "End of socket connection from %s (closed by peer)", peer.c_str()); + } else { + c4log(ListenerLog, kC4LogError, "Error reading HTTP request from %s: %s", peer.c_str(), + err.description().c_str()); + } + return; + } + + websocket::Headers headers; + headers.add("Date", timestamp()); + headers.add("Server", _serverName + "/" + _serverVersion); + HTTPStatus status; + + // HTTP auth: + if ( auto authCallback = _config.httpAuthCallback ) { + if ( !authCallback(_delegate, rq.header("Authorization"), _config.callbackContext) ) { + c4log(ListenerLog, kC4LogInfo, "Authentication failed"); + headers.add("WWW-Authenticate", "Basic charset=\"UTF-8\""); + writeResponse(HTTPStatus::Unauthorized, headers, socket.get()); + return; + } + } + + // Handle the request: + try { + status = handleRequest(rq, headers, socket); + } catch ( ... ) { + C4Error error = C4Error::fromCurrentException(); + c4log(ListenerLog, kC4LogWarning, "HTTP handler caught C++ exception: %s", error.description().c_str()); + status = StatusFromError(error); + } + if ( socket ) writeResponse(status, headers, socket.get()); + } + + void HTTPListener::writeResponse(HTTPStatus status, websocket::Headers const& headers, TCPSocket* socket) { + const char* statusMessage = StatusMessage(status); + if ( !statusMessage ) statusMessage = ""; + stringstream response; + response << "HTTP/1.1 " << int(status) << ' ' << statusMessage << "\r\n"; + headers.forEach([&](slice name, slice value) { response << name << ": " << value << "\r\n"; }); + response << "\r\n"; + (void)socket->write(response.str()); + } + + string HTTPListener::findMatchingSyncProtocol(DatabaseRegistry::DBShare const& share, string_view clientProtocols) { + auto boolToMode = [](bool enabled) { return enabled ? kC4Passive : kC4Disabled; }; + auto serverProtocols = repl::Replicator::compatibleProtocols(share.pool->getConfiguration().flags, + boolToMode(share.config.allowPush), + boolToMode(share.config.allowPull)); + + for ( auto protocol : split(clientProtocols, ",") ) { + if ( std::ranges::find(serverProtocols, protocol) != serverProtocols.end() ) return string(protocol); + } + return ""; + } + +# pragma mark - TASKS: + + void HTTPListener::Task::registerTask() { + if ( !_taskID ) { + _timeStarted = ::time(nullptr); + _taskID = _listener->registerTask(this); + } + } + + void HTTPListener::Task::unregisterTask() { + if ( _taskID ) { + _taskID = 0; + _listener->unregisterTask(this); + } + } + + void HTTPListener::Task::bumpTimeUpdated() { _timeUpdated = ::time(nullptr); } + + void HTTPListener::Task::writeDescription(JSONEncoder& json) { + unsigned long age = ::time(nullptr) - _timeStarted; + json.writeFormatted("task_id: %u, age_secs: %lu", _taskID, age); + } + + unsigned HTTPListener::registerTask(Task* task) { + lock_guard lock(_mutex); + _tasks.insert(task); + return _nextTaskID++; + } + + void HTTPListener::unregisterTask(Task* task) { + lock_guard lock(_mutex); + _tasks.erase(task); + _tasksCondition.notify_all(); + } + + vector> HTTPListener::tasks() { + lock_guard lock(_mutex); + vector> result; + for ( auto i = _tasks.begin(); i != _tasks.end(); ) { + if ( (*i)->listed() ) result.push_back(*i++); + else + i = _tasks.erase(i); // Clean up old finished tasks + } + return result; + } + + void HTTPListener::stopTasks() { + auto allTasks = tasks(); + if ( !allTasks.empty() ) { + for ( auto& task : allTasks ) { + if ( !task->finished() ) task->stop(); + } + unique_lock lock(_mutex); + _tasksCondition.wait(lock, [this] { return _tasks.empty(); }); + } + } + +} // namespace litecore::REST + +#endif diff --git a/REST/HTTPListener.hh b/REST/HTTPListener.hh new file mode 100644 index 000000000..429c79cb3 --- /dev/null +++ b/REST/HTTPListener.hh @@ -0,0 +1,164 @@ +// +// HTTPListener.hh +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#pragma once +#include "DatabaseRegistry.hh" +#include "HTTPTypes.hh" +#include "Server.hh" +#include "fleece/InstanceCounted.hh" +#include "fleece/RefCounted.hh" +#include +#include +#include +#include +#include + +#ifdef COUCHBASE_ENTERPRISE + +C4_ASSUME_NONNULL_BEGIN + +namespace fleece { + class JSONEncoder; +} + +namespace litecore::net { + struct Address; + class TCPSocket; +} // namespace litecore::net + +namespace litecore::websocket { + class Headers; +} + +namespace litecore::REST { + using fleece::RefCounted; + using fleece::Retained; + using namespace litecore::net; + + class Request; + + /** Listener subclass that serves HTTP requests. */ + class HTTPListener + : public RefCounted + , public InstanceCountedIn + , protected Server::Delegate { + public: + explicit HTTPListener(const C4ListenerConfig&); + ~HTTPListener() override; + + void setDelegate(C4Listener* d) { _delegate = d; } + + void stop(); + + uint16_t port() const { return _server->port(); } + + /** My root URL, or the URL of a database. */ + virtual std::vector
addresses(C4Database* C4NULLABLE dbOrNull = nullptr, + bool webSocketScheme = false) const; + + int connectionCount(); + + int activeConnectionCount() { return (int)tasks().size(); } + + bool registerDatabase(C4Database* db, std::optional name = std::nullopt, + C4ListenerDatabaseConfig const* C4NULLABLE dbConfig = nullptr); + bool unregisterDatabase(C4Database*); + bool registerCollection(const std::string& name, C4CollectionSpec const& collection); + bool unregisterCollection(const std::string& name, C4CollectionSpec const& collection); + + /** An asynchronous task (like a replication). */ + class Task + : public RefCounted + , public InstanceCountedIn { + public: + explicit Task(HTTPListener* listener) : _listener(listener) {} + + HTTPListener* listener() const { return _listener; } + + /// A unique integer ID, assigned when registerTask is called (until then, 0.) + unsigned taskID() const { return _taskID; } + + /// The time activity last occurred (i.e. when bumpTimeUpdated was called.) + time_t timeUpdated() const { return _timeUpdated; } + + /// Call this when activity occurs: it sets timeUpdated to now. + void bumpTimeUpdated(); + + /// Should return true if the task should be included in `tasks()`. + virtual bool listed() { return !finished(); } + + /// Should return true if the Task has completed its work. + virtual bool finished() const = 0; + + /// Should add keys+values to the encoder to describe the Task. + virtual void writeDescription(fleece::JSONEncoder&); + + /// Should stop whatever activity the Task is doing. + virtual void stop() = 0; + + void registerTask(); ///< Call this before returning from handler + void unregisterTask(); ///< Call this when the Task is finished. + + protected: + mutable std::recursive_mutex _mutex; + + private: + HTTPListener* const _listener; + unsigned _taskID{0}; + std::atomic _timeStarted{0}; + time_t _timeUpdated{0}; + }; + + /// The currently-running tasks. + std::vector> tasks(); + + protected: + friend class Task; + + Retained createTLSContext(const C4TLSConfig* C4NULLABLE); + static Retained loadTLSIdentity(const C4TLSConfig* C4NULLABLE); + + Server* server() const { return _server.get(); } + + unsigned registerTask(Task*); + void unregisterTask(Task*); + + // Socket::Delegate API + void handleConnection(std::unique_ptr) override; + + virtual HTTPStatus handleRequest(Request&, websocket::Headers&, std::unique_ptr&) = 0; + + void writeResponse(HTTPStatus, websocket::Headers const&, TCPSocket*); + + std::string findMatchingSyncProtocol(DatabaseRegistry::DBShare const&, std::string_view clientProtocols); + + C4ListenerConfig const _config; + C4Listener* C4NULLABLE _delegate = nullptr; + std::string _serverName, _serverVersion; + DatabaseRegistry _registry; + std::mutex _mutex; + + private: + void stopTasks(); + + Retained _identity; + Retained _server; + std::set> _tasks; + std::condition_variable _tasksCondition; + unsigned _nextTaskID{1}; + }; + +} // namespace litecore::REST + +C4_ASSUME_NONNULL_END + +#endif diff --git a/REST/Listener.cc b/REST/Listener.cc deleted file mode 100644 index 1ef184c45..000000000 --- a/REST/Listener.cc +++ /dev/null @@ -1,143 +0,0 @@ -// -// Listener.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "Listener.hh" -#include "c4Database.hh" -#include "c4ListenerInternal.hh" -#include "Error.hh" -#include - -using namespace std; -using namespace fleece; - -namespace litecore::REST { - - - Listener::Listener(const Config& config) : _config(config) { - if ( !ListenerLog ) ListenerLog = c4log_getDomain("Listener", true); - } - - string Listener::databaseNameFromPath(const FilePath& path) { - string name = path.fileOrDirName(); - auto split = FilePath::splitExtension(name); - if ( split.second != kC4DatabaseFilenameExtension ) - error::_throw(error::InvalidParameter, "Not a database path"); - name = split.first; - - // Make the name legal as a URI component in the REST API. - // It shouldn't be empty, nor start with an underscore, nor contain control characters. - if ( name.empty() ) name = "db"; - else if ( name[0] == '_' ) - name[0] = '-'; - for ( char& c : name ) { - if ( iscntrl(c) || c == '/' ) c = '-'; - } - return name; - } - - bool Listener::isValidDatabaseName(const string& name) { - if ( name.empty() || name.size() > 240 || name[0] == '_' ) return false; - return std::all_of(name.begin(), name.end(), [](auto& c) { return !iscntrl(c); }); - } - - bool Listener::registerDatabase(C4Database* db, optional name) { - if ( !name ) { - alloc_slice path(db->getPath()); - name = databaseNameFromPath(FilePath(string(path))); - } else if ( !isValidDatabaseName(*name) ) { - error::_throw(error::InvalidParameter, "Invalid name for sharing a database"); - } - lock_guard lock(_mutex); - if ( _databases.find(*name) != _databases.end() ) return false; - _databases.emplace(*name, db); - return true; - } - - bool Listener::unregisterDatabase(const std::string& name) { - lock_guard lock(_mutex); - auto i = _databases.find(name); - if ( i == _databases.end() ) return false; - _databases.erase(i); - - auto j = _allowedCollections.find(name); - if ( j != _allowedCollections.end() ) { _allowedCollections.erase(j); } - - return true; - } - - bool Listener::unregisterDatabase(C4Database* db) { - lock_guard lock(_mutex); - for ( auto i = _databases.begin(); i != _databases.end(); ++i ) { - if ( i->second == db ) { - _databases.erase(i); - return true; - } - } - return false; - } - - bool Listener::registerCollection(const string& name, CollectionSpec collection) { - lock_guard lock(_mutex); - auto i = _databases.find(name); - if ( i == _databases.end() ) return false; - - auto j = _allowedCollections.find(name); - if ( j == _allowedCollections.end() ) { - vector collections({collection}); - _allowedCollections.emplace(name, collections); - } else { - j->second.push_back(collection); - } - - return true; - } - - bool Listener::unregisterCollection(const string& name, CollectionSpec collection) { - lock_guard lock(_mutex); - auto i = _allowedCollections.find(name); - if ( i == _allowedCollections.end() ) return false; - - for ( auto j = i->second.begin(); j != i->second.end(); j++ ) { - if ( *j == collection ) { - i->second.erase(j); - return true; - } - } - - return false; - } - - Retained Listener::databaseNamed(const string& name) const { - lock_guard lock(_mutex); - auto i = _databases.find(name); - if ( i == _databases.end() ) return nullptr; - // Retain the database to avoid a race condition if it gets unregistered while this - // thread's handler is still using it. - return i->second; - } - - optional Listener::nameOfDatabase(C4Database* db) const { - lock_guard lock(_mutex); - for ( auto& [aName, aDB] : _databases ) - if ( aDB == db ) return aName; - return nullopt; - } - - vector Listener::databaseNames() const { - lock_guard lock(_mutex); - vector names; - names.reserve(_databases.size()); - for ( auto& d : _databases ) names.push_back(d.first); - return names; - } - -} // namespace litecore::REST diff --git a/REST/Listener.hh b/REST/Listener.hh deleted file mode 100644 index 6e5323174..000000000 --- a/REST/Listener.hh +++ /dev/null @@ -1,90 +0,0 @@ -// -// Listener.hh -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#pragma once -#include "fleece/RefCounted.hh" -#include "fleece/InstanceCounted.hh" -#include "c4Database.hh" -#include "c4ListenerTypes.h" -#include "FilePath.hh" -#include -#include -#include -#include - -namespace litecore::REST { - - /** Abstract superclass of network listeners that can serve access to databases. - Subclassed by RESTListener. */ - class Listener - : public fleece::RefCounted - , public fleece::InstanceCountedIn { - public: - using Config = C4ListenerConfig; - using CollectionSpec = C4Database::CollectionSpec; - - static constexpr uint16_t kDefaultPort = 4984; - - explicit Listener(const Config& config); - ~Listener() override = default; - - /** Determines whether a database name is valid for use as a URI path component. - It must be nonempty, no more than 240 bytes long, not start with an underscore, - and contain no control characters. */ - static bool isValidDatabaseName(const std::string&); - - /** Given a filesystem path to a database, returns the database name. - (This takes the last path component and removes the ".cblite2" extension.) - Returns an empty string if the path is not a database, or if the name would not - be valid according to isValidDatabaseName(). */ - static std::string databaseNameFromPath(const FilePath&); - - /** Makes a database visible via the REST API. - Retains the C4Database; the caller does not need to keep a reference to it. */ - bool registerDatabase(C4Database* NONNULL, std::optional name = std::nullopt); - - /** Unregisters a database by name. - The C4Database will be closed if there are no other references to it. */ - bool unregisterDatabase(const std::string& name); - - bool unregisterDatabase(C4Database* db); - - /** Adds a collection to be used by Sync Listener on a given database shared via name. - Only collections that are registered will be replicated. - If none are registered, the default collection will be replicated */ - bool registerCollection(const std::string& name, CollectionSpec collection); - - bool unregisterCollection(const std::string& name, CollectionSpec collection); - - /** Returns the database registered under the given name. */ - fleece::Retained databaseNamed(const std::string& name) const; - - /** Returns the name a database is registered under. */ - std::optional nameOfDatabase(C4Database* NONNULL) const; - - /** Returns all registered database names. */ - std::vector databaseNames() const; - - /** Returns the number of client connections. */ - virtual int connectionCount() = 0; - - /** Returns the number of active client connections (for some definition of "active"). */ - virtual int activeConnectionCount() = 0; - - protected: - mutable std::mutex _mutex; - Config _config; - std::map> _databases; - std::map> _allowedCollections; - }; - -} // namespace litecore::REST diff --git a/REST/RESTListener+Handlers.cc b/REST/RESTListener+Handlers.cc deleted file mode 100644 index 00ee4929d..000000000 --- a/REST/RESTListener+Handlers.cc +++ /dev/null @@ -1,385 +0,0 @@ - -// RESTListener+Handlers.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "RESTListener.hh" -#include "c4Private.h" -#include "c4Collection.hh" -#include "c4Document.hh" -#include "c4Database.hh" -#include "c4DocEnumerator.hh" -#include "fleece/Expert.hh" -#include - -using namespace std; -using namespace fleece; - -namespace litecore::REST { - using namespace net; - -#pragma mark - ROOT HANDLERS: - - void RESTListener::handleGetRoot(RequestResponse& rq) { - alloc_slice version(c4_getVersion()); - auto& json = rq.jsonEncoder(); - json.beginDict(); - json.writeKey("couchdb"_sl); - json.writeString("Welcome"_sl); - json.writeKey("vendor"_sl); - json.beginDict(); - json.writeKey("name"_sl); - json.writeString(kServerName); - json.writeKey("version"_sl); - json.writeString(version); - json.endDict(); - json.writeKey("version"_sl); - json.writeString(serverNameAndVersion()); - json.endDict(); - } - - void RESTListener::handleGetAllDBs(RequestResponse& rq) { - auto& json = rq.jsonEncoder(); - json.beginArray(); - for ( string& name : databaseNames() ) json.writeString(name); - json.endArray(); - } - - void RESTListener::handleActiveTasks(RequestResponse& rq) { - auto& json = rq.jsonEncoder(); - json.beginArray(); - for ( auto& task : tasks() ) { - json.beginDict(); - task->writeDescription(json); - json.endDict(); - } - json.endArray(); - } - -#pragma mark - DATABASE HANDLERS: - - void RESTListener::handleGetDatabase(RequestResponse& rq, C4Collection* coll) { - C4Database* db = coll->getDatabase(); - optional dbName = nameOfDatabase(db); - if ( !dbName ) return rq.respondWithStatus(HTTPStatus::NotFound); - auto docCount = coll->getDocumentCount(); - auto lastSequence = coll->getLastSequence(); - C4UUID uuid = db->getPublicUUID(); - auto uuidStr = slice(&uuid, sizeof(uuid)).hexString(); - slice scope = coll->getScope(); - if ( !scope ) scope = kC4DefaultScopeID; - - auto& json = rq.jsonEncoder(); - json.beginDict(); - json.writeKey("db_name"_sl); - json.writeString(*dbName); - json.writeKey("db_uuid"_sl); - json.writeString(uuidStr); - json.writeKey("scope_name"_sl); - json.writeString(scope); - json.writeKey("collection_name"_sl); - json.writeString(coll->getName()); - json.writeKey("doc_count"_sl); - json.writeUInt(docCount); - json.writeKey("update_seq"_sl); - json.writeUInt(uint64_t(lastSequence)); - json.writeKey("committed_update_seq"_sl); - json.writeUInt(uint64_t(lastSequence)); - - if ( !collectionGiven(rq) ) { - // List all the scope and collections: - json.writeKey("scopes"); - json.beginDict(); - db->forEachScope([&](slice scope) { - json.writeKey(scope); - json.beginDict(); - db->forEachCollection(scope, [&](C4CollectionSpec const& spec) { - C4Collection* coll = db->getCollection(spec); - json.writeKey(spec.name); - json.beginDict(); - json.writeKey("doc_count"_sl); - json.writeUInt(coll->getDocumentCount()); - json.writeKey("update_seq"_sl); - json.writeUInt(uint64_t(coll->getLastSequence())); - json.endDict(); - }); - json.endDict(); - }); - json.endDict(); - } - json.endDict(); - } - - void RESTListener::handleCreateDatabase(RequestResponse& rq) { - string keySpace = rq.path(0); - auto [dbName, spec] = parseKeySpace(keySpace); - auto db = databaseNamed(dbName); - if ( !collectionGiven(rq) ) { - // No collection given: create database: - if ( !_allowCreateDB ) return rq.respondWithStatus(HTTPStatus::Forbidden, "Cannot create databases"); - if ( db ) return rq.respondWithStatus(HTTPStatus::PreconditionFailed, "Database exists"); - FilePath path; - if ( !pathFromDatabaseName(dbName, path) ) - return rq.respondWithStatus(HTTPStatus::BadRequest, "Invalid database name"); - - db = C4Database::openNamed(dbName, {slice(path.dirName()), kC4DB_Create}); - _c4db_setDatabaseTag(db, DatabaseTag_RESTListener); - registerDatabase(db, dbName); - - rq.respondWithStatus(HTTPStatus::Created, "Created"); - - } else { - // Create collection in database: - if ( !_allowCreateCollection ) - return rq.respondWithStatus(HTTPStatus::Forbidden, "Cannot create collections"); - if ( !db ) return rq.respondWithStatus(HTTPStatus::NotFound, "No such database"); - if ( db->getCollection(spec) ) - return rq.respondWithStatus(HTTPStatus::PreconditionFailed, "Collection exists"); - (void)db->createCollection(spec); // This will throw on error - rq.respondWithStatus(HTTPStatus::Created, "Created"); - } - } - - void RESTListener::handleDeleteDatabase(RequestResponse& rq, C4Collection* coll) { - auto db = coll->getDatabase(); - if ( !collectionGiven(rq) ) { - // No collection given; delete database: - if ( !_allowDeleteDB ) return rq.respondWithStatus(HTTPStatus::Forbidden, "Cannot delete databases"); - optional dbName = nameOfDatabase(db); - if ( !dbName ) return rq.respondWithStatus(HTTPStatus::NotFound); - if ( !unregisterDatabase(*dbName) ) return rq.respondWithStatus(HTTPStatus::NotFound); - try { - db->closeAndDeleteFile(); - } catch ( ... ) { - registerDatabase(db, *dbName); - rq.respondWithError(C4Error::fromCurrentException()); - } - - } else { - // Delete scope/collection: - if ( !_allowDeleteCollection ) - return rq.respondWithStatus(HTTPStatus::Forbidden, "Cannot delete collections"); - db->deleteCollection(coll->getSpec()); - } - } - -#pragma mark - DOCUMENT HANDLERS: - - void RESTListener::handleGetAllDocs(RequestResponse& rq, C4Collection* coll) { - // Apply options: - C4EnumeratorOptions options; - options.flags = kC4IncludeNonConflicted; - if ( rq.boolQuery("descending") ) options.flags |= kC4Descending; - bool includeDocs = rq.boolQuery("include_docs"); - if ( includeDocs ) options.flags |= kC4IncludeBodies; - int64_t skip = rq.intQuery("skip", 0); - int64_t limit = rq.intQuery("limit", INT64_MAX); - // TODO: Implement startkey, endkey, etc. - - // Create enumerator: - C4DocEnumerator e(coll, options); - - // Enumerate, building JSON: - auto& json = rq.jsonEncoder(); - json.beginDict(); - json.writeKey("rows"_sl); - json.beginArray(); - while ( e.next() ) { - if ( skip-- > 0 ) continue; - else if ( limit-- <= 0 ) - break; - C4DocumentInfo info = e.documentInfo(); - json.beginDict(); - json.writeKey("key"_sl); - json.writeString(info.docID); - json.writeKey("id"_sl); - json.writeString(info.docID); - json.writeKey("value"_sl); - json.beginDict(); - json.writeKey("rev"_sl); - json.writeString(info.revID); - json.endDict(); - - if ( includeDocs ) { - json.writeKey("doc"_sl); - expert(json).writeRaw(e.getDocument()->bodyAsJSON()); - } - json.endDict(); - } - json.endArray(); - json.endDict(); - } - - void RESTListener::handleGetDoc(RequestResponse& rq, C4Collection* coll) { - string docID = rq.path(1); - string revID = rq.query("rev"); - Retained doc = coll->getDocument(docID, true, (revID.empty() ? kDocGetCurrentRev : kDocGetAll)); - if ( doc ) { - if ( revID.empty() ) { - if ( doc->flags() & kDocDeleted ) doc = nullptr; - else - revID = doc->revID().asString(); - } else { - if ( !doc->selectRevision(revID) ) doc = nullptr; - } - } - if ( !doc ) return rq.respondWithStatus(HTTPStatus::NotFound); - - // Get the revision - alloc_slice json = doc->bodyAsJSON(false); - - // Splice the _id and _rev into the start of the JSON: - rq.setHeader("Content-Type", "application/json"); - rq.write(R"({"_id":")"); - rq.write(docID); - rq.write(R"(","_rev":")"); - rq.write(revID); - if ( doc->selectedRev().flags & kRevDeleted ) rq.write(R"(","_deleted":true)"); - if ( json.size > 2 ) { - rq.write("\","); - slice suffix = json; - suffix.moveStart(1); - rq.write(suffix); - } else { - rq.write("}"); - } - } - - // Core code for create/update/delete operation on a single doc. - bool RESTListener::modifyDoc(Dict body, string docID, const string& revIDQuery, bool deleting, bool newEdits, - C4Collection* coll, fleece::JSONEncoder& json, C4Error* outError) noexcept { - try { - if ( !deleting && !body ) { - c4error_return(WebSocketDomain, (int)HTTPStatus::BadRequest, C4STR("body must be a JSON object"), - outError); - return false; - } - - // Get the revID from either the JSON body or the "rev" query param: - slice revID = body["_rev"_sl].asString(); - if ( !revIDQuery.empty() ) { - if ( !revID ) { - revID = slice(revIDQuery); - } else if ( revID != slice(revIDQuery) ) { - c4error_return(WebSocketDomain, (int)HTTPStatus::BadRequest, C4STR("\"_rev\" conflicts with ?rev"), - outError); - return false; - } - } - - if ( docID.empty() ) { - docID = slice(body["_id"].asString()).asString(); - if ( docID.empty() && revID ) { - // Can't specify revID on a POST - c4error_return(WebSocketDomain, (int)HTTPStatus::BadRequest, C4STR("Missing \"_id\""), outError); - return false; - } - } - - if ( !newEdits && (!revID || docID.empty()) ) { - c4error_return(WebSocketDomain, (int)HTTPStatus::BadRequest, - C4STR("Both \"_id\" and \"_rev\" must be given when \"new_edits\" is false"), outError); - return false; - } - - if ( body["_deleted"_sl].asBool() ) deleting = true; - - Retained doc; - { - C4Database::Transaction t(coll->getDatabase()); - - // Encode body as Fleece (and strip _id and _rev): - alloc_slice encodedBody; - if ( body ) - encodedBody = - doc->encodeStrippingOldMetaProperties(body, coll->getDatabase()->getFleeceSharedKeys()); - - // Save the revision: - C4Slice history[1] = {revID}; - C4DocPutRequest put = {}; - put.allocedBody = {(void*)encodedBody.buf, encodedBody.size}; - if ( !docID.empty() ) put.docID = slice(docID); - put.revFlags = (deleting ? kRevDeleted : 0); - put.existingRevision = !newEdits; - put.allowConflict = false; - put.history = history; - put.historyCount = revID ? 1 : 0; - put.save = true; - - doc = coll->putDocument(put, nullptr, outError); - if ( !doc ) return false; - t.commit(); - } - - json.writeKey("ok"_sl); - json.writeBool(true); - json.writeKey("id"_sl); - json.writeString(doc->docID()); - json.writeKey("rev"_sl); - json.writeString(doc->selectedRev().revID); - return true; - } catch ( ... ) { - *outError = C4Error::fromCurrentException(); - return false; - } - } - - // This handles PUT and DELETE of a document, as well as POST to a database. - void RESTListener::handleModifyDoc(RequestResponse& rq, C4Collection* coll) { - string docID = rq.path(1); // will be empty for POST - - // Parse the body: - bool deleting = (rq.method() == Method::DELETE); - Dict body = rq.bodyAsJSON().asDict(); - if ( !body ) { - if ( !deleting || rq.body() ) - return rq.respondWithStatus(HTTPStatus::BadRequest, "Invalid JSON in request body"); - } - - auto& json = rq.jsonEncoder(); - json.beginDict(); - C4Error error; - if ( !modifyDoc(body, docID, rq.query("rev"), deleting, true, coll, json, &error) ) { - rq.respondWithError(error); - return; - } - json.endDict(); - if ( deleting ) rq.setStatus(HTTPStatus::OK, "Deleted"); - else - rq.setStatus(HTTPStatus::Created, "Created"); - } - - void RESTListener::handleBulkDocs(RequestResponse& rq, C4Collection* coll) { - Dict body = rq.bodyAsJSON().asDict(); - Array docs = body["docs"].asArray(); - if ( !docs ) - return rq.respondWithStatus(HTTPStatus::BadRequest, - "Request body is invalid JSON, or has no \"docs\" array"); - - Value v = body["new_edits"]; - bool newEdits = v ? v.asBool() : true; - - C4Database::Transaction t(coll->getDatabase()); - - auto& json = rq.jsonEncoder(); - json.beginArray(); - for ( Array::iterator i(docs); i; ++i ) { - json.beginDict(); - Dict doc = i.value().asDict(); - C4Error error; - if ( !modifyDoc(doc, "", "", false, newEdits, coll, json, &error) ) rq.writeErrorJSON(error); - json.endDict(); - } - json.endArray(); - - t.commit(); - } - -} // namespace litecore::REST diff --git a/REST/RESTListener+Replicate.cc b/REST/RESTListener+Replicate.cc deleted file mode 100644 index 559780e44..000000000 --- a/REST/RESTListener+Replicate.cc +++ /dev/null @@ -1,343 +0,0 @@ -// -// RESTListener+Replicate.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "RESTListener.hh" -#include "c4Database.hh" -#include "c4Replicator.hh" -#include "c4ListenerInternal.hh" -#include "fleece/RefCounted.hh" -#include "ReplicatorOptions.hh" -#include "StringUtil.hh" -#include "fleece/Expert.hh" // for AllocedDict -#include -#include -#include -#include - -using namespace std; -using namespace fleece; - -namespace litecore::REST { - using namespace net; - - class ReplicationTask : public RESTListener::Task { - public: - using Mutex = recursive_mutex; - using Lock = unique_lock; - - ReplicationTask(RESTListener* listener, slice source, slice target, bool bidi, bool continuous) - : Task(listener), _source(source), _target(target), _bidi(bidi), _continuous(continuous) {} - - void start(C4Database* localDB, C4String localDbName, const C4Address& remoteAddress, C4String remoteDbName, - C4ReplicatorMode pushMode, C4ReplicatorMode pullMode, - const std::vector& collections = {kC4DefaultCollectionSpec}) { - if ( findMatchingTask() ) C4Error::raise(WebSocketDomain, 409, "Equivalent replication already running"); - - Lock lock(_mutex); - _push = (pushMode >= kC4OneShot); - registerTask(); - try { - c4log(ListenerLog, kC4LogInfo, - "Replicator task #%d starting: local=%.*s, mode=%s, scheme=%.*s, host=%.*s," - " port=%u, db=%.*s, bidi=%d, continuous=%d", - taskID(), SPLAT(localDbName), (pushMode > kC4Disabled ? "push" : "pull"), - SPLAT(remoteAddress.scheme), SPLAT(remoteAddress.hostname), remoteAddress.port, - SPLAT(remoteDbName), _bidi, _continuous); - - std::vector replCollections{collections.size()}; - for ( size_t i = 0; i < collections.size(); ++i ) { - replCollections[i].collection = collections[i]; - replCollections[i].push = pushMode; - replCollections[i].pull = pullMode; - } - C4ReplicatorParameters params{}; - AllocedDict optionsDict; - params.collectionCount = collections.size(); - params.collections = replCollections.data(); - params.onStatusChanged = [](C4Replicator*, C4ReplicatorStatus status, void* context) { - ((ReplicationTask*)context)->onReplStateChanged(status); - }; - params.callbackContext = this; - if ( _user ) { - Encoder enc; - enc.beginDict(); - enc.writeKey(C4STR(kC4ReplicatorOptionAuthentication)); - enc.beginDict(); - enc.writeKey(C4STR(kC4ReplicatorAuthType)); - enc.writeString(kC4AuthTypeBasic); - enc.writeKey(C4STR(kC4ReplicatorAuthUserName)); - enc.writeString(_user); - enc.writeKey(C4STR(kC4ReplicatorAuthPassword)); - enc.writeString(_password); - enc.endDict(); - enc.endDict(); - optionsDict = AllocedDict(enc.finish()); - params.optionsDictFleece = optionsDict.data(); - } - _repl = localDB->newReplicator(remoteAddress, remoteDbName, params); - _repl->start(); - } catch ( ... ) { - c4log(ListenerLog, kC4LogInfo, "Replicator task #%d failed to start!", taskID()); - unregisterTask(); - throw; - } - onReplStateChanged(_repl->getStatus()); - } - - ReplicationTask* findMatchingTask() { - for ( const auto& task : listener()->tasks() ) { - // Note that either direction is considered a match - auto* repl = dynamic_cast(task.get()); - if ( repl - && ((repl->_source == _source && repl->_target == _target) - || (repl->_source == _target && repl->_target == _source)) ) { - return repl; - } - } - return nullptr; - } - - // Cancel any existing task with the same parameters as me: - bool cancelExisting() { - if ( auto task = findMatchingTask(); task ) { - task->stop(); - return true; - } - return false; - } - - bool finished() const override { - Lock lock(_mutex); - return _finalResult != HTTPStatus::undefined; - } - - C4ReplicatorStatus status() { - Lock lock(_mutex); - return _status; - } - - alloc_slice message() { - Lock lock(_mutex); - return _message; - } - - void writeDescription(fleece::JSONEncoder& json) override { - Task::writeDescription(json); - - json.writeKey("type"_sl); - json.writeString("replication"_sl); - json.writeKey("session_id"_sl); - json.writeUInt(taskID()); - json.writeKey("source"_sl); - json.writeString(_source); - json.writeKey("target"_sl); - json.writeString(_target); - if ( _continuous ) { - json.writeKey("continuous"_sl); - json.writeBool(true); - } - if ( _bidi ) { - json.writeKey("bidi"_sl); - json.writeBool(true); - } - - Lock lock(_mutex); - - json.writeKey("updated_on"_sl); - json.writeUInt(_timeUpdated); - - static slice const kStatusName[] = {"Stopped"_sl, "Offline"_sl, "Connecting"_sl, "Idle"_sl, "Active"_sl}; - json.writeKey("status"_sl); - json.writeString(kStatusName[_status.level]); - - if ( _status.error.code ) { - json.writeKey("error"_sl); - writeErrorInfo(json); - } - - if ( _status.progress.unitsTotal > 0 ) { - double fraction = narrow_cast(_status.progress.unitsCompleted) * 100.0 - / narrow_cast(_status.progress.unitsTotal); - json.writeKey("progress"_sl); - json.writeInt(int64_t(fraction)); - } - - if ( _status.progress.documentCount > 0 ) { - slice key; - if ( _bidi ) key = "docs_transferred"_sl; - else - key = _push ? "docs_written"_sl : "docs_read"_sl; - json.writeKey(key); - json.writeUInt(_status.progress.documentCount); - } - } - - void writeErrorInfo(JSONEncoder& json) { - Lock lock(_mutex); - json.beginDict(); - json.writeKey("error"_sl); - json.writeString(_message); - json.writeKey("x-litecore-domain"_sl); - json.writeInt(_status.error.domain); - json.writeKey("x-litecore-code"_sl); - json.writeInt(_status.error.code); - json.endDict(); - } - - HTTPStatus wait() { - Lock lock(_mutex); - _cv.wait(lock, [this] { return finished(); }); - return _finalResult; - } - - void stop() override { - Lock lock(_mutex); - if ( _repl ) { - c4log(ListenerLog, kC4LogInfo, "Replicator task #%u stopping...", taskID()); - _repl->stop(); - } - } - - void setAuth(slice user, slice psw) { - _user = user; - _password = psw; - } - - - private: - void onReplStateChanged(const C4ReplicatorStatus& status) { - { - Lock lock(_mutex); - _status = status; - _message = c4error_getMessage(status.error); - if ( status.level == kC4Stopped ) { - _finalResult = status.error.code ? HTTPStatus::GatewayError : HTTPStatus::OK; - _repl = nullptr; - } - time(&_timeUpdated); - } - if ( finished() ) { - c4log(ListenerLog, kC4LogInfo, "Replicator task #%u finished", taskID()); - _cv.notify_all(); - } - //unregisterTask(); --no, leave it so a later call to _active_tasks can get its state - } - - alloc_slice _source, _target; - alloc_slice _user, _password; - bool _bidi, _continuous, _push{}; - mutable Mutex _mutex; - condition_variable_any _cv; - Retained _repl; - C4ReplicatorStatus _status{}; - alloc_slice _message; - HTTPStatus _finalResult{HTTPStatus::undefined}; - }; - -#pragma mark - HTTP HANDLER: - - void RESTListener::handleReplicate(litecore::REST::RequestResponse& rq) { - // Parse the JSON body: - auto params = rq.bodyAsJSON().asDict(); - if ( !params ) - return rq.respondWithStatus(HTTPStatus::BadRequest, - "Invalid JSON in request body (or body is not an object)"); - slice source = params["source"].asString(); - slice target = params["target"].asString(); - if ( !source || !target ) - return rq.respondWithStatus(HTTPStatus::BadRequest, "Missing source or target parameters"); - - bool bidi = params["bidi"].asBool(); - bool continuous = params["continuous"].asBool(); - C4ReplicatorMode activeMode = continuous ? kC4Continuous : kC4OneShot; - - Array collections = params["collections"].asArray(); - std::vector collSpecs; - if ( collections.empty() ) { - collSpecs.push_back(kC4DefaultCollectionSpec); - } else { - for ( Array::iterator iter(collections); iter; iter.next() ) { - slice collPath = iter.value().asString(); - collSpecs.push_back(repl::Options::collectionPathToSpec(collPath)); - } - } - - slice localName; - slice remoteURL; - C4ReplicatorMode pushMode, pullMode; - pushMode = pullMode = (bidi ? activeMode : kC4Disabled); - if ( C4Replicator::isValidDatabaseName(source) ) { - localName = source; - pushMode = activeMode; - remoteURL = target; - } else if ( C4Replicator::isValidDatabaseName(target) ) { - localName = target; - pullMode = activeMode; - remoteURL = source; - } else { - return rq.respondWithStatus(HTTPStatus::BadRequest, "Neither source nor target is a local database name"); - } - - Retained localDB = databaseNamed(localName.asString()); - if ( !localDB ) return rq.respondWithStatus(HTTPStatus::NotFound); - - C4Address remoteAddress; - slice remoteDbName; - if ( !C4Address::fromURL(remoteURL, &remoteAddress, &remoteDbName) ) - return rq.respondWithStatus(HTTPStatus::BadRequest, "Invalid database URL"); - - // Start the replication! - Retained task = new ReplicationTask(this, source, target, bidi, continuous); - - if ( params["cancel"].asBool() ) { - // Hang on, stop the presses -- we're canceling, not starting - bool canceled = task->cancelExisting(); - rq.setStatus(canceled ? HTTPStatus::OK : HTTPStatus::NotFound, canceled ? "Stopped" : "No matching task"); - return; - } - // Auth: - slice user = params["user"].asString(); - if ( user ) { - slice psw = params["password"].asString(); - task->setAuth(user, psw); - } - task->start(localDB, localName, remoteAddress, remoteDbName, pushMode, pullMode, collSpecs); - - HTTPStatus statusCode = HTTPStatus::OK; - if ( !continuous ) { - statusCode = task->wait(); - task->unregisterTask(); - } - - auto& json = rq.jsonEncoder(); - if ( statusCode == HTTPStatus::OK ) { - json.beginDict(); - json.writeKey("ok"_sl); - json.writeBool(true); - json.writeKey("session_id"_sl); - json.writeUInt(task->taskID()); - json.endDict(); - } else { - task->writeErrorInfo(json); - } - - string message = task->message().asString(); - if ( statusCode == HTTPStatus::GatewayError ) message = "Replicator error: " + message; - rq.setStatus(statusCode, message.c_str()); - } - - void RESTListener::handleSync(RequestResponse& rq, C4Database*) { - rq.setStatus(HTTPStatus::NotImplemented, nullptr); - } - - -} // namespace litecore::REST diff --git a/REST/RESTListener.cc b/REST/RESTListener.cc deleted file mode 100644 index 0bc83e3de..000000000 --- a/REST/RESTListener.cc +++ /dev/null @@ -1,327 +0,0 @@ -// -// RESTListener.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "RESTListener.hh" -#include "c4Certificate.hh" -#include "c4Database.hh" -#include "Server.hh" -#include "TLSContext.hh" -#include "Certificate.hh" -#include "PublicKey.hh" -#include "Error.hh" -#include "StringUtil.hh" -#include "slice_stream.hh" - -using namespace std; -using namespace fleece; -using namespace litecore::crypto; - -namespace litecore::REST { - using namespace net; - - //static constexpr const char* kKeepAliveTimeoutMS = "1000"; - //static constexpr const char* kMaxConnections = "8"; - - static int kTaskExpirationTime = 10; - - - string RESTListener::kServerName = "LiteCoreServ"; - - string RESTListener::serverNameAndVersion() { - alloc_slice version(c4_getVersion()); - return stringprintf("%s/%.*s", kServerName.c_str(), SPLAT(version)); - } - - RESTListener::RESTListener(const Config& config) - : Listener(config) - , _directory(config.directory.buf ? new FilePath(slice(config.directory).asString(), "") : nullptr) - , _allowCreateDB(config.allowCreateDBs && _directory) - , _allowDeleteDB(config.allowDeleteDBs) - , _allowCreateCollection(config.allowCreateCollections) - , _allowDeleteCollection(config.allowDeleteCollections) { - _server = new Server(); - _server->setExtraHeaders({{"Server", serverNameAndVersion()}}); - - if ( auto callback = config.httpAuthCallback; callback ) { - void* context = config.callbackContext; - _server->setAuthenticator([this, callback, context](slice authorizationHeader) { - return callback((C4Listener*)this, authorizationHeader, context); - }); - } - - if ( config.apis & kC4RESTAPI ) { - // Root: - addHandler(Method::GET, "/", &RESTListener::handleGetRoot); - - // Top-level special handlers: - addHandler(Method::GET, "/_all_dbs", &RESTListener::handleGetAllDBs); - addHandler(Method::GET, "/_active_tasks", &RESTListener::handleActiveTasks); - addHandler(Method::POST, "/_replicate", &RESTListener::handleReplicate); - - // Database: - addCollectionHandler(Method::GET, "/[^_][^/]*|/[^_][^/]*/", &RESTListener::handleGetDatabase); - addHandler(Method::PUT, "/[^_][^/]*|/[^_][^/]*/", &RESTListener::handleCreateDatabase); - addCollectionHandler(Method::DELETE, "/[^_][^/]*|/[^_][^/]*/", &RESTListener::handleDeleteDatabase); - addCollectionHandler(Method::POST, "/[^_][^/]*|/[^_][^/]*/", &RESTListener::handleModifyDoc); - - // Database-level special handlers: - addCollectionHandler(Method::GET, "/[^_][^/]*/_all_docs", &RESTListener::handleGetAllDocs); - addCollectionHandler(Method::POST, "/[^_][^/]*/_bulk_docs", &RESTListener::handleBulkDocs); - - // Document: - addCollectionHandler(Method::GET, "/[^_][^/]*/[^_].*", &RESTListener::handleGetDoc); - addCollectionHandler(Method::PUT, "/[^_][^/]*/[^_].*", &RESTListener::handleModifyDoc); - addCollectionHandler(Method::DELETE, "/[^_][^/]*/[^_].*", &RESTListener::handleModifyDoc); - } - if ( config.apis & kC4SyncAPI ) { - addDBHandler(Method::UPGRADE, "/[^_][^/]*/_blipsync", &RESTListener::handleSync); - } - - _server->start(config.port, config.networkInterface, createTLSContext(config.tlsConfig).get()); - } - - RESTListener::~RESTListener() { stop(); } - - void RESTListener::stop() { - if ( _server ) _server->stop(); - } - - vector
RESTListener::_addresses(C4Database* dbOrNull, C4ListenerAPIs api) const { - optional dbNameStr; - slice dbName; - if ( dbOrNull ) { - dbNameStr = nameOfDatabase(dbOrNull); - if ( dbNameStr ) dbName = *dbNameStr; - } - - slice scheme; - Assert((api == kC4RESTAPI || api == kC4SyncAPI)); - if ( api == kC4RESTAPI ) { - scheme = _identity ? "https" : "http"; - } else if ( api == kC4SyncAPI ) { - scheme = _identity ? "wss" : "ws"; - } - - uint16_t port = _server->port(); - vector
addresses; - for ( auto& host : _server->addresses() ) addresses.emplace_back(scheme, host, port, dbName); - return addresses; - } - - vector
RESTListener::addresses(C4Database* dbOrNull, C4ListenerAPIs api) const { - if ( api != kC4RESTAPI ) { - error::_throw(error::LiteCoreError::InvalidParameter, - "The listener is not running in the specified API mode."); - } - - return _addresses(dbOrNull, api); - } - - -#ifdef COUCHBASE_ENTERPRISE - Retained RESTListener::loadTLSIdentity(const C4TLSConfig* config) { - if ( !config ) return nullptr; - Retained cert = config->certificate->assertSignedCert(); - Retained privateKey; - switch ( config->privateKeyRepresentation ) { - case kC4PrivateKeyFromKey: - privateKey = config->key->getPrivateKey(); - break; - case kC4PrivateKeyFromCert: -# ifdef PERSISTENT_PRIVATE_KEY_AVAILABLE - privateKey = cert->loadPrivateKey(); - if ( !privateKey ) - error::_throw(error::CryptoError, - "No persistent private key found matching certificate public key"); - break; -# else - error::_throw(error::Unimplemented, "kC4PrivateKeyFromCert not implemented"); -# endif - } - return new Identity(cert, privateKey); - } -#endif // COUCHBASE_ENTERPRISE - - - Retained RESTListener::createTLSContext(const C4TLSConfig* tlsConfig) { - if ( !tlsConfig ) return nullptr; -#ifdef COUCHBASE_ENTERPRISE - _identity = loadTLSIdentity(tlsConfig); - - auto tlsContext = retained(new TLSContext(TLSContext::Server)); - tlsContext->setIdentity(_identity); - if ( tlsConfig->requireClientCerts ) tlsContext->requirePeerCert(true); - if ( tlsConfig->rootClientCerts ) tlsContext->setRootCerts(tlsConfig->rootClientCerts->assertSignedCert()); - if ( auto callback = tlsConfig->certAuthCallback; callback ) { - auto context = tlsConfig->tlsCallbackContext; - tlsContext->setCertAuthCallback([callback, this, context](slice certData) { - return callback((C4Listener*)this, certData, context); - }); - } - return tlsContext; -#else - error::_throw(error::Unimplemented, "TLS server is an Enterprise Edition feature"); -#endif - } - - int RESTListener::connectionCount() { return _server->connectionCount(); } - -#pragma mark - REGISTERING DATABASES: - - static void replace(string& str, char oldChar, char newChar) { - for ( char& c : str ) - if ( c == oldChar ) c = newChar; - } - - bool RESTListener::pathFromDatabaseName(const string& name, FilePath& path) { - if ( !_directory || !isValidDatabaseName(name) ) return false; - string filename = name; - replace(filename, '/', ':'); - path = (*_directory)[filename + kC4DatabaseFilenameExtension + "/"]; - return true; - } - -#pragma mark - TASKS: - - void RESTListener::Task::registerTask() { - if ( !_taskID ) { - time(&_timeStarted); - _taskID = _listener->registerTask(this); - } - } - - void RESTListener::Task::unregisterTask() { - if ( _taskID ) { - _listener->unregisterTask(this); - _taskID = 0; - } - } - - void RESTListener::Task::writeDescription(fleece::JSONEncoder& json) { - json.writeKey("pid"_sl); - json.writeUInt(_taskID); - json.writeKey("started_on"_sl); - json.writeUInt(_timeStarted); - } - - unsigned RESTListener::registerTask(Task* task) { - lock_guard lock(_mutex); - _tasks.insert(task); - return _nextTaskID++; - } - - void RESTListener::unregisterTask(Task* task) { - lock_guard lock(_mutex); - _tasks.erase(task); - } - - vector> RESTListener::tasks() { - lock_guard lock(_mutex); - - // Clean up old finished tasks: - time_t now; - time(&now); - for ( auto i = _tasks.begin(); i != _tasks.end(); ) { - if ( (*i)->finished() && (now - (*i)->timeUpdated()) >= kTaskExpirationTime ) i = _tasks.erase(i); - else - ++i; - } - - return {_tasks.begin(), _tasks.end()}; - } - -#pragma mark - UTILITIES: - - void RESTListener::addHandler(Method method, const char* uri, HandlerMethod handler) { - using namespace std::placeholders; - _server->addHandler(method, uri, bind(handler, this, _1)); - } - - void RESTListener::addDBHandler(Method method, const char* uri, DBHandlerMethod handler) { - _server->addHandler(method, uri, [this, handler](RequestResponse& rq) { - Retained db = getDatabase(rq, rq.path(0)); - if ( db ) { - db->lockClientMutex(); - try { - (this->*handler)(rq, db); - } catch ( ... ) { - db->unlockClientMutex(); - throw; - } - db->unlockClientMutex(); - } - }); - } - - void RESTListener::addCollectionHandler(Method method, const char* uri, CollectionHandlerMethod handler) { - _server->addHandler(method, uri, [this, handler](RequestResponse& rq) { - auto [db, collection] = collectionFor(rq); - if ( db ) { - db->lockClientMutex(); - try { - (this->*handler)(rq, collection); - } catch ( ... ) { - db->unlockClientMutex(); - throw; - } - db->unlockClientMutex(); - } - }); - } - - pair RESTListener::parseKeySpace(slice keySpace) { - slice_istream in(keySpace); - slice dbName = in.readToDelimiter("."); - if ( !dbName ) return {string(keySpace), {}}; - C4CollectionSpec spec = {}; - spec.name = in.readToDelimiterOrEnd("."); - if ( in.size > 0 ) { - spec.scope = spec.name; - spec.name = in; - } - return {string(dbName), spec}; - } - - bool RESTListener::collectionGiven(RequestResponse& rq) { return !!slice(rq.path(0)).findByte('.'); } - - Retained RESTListener::getDatabase(RequestResponse& rq, const string& dbName) { - Retained db = databaseNamed(dbName); - if ( !db ) { - if ( isValidDatabaseName(dbName) ) rq.respondWithStatus(HTTPStatus::NotFound, "No such database"); - else - rq.respondWithStatus(HTTPStatus::BadRequest, "Invalid databasename"); - } - return db; - } - - // returning the retained db is necessary because retaining a collection does not retain its db! - pair, C4Collection*> RESTListener::collectionFor(RequestResponse& rq) { - string keySpace = rq.path(0); - auto [dbName, spec] = parseKeySpace(keySpace); - auto db = getDatabase(rq, dbName); - if ( !db ) return {}; - if ( !spec.name.buf ) spec.name = kC4DefaultCollectionName; - C4Collection* collection; - try { - collection = db->getCollection(spec); - } catch ( const std::exception& x ) { - rq.respondWithError(C4Error::fromCurrentException()); - return {}; - } - if ( !collection ) { - rq.respondWithStatus(HTTPStatus::NotFound, "No such collection"); - return {}; - } - return {db, collection}; - } - -} // namespace litecore::REST diff --git a/REST/RESTListener.hh b/REST/RESTListener.hh deleted file mode 100644 index 85d7e2a68..000000000 --- a/REST/RESTListener.hh +++ /dev/null @@ -1,143 +0,0 @@ -// -// RESTListener.hh -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#pragma once -#include "c4DatabaseTypes.h" -#include "Listener.hh" -#include "Server.hh" -#include "FilePath.hh" -#include "fleece/RefCounted.hh" -#include -#include -#include -#include - -namespace litecore::REST { - using fleece::RefCounted; - using fleece::Retained; - class Server; - - /** Listener subclass that serves (some of) the venerable CouchDB REST API. - The HTTP work is done by a Server object. */ - class RESTListener : public Listener { - public: - explicit RESTListener(const Config&); - ~RESTListener() override; - - virtual void stop(); - - uint16_t port() const { return _server->port(); } - - /** My root URL, or the URL of a database. */ - virtual std::vector addresses(C4Database* dbOrNull = nullptr, - C4ListenerAPIs api = kC4RESTAPI) const; - - int connectionCount() override; - - int activeConnectionCount() override { return (int)tasks().size(); } - - /** Given a database name (from a URI path) returns the filesystem path to the database. */ - bool pathFromDatabaseName(const std::string& name, FilePath& outPath); - - /** An asynchronous task (like a replication). */ - class Task : public RefCounted { - public: - explicit Task(RESTListener* listener) : _listener(listener) {} - - RESTListener* listener() const { return _listener; } - - unsigned taskID() const { return _taskID; } - - time_t timeUpdated() const { return _timeUpdated; } - - virtual bool finished() const = 0; - virtual void writeDescription(fleece::JSONEncoder&); - - virtual void stop() = 0; - - void registerTask(); - void unregisterTask(); - - protected: - ~Task() override = default; - - time_t _timeUpdated{0}; - - private: - RESTListener* const _listener; - unsigned _taskID{0}; - time_t _timeStarted{0}; - }; - - /** The currently-running tasks. */ - std::vector> tasks(); - - protected: - friend class Task; - - Retained createTLSContext(const C4TLSConfig*); - static Retained loadTLSIdentity(const C4TLSConfig*); - - Server* server() const { return _server.get(); } - - Retained getDatabase(RequestResponse& rq, const std::string& dbName); - - /** Returns the collection for this request, or null on error */ - std::pair, C4Collection*> collectionFor(RequestResponse&); - unsigned registerTask(Task*); - void unregisterTask(Task*); - - using HandlerMethod = void (RESTListener::*)(RequestResponse&); - using DBHandlerMethod = void (RESTListener::*)(RequestResponse&, C4Database*); - using CollectionHandlerMethod = void (RESTListener::*)(RequestResponse&, C4Collection*); - - void addHandler(net::Method, const char* uri, HandlerMethod); - void addDBHandler(net::Method, const char* uri, DBHandlerMethod); - void addCollectionHandler(net::Method, const char* uri, CollectionHandlerMethod); - - std::vector _addresses(C4Database* dbOrNull = nullptr, C4ListenerAPIs api = kC4RESTAPI) const; - - virtual void handleSync(RequestResponse&, C4Database*); - - static std::string serverNameAndVersion(); - static std::string kServerName; - - private: - static std::pair parseKeySpace(slice keySpace); - static bool collectionGiven(RequestResponse&); - - void handleGetRoot(RequestResponse&); - void handleGetAllDBs(RequestResponse&); - void handleReplicate(RequestResponse&); - void handleActiveTasks(RequestResponse&); - - void handleGetDatabase(RequestResponse&, C4Collection*); - void handleCreateDatabase(RequestResponse&); - void handleDeleteDatabase(RequestResponse&, C4Collection*); - - void handleGetAllDocs(RequestResponse&, C4Collection*); - void handleGetDoc(RequestResponse&, C4Collection*); - void handleModifyDoc(RequestResponse&, C4Collection*); - void handleBulkDocs(RequestResponse&, C4Collection*); - - bool modifyDoc(fleece::Dict body, std::string docID, const std::string& revIDQuery, bool deleting, - bool newEdits, C4Collection* coll, fleece::JSONEncoder& json, C4Error* outError) noexcept; - - std::unique_ptr _directory; - const bool _allowCreateDB, _allowDeleteDB, _allowCreateCollection, _allowDeleteCollection; - Retained _identity; - Retained _server; - std::set> _tasks; - unsigned _nextTaskID{1}; - }; - -} // namespace litecore::REST diff --git a/REST/Request.cc b/REST/Request.cc index affdd9ea4..c53c08ab5 100644 --- a/REST/Request.cc +++ b/REST/Request.cc @@ -12,39 +12,45 @@ #include "Request.hh" #include "HTTPLogic.hh" -#include "Server.hh" -#include "Writer.hh" -#include "Error.hh" -#include "Logging.hh" #include "netUtils.hh" #include "TCPSocket.hh" -#include "date/date.h" #include "slice_stream.hh" -#include -#include #include #include #include -#include -#ifdef _MSC_VER -# include "PlatformIO.hh" -#endif +#ifdef COUCHBASE_ENTERPRISE + +# ifdef _MSC_VER +# include "PlatformIO.hh" +# endif using namespace std; -using namespace std::chrono; using namespace fleece; namespace litecore::REST { using namespace net; -#pragma mark - REQUEST: +# pragma mark - REQUEST: Request::Request(Method method, string path, string queries, websocket::Headers headers, fleece::alloc_slice body) : Body(std::move(headers), std::move(body)) , _method(method) , _path(std::move(path)) - , _queries(std::move(queries)) {} + , _queries(std::move(queries)) + , _version(HTTP1_1) {} + + Request::Request(TCPSocket* socket) { + alloc_slice request = socket->readToDelimiter("\r\n\r\n"_sl); + if ( !request ) { + _error = socket->error(); + if ( _error == C4Error{WebSocketDomain, 400} ) _error = C4Error{NetworkDomain, kC4NetErrConnectionReset}; + } else if ( !readFromHTTP(request) ) { + _error = C4Error::make(WebSocketDomain, int(HTTPStatus::BadRequest)); + } else if ( _method == Method::POST || _method == Method::PUT ) { + if ( !socket->readHTTPBody(_headers, _body) ) { _error = socket->error(); } + } + } bool Request::readFromHTTP(slice httpData) { slice_istream in(httpData); @@ -52,8 +58,17 @@ namespace litecore::REST { _method = Method::None; Method method = MethodNamed(in.readToDelimiter(" "_sl)); slice uri = in.readToDelimiter(" "_sl); + slice http = in.readToDelimiter("/"_sl); slice version = in.readToDelimiter("\r\n"_sl); - if ( method == Method::None || uri.size == 0 || !version.hasPrefix("HTTP/"_sl) ) return false; + if ( method == Method::None || !uri.hasPrefix('/') || http != "HTTP"_sl ) return false; + + if ( version == "1.1" ) _version = HTTP1_1; + else if ( version == "1.0" ) + _version = HTTP1_0; + else + return false; + + if ( !HTTPLogic::parseHeaders(in, _headers) ) return false; const uint8_t* q = uri.findByte('?'); if ( q ) { @@ -64,12 +79,17 @@ namespace litecore::REST { } _path = string(uri); - if ( !HTTPLogic::parseHeaders(in, _headers) ) return false; - _method = method; return true; } + size_t Request::pathLength() const { + Assert(_path[0] == '/'); + return std::count_if(_path.begin(), + _path.end() - _path.ends_with('/'), // skip any trailing '/' + [](char c) { return c == '/'; }); + } + string Request::path(int i) const { slice path = _path; Assert(path[0] == '/'); @@ -97,253 +117,35 @@ namespace litecore::REST { return defaultValue; } - bool Request::boolQuery(const char* param, bool defaultValue) const { - string val = query(param); - if ( val.empty() ) return defaultValue; - return val != "false" && val != "0"; // same behavior as Obj-C CBL 1.x - } - -#pragma mark - RESPONSE STATUS LINE: - - RequestResponse::RequestResponse(Server* server, std::unique_ptr socket) - : _server(server), _socket(std::move(socket)) { - auto request = _socket->readToDelimiter("\r\n\r\n"_sl); - if ( !request ) { - handleSocketError(); - return; - } - if ( !readFromHTTP(request) ) return; - if ( _method == Method::POST || _method == Method::PUT ) { - if ( !_socket->readHTTPBody(_headers, _body) ) { - handleSocketError(); - return; - } - } - } - - void RequestResponse::setStatus(HTTPStatus status, const char* message) { - Assert(!_sentStatus); - _status = status; - _statusMessage = message ? message : ""; - sendStatus(); - } - - void RequestResponse::sendStatus() { - if ( _sentStatus ) return; - Log("Response status: %d", static_cast(_status)); - if ( _statusMessage.empty() ) { - const char* defaultMessage = StatusMessage(_status); - if ( defaultMessage ) _statusMessage = defaultMessage; - } - string statusLine = stringprintf("HTTP/1.1 %d %s\r\n", static_cast(_status), _statusMessage.c_str()); - _responseHeaderWriter.write(statusLine); - _sentStatus = true; - - // Add the 'Date:' header: - stringstream s; - auto tp = floor(system_clock::now()); - s << date::format("%a, %d %b %Y %H:%M:%S GMT", tp); - setHeader("Date", s.str().c_str()); - } - - void RequestResponse::writeStatusJSON(HTTPStatus status, const char* message) { - auto& json = jsonEncoder(); - if ( int(status) < 300 ) { - json.writeKey("ok"_sl); - json.writeBool(true); - } else { - json.writeKey("status"_sl); - json.writeInt(int(status)); - const char* defaultMessage = StatusMessage(status); - if ( defaultMessage ) { - json.writeKey("error"_sl); - json.writeString(defaultMessage); - } - if ( message && defaultMessage && 0 != strcasecmp(message, defaultMessage) ) { - json.writeKey("reason"_sl); - json.writeString(message); - } - } - } - - void RequestResponse::writeErrorJSON(C4Error err) { - alloc_slice message = c4error_getMessage(err); - writeStatusJSON(errorToStatus(err), (message ? message.asString().c_str() : nullptr)); - } - - void RequestResponse::respondWithStatus(HTTPStatus status, const char* message) { - setStatus(status, message); - uncacheable(); - - if ( status >= HTTPStatus::OK && status != HTTPStatus::NoContent && status != HTTPStatus::NotModified ) { - _jsonEncoder.reset(); // drop any prior buffered output - auto& json = jsonEncoder(); - json.beginDict(); - writeStatusJSON(status, message); - json.endDict(); - } - } - - void RequestResponse::respondWithError(C4Error err) { - Assert(err.code != 0); - alloc_slice message = c4error_getMessage(err); - respondWithStatus(errorToStatus(err), (message ? message.asString().c_str() : nullptr)); - } - - HTTPStatus RequestResponse::errorToStatus(C4Error err) { - if ( err.code == 0 ) return HTTPStatus::OK; - HTTPStatus status = HTTPStatus::ServerError; - // TODO: Add more mappings, and make these table-driven - switch ( err.domain ) { - case LiteCoreDomain: - switch ( err.code ) { - case kC4ErrorInvalidParameter: - case kC4ErrorBadRevisionID: - status = HTTPStatus::BadRequest; - break; - case kC4ErrorNotADatabaseFile: - case kC4ErrorCrypto: - status = HTTPStatus::Unauthorized; - break; - case kC4ErrorNotWriteable: - status = HTTPStatus::Forbidden; - break; - case kC4ErrorNotFound: - status = HTTPStatus::NotFound; - break; - case kC4ErrorConflict: - status = HTTPStatus::Conflict; - break; - case kC4ErrorUnimplemented: - case kC4ErrorUnsupported: - status = HTTPStatus::NotImplemented; - break; - case kC4ErrorRemoteError: - status = HTTPStatus::GatewayError; - break; - case kC4ErrorBusy: - status = HTTPStatus::Locked; - break; - } - break; - case WebSocketDomain: - if ( err.code < 1000 ) status = HTTPStatus(err.code); - default: - break; - } - return status; - } - - void RequestResponse::handleSocketError() { - C4Error err = _socket->error(); - WarnError("Socket error sending response: %s", err.description().c_str()); - } - -#pragma mark - RESPONSE HEADERS: - - void RequestResponse::setHeader(const char* header, const char* value) { - sendStatus(); - Assert(!_endedHeaders); - _responseHeaderWriter.write(slice(header)); - _responseHeaderWriter.write(": "_sl); - _responseHeaderWriter.write(slice(value)); - _responseHeaderWriter.write("\r\n"_sl); - } - - void RequestResponse::addHeaders(const map& headers) { - for ( auto& entry : headers ) setHeader(entry.first.c_str(), entry.second.c_str()); - } - - void RequestResponse::setContentLength(uint64_t length) { - sendStatus(); - Assert(_contentLength < 0, "Content-Length has already been set"); - Log("Content-Length: %" PRIu64, length); - _contentLength = (int64_t)length; - constexpr size_t bufSize = 20; - char len[bufSize]; - snprintf(len, bufSize, "%" PRIu64, length); - setHeader("Content-Length", len); - } - - void RequestResponse::sendHeaders() { - if ( _jsonEncoder ) setHeader("Content-Type", "application/json"); - _responseHeaderWriter.write("\r\n"_sl); - if ( _socket->write_n(_responseHeaderWriter.finish()) < 0 ) handleSocketError(); - _endedHeaders = true; - } - -#pragma mark - RESPONSE BODY: - - void RequestResponse::uncacheable() { - setHeader("Cache-Control", "no-cache, no-store, must-revalidate, private, max-age=0"); - setHeader("Pragma", "no-cache"); - setHeader("Expires", "0"); - } - - void RequestResponse::write(slice content) { - Assert(!_finished); - _responseWriter.write(content); + string Request::uri() const { + if ( _queries.empty() ) return _path; + else + return _path + "?" + _queries; } - void RequestResponse::printf(const char* format, ...) { - char* str; - va_list args; - va_start(args, format); - int length = vasprintf(&str, format, args); - if ( length < 0 ) throw bad_alloc(); - va_end(args); - write({str, size_t(length)}); - free(str); + uint64_t Request::uintQuery(const char* param, uint64_t defaultValue) const { + defaultValue = std::min(defaultValue, uint64_t(INT64_MAX)); + return std::max(int64_t(0), intQuery(param, defaultValue)); } - fleece::JSONEncoder& RequestResponse::jsonEncoder() { - if ( !_jsonEncoder ) _jsonEncoder = std::make_unique(); - return *_jsonEncoder; + bool Request::boolQuery(const char* param, bool defaultValue) const { + string val = query(param); + if ( val.empty() ) return defaultValue; + return val != "false" && val != "0"; // same behavior as Obj-C CBL 1.x } - void RequestResponse::finish() { - if ( _finished ) return; - - if ( _jsonEncoder ) { - alloc_slice json = _jsonEncoder->finish(); - write(json); - } - - alloc_slice responseData = _responseWriter.finish(); - if ( _contentLength < 0 ) setContentLength(responseData.size); - else - Assert(_contentLength == responseData.size); - - sendHeaders(); - - Log("Now sending body..."); - if ( _socket->write_n(responseData) < 0 ) handleSocketError(); - _finished = true; + bool Request::keepAlive() const { + auto connection = header("Connection"); + return (_version == Request::HTTP1_1) ? (connection != "close") : (connection == "keep-alive"); } - bool RequestResponse::isValidWebSocketRequest() { - return header("Connection").caseEquivalent("upgrade"_sl) && header("Upgrade").caseEquivalent("websocket"_sl) + bool Request::isValidWebSocketRequest() { + return _method == GET && header("Connection").caseEquivalent("upgrade"_sl) + && header("Upgrade").caseEquivalent("websocket"_sl) && slice_istream(header("Sec-WebSocket-Version")).readDecimal() >= 13 && header("Sec-WebSocket-Key").size >= 10; } - void RequestResponse::sendWebSocketResponse(const string& protocol) { - string nonce(header("Sec-WebSocket-Key")); - setStatus(HTTPStatus::Upgraded, "Upgraded"); - setHeader("Connection", "Upgrade"); - setHeader("Upgrade", "websocket"); - setHeader("Sec-WebSocket-Accept", HTTPLogic::webSocketKeyResponse(nonce).c_str()); - if ( !protocol.empty() ) setHeader("Sec-WebSocket-Protocol", protocol.c_str()); - finish(); - } - - void RequestResponse::onClose(std::function&& callback) { _socket->onClose(std::move(callback)); } - - unique_ptr RequestResponse::extractSocket() { - finish(); - return std::move(_socket); - } - - string RequestResponse::peerAddress() { return _socket->peerAddress(); } - } // namespace litecore::REST + +#endif diff --git a/REST/Request.hh b/REST/Request.hh index 3f8caa0ef..85a13faf1 100644 --- a/REST/Request.hh +++ b/REST/Request.hh @@ -13,25 +13,23 @@ #pragma once #include "Response.hh" #include "HTTPTypes.hh" -#include "Writer.hh" -#include -#include -#include -#include + +#ifdef COUCHBASE_ENTERPRISE namespace litecore::net { - class ResponderSocket; + class TCPSocket; } // namespace litecore::net namespace litecore::REST { - class Server; /** Incoming HTTP request; read-only */ class Request : public Body { public: using Method = net::Method; - explicit Request(fleece::slice httpData); + /// Reads an HTTP request from a socket. + /// If any errors occur, it sets `socketError`. + explicit Request(net::TCPSocket*); bool isValid() const { return _method != Method::None; } @@ -39,108 +37,42 @@ namespace litecore::REST { Method method() const { return _method; } - std::string path() const { return _path; } + std::string const& path() const LIFETIMEBOUND { return _path; } + size_t pathLength() const; std::string path(int i) const; + std::string const& queries() const { return _queries; } + std::string query(const char* param) const; int64_t intQuery(const char* param, int64_t defaultValue = 0) const; + uint64_t uintQuery(const char* param, uint64_t defaultValue = 0) const; bool boolQuery(const char* param, bool defaultValue = false) const; - protected: - friend class Server; - - Request(Method, std::string path, std::string queries, websocket::Headers headers, fleece::alloc_slice body); - Request() = default; - - bool readFromHTTP(fleece::slice httpData); // data must extend at least to CRLF - - Method _method{Method::None}; - std::string _path; - std::string _queries; - }; - - /** Incoming HTTP request (inherited from Request), plus setters for the response. */ - class RequestResponse : public Request { - public: - // Response status: - - void respondWithStatus(HTTPStatus, const char* message = nullptr); - void respondWithError(C4Error); - - void setStatus(HTTPStatus status, const char* message); - - HTTPStatus status() const { return _status; } - - static HTTPStatus errorToStatus(C4Error); - - // Response headers: - - void setHeader(const char* header, const char* value); + std::string uri() const; - void setHeader(const char* header, int64_t value) { setHeader(header, std::to_string(value).c_str()); } + enum HTTPVersion { HTTP1_0, HTTP1_1 }; - void addHeaders(const std::map&); + HTTPVersion httpVersion() const { return _version; } - // Response body: - - void setContentLength(uint64_t length); - void uncacheable(); - - void write(fleece::slice); - - void write(const char* content) { write(fleece::slice(content)); } - - void printf(const char* format, ...) __printflike(2, 3); - - fleece::JSONEncoder& jsonEncoder(); - - void writeStatusJSON(HTTPStatus status, const char* message = nullptr); - void writeErrorJSON(C4Error); - - // Must be called after everything's written: - void finish(); - - // WebSocket stuff: + bool keepAlive() const; bool isValidWebSocketRequest(); - void sendWebSocketResponse(const std::string& protocol); - - void onClose(std::function&& callback); - - std::unique_ptr extractSocket(); - - std::string peerAddress(); + C4Error socketError() const { return _error; } protected: - RequestResponse(Server* server, std::unique_ptr); - void sendStatus(); - void sendHeaders(); - void handleSocketError(); - - private: - friend class Server; - - fleece::Retained _server; - std::unique_ptr _socket; - C4Error _error{}; - - std::vector _requestBody; - - HTTPStatus _status{HTTPStatus::OK}; // Response status code - std::string _statusMessage; // Response custom status message - bool _sentStatus{false}; // Sent the response line yet? + Request(Method, std::string path, std::string queries, websocket::Headers headers, alloc_slice body); + Request() = default; - fleece::Writer _responseHeaderWriter; - bool _endedHeaders{false}; // True after headers are ended - int64_t _contentLength{-1}; // Content-Length, once it's set + bool readFromHTTP(slice httpData); // data must extend at least to CRLF - fleece::Writer _responseWriter; // Output stream for response body - std::unique_ptr _jsonEncoder; // Used for writing JSON to response - fleece::alloc_slice _responseBody; // Finished response body - fleece::slice _unsentBody; // Unsent portion of _responseBody - bool _finished{false}; // Finished configuring the response? + Method _method{Method::None}; + std::string _path; + std::string _queries; + HTTPVersion _version; + C4Error _error{}; }; } // namespace litecore::REST +#endif diff --git a/REST/Response.cc b/REST/Response.cc index 82a50e424..8005a0090 100644 --- a/REST/Response.cc +++ b/REST/Response.cc @@ -28,19 +28,16 @@ namespace litecore::REST { using namespace litecore::net; using namespace litecore::crypto; - bool Body::hasContentType(slice contentType) const { - slice actualType = header("Content-Type"); - return actualType.size >= contentType.size && memcmp(actualType.buf, contentType.buf, contentType.size) == 0 - && (actualType.size == contentType.size || actualType[contentType.size] == ';'); - } - alloc_slice Body::body() const { return _body; } Value Body::bodyAsJSON() const { if ( !_gotBodyFleece ) { - if ( hasContentType("application/json"_sl) ) { - alloc_slice b = body(); - if ( b ) _bodyFleece = Doc::fromJSON(b, nullptr); + if ( header("Content-Type").hasPrefix("application/json") ) { + if ( alloc_slice b = body() ) { + FLError err; + _bodyFleece = Doc::fromJSON(b, &err); + if ( !_bodyFleece ) Warn("HTTP Body has unparseable JSON (%d): %.*s", err, FMTSLICE(b)); + } } _gotBodyFleece = true; } diff --git a/REST/Response.hh b/REST/Response.hh index 89224c292..78bfb28a8 100644 --- a/REST/Response.hh +++ b/REST/Response.hh @@ -13,6 +13,7 @@ #pragma once #include "Headers.hh" #include "HTTPTypes.hh" +#include "StringUtil.hh" #include "fleece/RefCounted.hh" #include "fleece/slice.hh" #include "fleece/Fleece.hh" @@ -36,28 +37,26 @@ namespace litecore::REST { public: using HTTPStatus = net::HTTPStatus; - fleece::slice header(const char* name) const { return _headers[fleece::slice(name)]; } + slice header(const char* name) const LIFETIMEBOUND { return _headers[slice(name)]; } - fleece::slice operator[](const char* name) const { return header(name); } + slice operator[](const char* name) const LIFETIMEBOUND { return header(name); } - bool hasContentType(fleece::slice contentType) const; - fleece::alloc_slice body() const; - fleece::Value bodyAsJSON() const; + alloc_slice body() const; + Value bodyAsJSON() const; protected: Body() = default; - Body(websocket::Headers headers, fleece::alloc_slice body) - : _headers(std::move(headers)), _body(std::move(body)) {} + Body(websocket::Headers headers, alloc_slice body) : _headers(std::move(headers)), _body(std::move(body)) {} void setHeaders(const websocket::Headers& h) { _headers = h; } - void setBody(fleece::alloc_slice body) { _body = std::move(body); } + void setBody(alloc_slice body) { _body = std::move(body); } - websocket::Headers _headers; - fleece::alloc_slice _body; - mutable bool _gotBodyFleece{false}; - mutable fleece::Doc _bodyFleece; + websocket::Headers _headers; + alloc_slice _body; + mutable bool _gotBodyFleece{false}; + mutable Doc _bodyFleece; }; /** An HTTP response from a server, created by specifying a request to send. @@ -74,11 +73,11 @@ namespace litecore::REST { ~Response(); - Response& setHeaders(const fleece::Doc& headers); + Response& setHeaders(const Doc& headers); Response& setHeaders(const websocket::Headers& headers); - Response& setAuthHeader(fleece::slice authHeader); - Response& setBody(fleece::slice body); + Response& setAuthHeader(slice authHeader); + Response& setBody(slice body); Response& setTLSContext(net::TLSContext*); Response& setProxy(const net::ProxySpec&); @@ -89,8 +88,8 @@ namespace litecore::REST { return *this; } - Response& allowOnlyCert(fleece::slice certData); - Response& setRootCerts(fleece::slice certsData); + Response& allowOnlyCert(slice certData); + Response& setRootCerts(slice certsData); #ifdef COUCHBASE_ENTERPRISE Response& allowOnlyCert(C4Cert*); Response& setRootCerts(C4Cert*); @@ -127,13 +126,13 @@ namespace litecore::REST { } private: - double _timeout{0}; - std::unique_ptr _logic; - fleece::Retained _tlsContext; - fleece::alloc_slice _requestBody; - HTTPStatus _status{HTTPStatus::undefined}; - std::string _statusMessage; - C4Error _error{}; + double _timeout{0}; + std::unique_ptr _logic; + Retained _tlsContext; + alloc_slice _requestBody; + HTTPStatus _status{HTTPStatus::undefined}; + std::string _statusMessage; + C4Error _error{}; }; } // namespace litecore::REST diff --git a/REST/Server.cc b/REST/Server.cc index cac4f7922..80eb19270 100644 --- a/REST/Server.cc +++ b/REST/Server.cc @@ -11,7 +11,6 @@ // #include "Server.hh" -#include "Request.hh" #include "TCPSocket.hh" #include "TLSContext.hh" #include "Poller.hh" @@ -23,12 +22,14 @@ #include #include +#ifdef COUCHBASE_ENTERPRISE + // TODO: Remove these pragmas when doc-comments in sockpp are fixed -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdocumentation" -#include "sockpp/tcp_acceptor.h" -#include "sockpp/inet6_address.h" -#pragma clang diagnostic pop +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wdocumentation" +# include "sockpp/tcp_acceptor.h" +# include "sockpp/inet6_address.h" +# pragma clang diagnostic pop namespace litecore::REST { using namespace std; @@ -51,7 +52,7 @@ namespace litecore::REST { error::_throw(error::LiteCoreError::Unimplemented); } - Server::Server() { + Server::Server(Delegate& delegate) : _delegate(delegate) { if ( !ListenerLog ) ListenerLog = c4log_getDomain("Listener", true); } @@ -123,7 +124,6 @@ namespace litecore::REST { Poller::instance().removeListeners(_acceptor->handle()); _acceptor->close(); _acceptor.reset(); - _rules.clear(); } void Server::awaitConnection() { @@ -131,6 +131,7 @@ namespace litecore::REST { if ( !_acceptor ) return; Poller::instance().addListener(_acceptor->handle(), Poller::kReadable, [this] { + // Callback when a socket is accepted: Retained selfRetain = this; acceptConnection(); }); @@ -161,93 +162,48 @@ namespace litecore::REST { } catch ( const std::exception& x ) { c4log(ListenerLog, kC4LogWarning, "Caught C++ exception accepting connection: %s", x.what()); } + // Start another async accept: awaitConnection(); } void Server::handleConnection(sockpp::stream_socket&& sock) { - auto responder = make_unique(_tlsContext); - if ( !responder->acceptSocket(std::move(sock)) || (_tlsContext && !responder->wrapTLS()) ) { - c4log(ListenerLog, kC4LogError, "Error accepting incoming connection: %s", - responder->error().description().c_str()); - return; - } - if ( c4log_willLog(ListenerLog, kC4LogVerbose) ) { - auto cert = responder->peerTLSCertificate(); - if ( cert ) - c4log(ListenerLog, kC4LogVerbose, "Accepted connection from %s with TLS cert %s", - responder->peerAddress().c_str(), cert->subjectPublicKey()->digestString().c_str()); - else - c4log(ListenerLog, kC4LogVerbose, "Accepted connection from %s", responder->peerAddress().c_str()); - } - RequestResponse rq(this, std::move(responder)); - if ( rq.isValid() ) { - dispatchRequest(&rq); - rq.finish(); + auto responder = make_unique(_tlsContext); + bool ok = responder->acceptSocket(std::move(sock)); + string peerAddr = ok ? responder->peerAddress() : "???"; + if ( ok && _tlsContext ) { + c4log(ListenerLog, kC4LogVerbose, "Incoming TLS connection from %s -- starting handshake", + peerAddr.c_str()); + ok = responder->wrapTLS(); } - } - - void Server::setExtraHeaders(const std::map& headers) { - lock_guard lock(_mutex); - _extraHeaders = headers; - } - - void Server::addHandler(Methods methods, const string& patterns, const Handler& handler) { - lock_guard lock(_mutex); - split(patterns, "|", [&](string_view pattern) { - _rules.push_back({methods, string(pattern), regex(pattern.data(), pattern.size()), handler}); - }); - } - - Server::URIRule* Server::findRule(Method method, const string& path) { - //lock_guard lock(_mutex); // called from dispatchResponder which locks - for ( auto& rule : _rules ) { - if ( (rule.methods & method) && regex_match(path.c_str(), rule.regex) ) return &rule; - } - return nullptr; - } - - void Server::dispatchRequest(RequestResponse* rq) { - Method method = rq->method(); - if ( method == Method::GET && rq->header("Connection") == "Upgrade"_sl ) method = Method::UPGRADE; - - c4log(ListenerLog, kC4LogInfo, "%s %s", MethodName(method), rq->path().c_str()); - - if ( _authenticator ) { - if ( !_authenticator(rq->header("Authorization")) ) { - c4log(ListenerLog, kC4LogInfo, "Authentication failed"); - rq->setStatus(HTTPStatus::Unauthorized, "Unauthorized"); - rq->setHeader("WWW-Authenticate", "Basic charset=\"UTF-8\""); - return; + if ( !ok ) { + C4Error error = responder->error(); + string description = error.description(); + if ( error.domain == NetworkDomain ) { + // The default messages call the peer "server" and me "client"; reverse that: + replace(description, "server", "CLIENT"); + replace(description, "client", "server"); + replace(description, "CLIENT", "client"); } + c4log(ListenerLog, kC4LogError, "Error accepting incoming connection from %s: %s", peerAddr.c_str(), + description.c_str()); + return; } - lock_guard lock(_mutex); - - ++_connectionCount; - Retained retainedSelf = this; - rq->onClose([=] { --retainedSelf->_connectionCount; }); - - try { - string pathStr(rq->path()); - auto rule = findRule(method, pathStr); - if ( rule ) { - c4log(ListenerLog, kC4LogInfo, "Matched rule %s for path %s", rule->pattern.c_str(), pathStr.c_str()); - rule->handler(*rq); - } else if ( nullptr == (rule = findRule(Methods::ALL, pathStr)) ) { - c4log(ListenerLog, kC4LogInfo, "No rule matched path %s", pathStr.c_str()); - rq->respondWithStatus(HTTPStatus::NotFound, "Not found"); - } else { - c4log(ListenerLog, kC4LogInfo, "Wrong method for rule %s for path %s", rule->pattern.c_str(), - pathStr.c_str()); - if ( method == Method::UPGRADE ) rq->respondWithStatus(HTTPStatus::Forbidden, "No upgrade available"); - else - rq->respondWithStatus(HTTPStatus::MethodNotAllowed, "Method not allowed"); + bool loggedConnection = false; + if ( c4log_willLog(ListenerLog, kC4LogVerbose) ) { + if ( auto cert = responder->peerTLSCertificate() ) { + c4log(ListenerLog, kC4LogVerbose, "Accepted connection from %s with TLS cert %s", peerAddr.c_str(), + cert->subjectPublicKey()->digestString().c_str()); + loggedConnection = true; } - } catch ( const std::exception& x ) { - c4log(ListenerLog, kC4LogWarning, "HTTP handler caught C++ exception: %s", x.what()); - rq->respondWithStatus(HTTPStatus::ServerError, "Internal exception"); } + if ( !loggedConnection ) c4log(ListenerLog, kC4LogInfo, "Accepted connection from %s", peerAddr.c_str()); + + //TODO: Increment/decrement _connectionCount + _delegate.handleConnection(std::move(responder)); } } // namespace litecore::REST + +#endif diff --git a/REST/Server.hh b/REST/Server.hh index 7f087bfd2..8e7f977cd 100644 --- a/REST/Server.hh +++ b/REST/Server.hh @@ -11,18 +11,19 @@ // #pragma once +#include "c4Compat.h" #include "fleece/RefCounted.hh" #include "fleece/InstanceCounted.hh" -#include "Request.hh" -#include "StringUtil.hh" +#include "fleece/slice.hh" #include #include #include -#include #include -#include #include -#include + +#ifdef COUCHBASE_ENTERPRISE + +C4_ASSUME_NONNULL_BEGIN namespace sockpp { class acceptor; @@ -35,21 +36,28 @@ namespace litecore::crypto { } namespace litecore::net { + class ResponderSocket; class TLSContext; -} +} // namespace litecore::net namespace litecore::REST { using namespace fleece; - /** HTTP server with configurable URI handlers. */ + /** A basic TCP server. Incoming TCP connections are passed on to its delegate. */ class Server final - : public fleece::RefCounted - , public fleece::InstanceCountedIn { + : public RefCounted + , public InstanceCountedIn { public: - Server(); + class Delegate { + public: + virtual ~Delegate() = default; + virtual void handleConnection(std::unique_ptr) = 0; + }; + + explicit Server(Delegate&); - void start(uint16_t port, slice networkInterface = nullslice, net::TLSContext* = nullptr); + void start(uint16_t port, slice networkInterface = {}, net::TLSContext* C4NULLABLE = nullptr); virtual void stop(); @@ -61,51 +69,24 @@ namespace litecore::REST { "norbert.local". */ std::vector addresses() const; - /** A function that authenticates an HTTP request, given the "Authorization" header. */ - using Authenticator = std::function; - - void setAuthenticator(Authenticator auth) { _authenticator = std::move(auth); } - - /** Extra HTTP headers to add to every response. */ - void setExtraHeaders(const std::map& headers); - - /** A function that handles a request. */ - using Handler = std::function; - - /** Registers a handler function for a URI pattern. - Patterns use glob syntax: - Multiple patterns can be joined with a "|". - Patterns are tested in the order the handlers are added, and the first match is used.*/ - void addHandler(net::Methods, const std::string& pattern, const Handler&); - int connectionCount() { return _connectionCount; } - protected: - struct URIRule { - net::Methods methods; - std::string pattern; - std::regex regex; - Handler handler; - }; - - URIRule* findRule(net::Method method, const std::string& path); - ~Server() override; - - void dispatchRequest(RequestResponse*); - private: + ~Server() override; void awaitConnection(); void acceptConnection(); void handleConnection(sockpp::stream_socket&&); - fleece::Retained _identity; - fleece::Retained _tlsContext; - std::unique_ptr _acceptor; - std::mutex _mutex; - std::vector _rules; - std::map _extraHeaders; - std::atomic _connectionCount{0}; - Authenticator _authenticator; + std::mutex _mutex; + Delegate& _delegate; + Retained _identity; + Retained _tlsContext; + std::unique_ptr _acceptor; + std::atomic _connectionCount{0}; }; } // namespace litecore::REST + +C4_ASSUME_NONNULL_END + +#endif diff --git a/REST/SyncListener.hh b/REST/SyncListener.hh new file mode 100644 index 000000000..98581ca9c --- /dev/null +++ b/REST/SyncListener.hh @@ -0,0 +1,39 @@ +// +// SyncListener.hh +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#pragma once +#include "HTTPListener.hh" +#include +#include + +#ifdef COUCHBASE_ENTERPRISE + +namespace litecore::REST { + + class SyncListener : public HTTPListener { + public: + static constexpr int kAPIVersion = 3; + + explicit SyncListener(const C4ListenerConfig&); + ~SyncListener(); + + protected: + HTTPStatus handleRequest(Request& rq, websocket::Headers& headers, + std::unique_ptr& socket) override; + + private: + class SyncTask; + }; + +} // namespace litecore::REST + +#endif diff --git a/REST/c4Listener+RESTFactory.cc b/REST/c4Listener+RESTFactory.cc deleted file mode 100644 index 2854f3391..000000000 --- a/REST/c4Listener+RESTFactory.cc +++ /dev/null @@ -1,30 +0,0 @@ -// -// c4Listener+Factory.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "c4ListenerInternal.hh" -#include "RESTListener.hh" - -#ifndef COUCHBASE_ENTERPRISE - -namespace litecore { namespace REST { - - const C4ListenerAPIs kListenerAPIs = kC4RESTAPI; - - Retained NewListener(const C4ListenerConfig* config) { - if ( config->apis == kC4RESTAPI ) return new RESTListener(*config); - else - return nullptr; - } - -}} // namespace litecore::REST - -#endif diff --git a/REST/c4Listener.cc b/REST/c4Listener.cc index 785aecf86..58e847325 100644 --- a/REST/c4Listener.cc +++ b/REST/c4Listener.cc @@ -15,9 +15,13 @@ #include "c4ExceptionUtils.hh" #include "c4Database.hh" #include "c4Collection.hh" -#include "Listener.hh" -#include "RESTListener.hh" +#include "Address.hh" +#include "HTTPListener.hh" +#include "SyncListener.hh" #include +#include + +#ifdef COUCHBASE_ENTERPRISE using namespace std; using namespace fleece; @@ -28,14 +32,13 @@ namespace litecore::REST { C4LogDomain ListenerLog; } // namespace litecore::REST -C4ListenerAPIs C4Listener::availableAPIs() { return kListenerAPIs; } - -string C4Listener::URLNameFromPath(slice pathSlice) { return Listener::databaseNameFromPath(FilePath(pathSlice, "")); } +string C4Listener::URLNameFromPath(slice pathSlice) { + return DatabaseRegistry::databaseNameFromPath(FilePath(pathSlice, "")); +} namespace { std::stringstream& operator<<(std::stringstream& ss, const C4ListenerConfig& config) { ss << "{" - << "apis: " << (config.apis == kC4RESTAPI ? "REST" : "Sync") << ", " << "networkInterface: " << (!config.networkInterface ? "NULL" : config.networkInterface.size == 0 ? "\"\"" @@ -59,57 +62,43 @@ namespace { ss << "}, " << "httpAuthCallback: " << (config.httpAuthCallback == nullptr ? "NULL" : "***") << ", " << "callbackContext: " << (config.callbackContext == nullptr ? "NULL" : "***") << ", " - << "directory: " - << (!config.directory ? "NULL" - : config.directory.size == 0 ? "\"\"" - : std::string(config.directory)) - << ", "; - if ( config.apis == kC4RESTAPI ) { - ss << "allowCreateDBs: " << config.allowCreateDBs << ", allowDeleteDBs: " << config.allowDeleteDBs - << ", allowCreateCollections: " << config.allowCreateCollections - << ", allowDeleteCollections: " << config.allowDeleteCollections; - } else { - ss << "allowPush: " << config.allowPush << ", " - << "allowPull: " << config.allowPull << ", " - << "enableDeltaSync: " << config.enableDeltaSync; - } + << "allowPush: " << config.allowPush << ", " + << "allowPull: " << config.allowPull << ", " + << "enableDeltaSync: " << config.enableDeltaSync; ss << "}"; return ss; } } // namespace -C4Listener::C4Listener(C4ListenerConfig config) - : _httpAuthCallback(config.httpAuthCallback), _callbackContext(config.callbackContext) { - // Replace the callback, if any, with one to myself. This allows me to pass the correct - // C4Listener* to the client's callback. - if ( config.httpAuthCallback ) { - config.callbackContext = this; - config.httpAuthCallback = [](C4Listener*, C4Slice authHeader, void* context) { - auto listener = (C4Listener*)context; - return listener->_httpAuthCallback(listener, authHeader, listener->_callbackContext); - }; - } - - _impl = dynamic_cast(NewListener(&config).get()); - if ( !_impl ) { - C4Error::raise(LiteCoreDomain, kC4ErrorUnsupported, "Unsupported listener API"); - } else { - std::stringstream ss; - ss << config; - c4log(ListenerLog, kC4LogInfo, "Listener config: %s", ss.str().c_str()); - } +C4Listener::C4Listener(C4ListenerConfig const& config, Retained impl) : _impl(std::move(impl)) { + _impl->setDelegate(this); + std::stringstream ss; + ss << config; + c4log(ListenerLog, kC4LogInfo, "Listener config: %s", ss.str().c_str()); } -C4Listener::C4Listener(C4Listener&&) = default; +C4Listener::C4Listener(C4ListenerConfig const& config) : C4Listener(config, make_retained(config)) {} + +C4Listener::C4Listener(C4Listener&&) noexcept = default; -C4Listener::~C4Listener() { - if ( _impl ) _impl->stop(); +C4Listener::~C4Listener() noexcept { stop(); } + +C4Error C4Listener::stop() noexcept { + C4Error result{}; + if ( _impl ) { + try { + _impl->stop(); + _impl = nullptr; + } + catchError(&result); + } + return result; } -bool C4Listener::shareDB(slice name, C4Database* db) { +bool C4Listener::shareDB(slice name, C4Database* db, C4ListenerDatabaseConfig const* dbConfig) { optional nameStr; if ( name.buf ) nameStr = name; - return _impl->registerDatabase(db, nameStr); + return _impl->registerDatabase(db, nameStr, dbConfig); } bool C4Listener::unshareDB(C4Database* db) { return _impl->unregisterDatabase(db); } @@ -118,9 +107,6 @@ bool C4Listener::shareCollection(slice name, C4Collection* coll) { if ( _usuallyFalse(!coll || !coll->isValid()) ) { C4Error::raise(LiteCoreDomain, kC4ErrorNotOpen, "Invalid collection: either deleted, or db closed"); } - - optional nameStr; - if ( name.buf ) nameStr = name; return _impl->registerCollection((string)name, coll->getSpec()); } @@ -128,10 +114,9 @@ bool C4Listener::unshareCollection(slice name, C4Collection* coll) { return _impl->unregisterCollection((string)name, coll->getSpec()); } -std::vector C4Listener::URLs(C4Database* C4NULLABLE db, C4ListenerAPIs api) const { - AssertParam(api == kC4RESTAPI || api == kC4SyncAPI, "The provided API must be one of the following: REST, Sync."); +std::vector C4Listener::URLs(C4Database* C4NULLABLE db) const { vector urls; - for ( net::Address& address : _impl->addresses(db, api) ) urls.emplace_back(string(address.url())); + for ( net::Address& address : _impl->addresses(db, true) ) urls.emplace_back(string(address.url())); return urls; } @@ -143,3 +128,5 @@ std::pair C4Listener::connectionStatus() const { auto activeConnectionCount = active; return {connectionCount, activeConnectionCount}; } + +#endif // COUCHBASE_ENTERPRISE diff --git a/REST/c4ListenerInternal.hh b/REST/c4ListenerInternal.hh index aa5a8c832..f6555a828 100644 --- a/REST/c4ListenerInternal.hh +++ b/REST/c4ListenerInternal.hh @@ -16,14 +16,8 @@ #include "c4ListenerTypes.h" namespace litecore::REST { - class Listener; - + class HTTPListener; extern C4LogDomain ListenerLog; - - extern const C4ListenerAPIs kListenerAPIs; - fleece::Retained NewListener(const C4ListenerConfig* config); - - } // namespace litecore::REST diff --git a/REST/REST_CAPI.cc b/REST/c4Listener_CAPI.cc similarity index 57% rename from REST/REST_CAPI.cc rename to REST/c4Listener_CAPI.cc index 23b748ad2..961fb5e50 100644 --- a/REST/REST_CAPI.cc +++ b/REST/c4Listener_CAPI.cc @@ -1,5 +1,5 @@ // -// REST_CAPI.cc +// c4Listener_CAPI.cc // // Copyright 2021-Present Couchbase, Inc. // @@ -15,22 +15,22 @@ #include "c4ExceptionUtils.hh" #include "fleece/Mutable.hh" +#ifdef COUCHBASE_ENTERPRISE + using namespace std; using namespace fleece; using namespace litecore; -CBL_CORE_API C4ListenerAPIs c4listener_availableAPIs(void) noexcept { return C4Listener::availableAPIs(); } - -CBL_CORE_API C4Listener* c4listener_start(const C4ListenerConfig* config, C4Error* outError) noexcept { +C4Listener* c4listener_start(const C4ListenerConfig* config, C4Error* outError) noexcept { try { return new C4Listener(*config); } catchError(outError) return nullptr; } -CBL_CORE_API void c4listener_free(C4Listener* listener) noexcept { delete listener; } +void c4listener_free(C4Listener* listener) noexcept { delete listener; } -CBL_CORE_API C4StringResult c4db_URINameFromPath(C4String pathSlice) noexcept { +C4StringResult c4db_URINameFromPath(C4String pathSlice) noexcept { try { if ( string name = C4Listener::URLNameFromPath(pathSlice); name.empty() ) return {}; else @@ -39,7 +39,7 @@ CBL_CORE_API C4StringResult c4db_URINameFromPath(C4String pathSlice) noexcept { catchAndWarn() return {}; } -CBL_CORE_API bool c4listener_shareDB(C4Listener* listener, C4String name, C4Database* db, C4Error* outError) noexcept { +bool c4listener_shareDB(C4Listener* listener, C4String name, C4Database* db, C4Error* outError) noexcept { try { return listener->shareDB(name, db); } @@ -47,7 +47,7 @@ CBL_CORE_API bool c4listener_shareDB(C4Listener* listener, C4String name, C4Data return false; } -CBL_CORE_API bool c4listener_unshareDB(C4Listener* listener, C4Database* db, C4Error* outError) noexcept { +bool c4listener_unshareDB(C4Listener* listener, C4Database* db, C4Error* outError) noexcept { try { if ( listener->unshareDB(db) ) return true; c4error_return(LiteCoreDomain, kC4ErrorNotOpen, "Database not shared"_sl, outError); @@ -56,8 +56,8 @@ CBL_CORE_API bool c4listener_unshareDB(C4Listener* listener, C4Database* db, C4E return false; } -CBL_CORE_API bool c4listener_shareCollection(C4Listener* listener, C4String name, C4Collection* collection, - C4Error* outError) noexcept { +bool c4listener_shareCollection(C4Listener* listener, C4String name, C4Collection* collection, + C4Error* outError) noexcept { try { return listener->shareCollection(name, collection); } @@ -65,8 +65,8 @@ CBL_CORE_API bool c4listener_shareCollection(C4Listener* listener, C4String name return false; } -CBL_CORE_API bool c4listener_unshareCollection(C4Listener* listener, C4String name, C4Collection* collection, - C4Error* outError) noexcept { +bool c4listener_unshareCollection(C4Listener* listener, C4String name, C4Collection* collection, + C4Error* outError) noexcept { try { if ( listener->unshareCollection(name, collection) ) return true; c4error_return(LiteCoreDomain, kC4ErrorNotOpen, "Collection not shared"_sl, outError); @@ -75,27 +75,28 @@ CBL_CORE_API bool c4listener_unshareCollection(C4Listener* listener, C4String na return false; } -CBL_CORE_API uint16_t c4listener_getPort(const C4Listener* listener) noexcept { +uint16_t c4listener_getPort(const C4Listener* listener) noexcept { try { return listener->port(); } catchAndWarn() return 0; } -CBL_CORE_API FLMutableArray c4listener_getURLs(const C4Listener* listener, C4Database* db, C4ListenerAPIs api, - C4Error* err) noexcept { +FLMutableArray c4listener_getURLs(const C4Listener* listener, C4Database* db, C4Error* err) noexcept { try { auto urls = fleece::MutableArray::newArray(); - for ( string& url : listener->URLs(db, api) ) urls.append(url); + for ( string& url : listener->URLs(db) ) urls.append(url); return (FLMutableArray)FLValue_Retain(urls); } catchError(err); return nullptr; } -CBL_CORE_API void c4listener_getConnectionStatus(const C4Listener* listener, unsigned* connectionCount, - unsigned* activeConnectionCount) noexcept { +void c4listener_getConnectionStatus(const C4Listener* listener, unsigned* connectionCount, + unsigned* activeConnectionCount) noexcept { auto [conns, active] = listener->connectionStatus(); if ( connectionCount ) *connectionCount = conns; if ( activeConnectionCount ) *activeConnectionCount = active; } + +#endif // COUCHBASE_ENTERPRISE diff --git a/REST/netUtils.cc b/REST/netUtils.cc index 1c636c67a..3f74a92f2 100644 --- a/REST/netUtils.cc +++ b/REST/netUtils.cc @@ -21,9 +21,8 @@ #include "NumConversion.hh" #include #include -// 'digittoint' function +#include "StringUtil.hh" #ifndef __APPLE__ -# include "StringUtil.hh" # include "PlatformIO.hh" #endif @@ -75,34 +74,41 @@ namespace litecore::REST { return result; } - string getURLQueryParam(slice queries, const char* name, char delimiter, size_t occurrence) { - auto data = (const char*)queries.buf; - size_t data_len = queries.size; - const char* end = data + data_len; - - string dst; - if ( data == nullptr || name == nullptr || data_len == 0 ) { return dst; } - size_t name_len = strlen(name); - - // data is "var1=val1&var2=val2...". Find variable first - for ( const char* p = data; p + name_len < end; p++ ) { - if ( (p == data || p[-1] == delimiter) && p[name_len] == '=' && !strncasecmp(name, p, name_len) - && 0 == occurrence-- ) { - // Point p to variable value - p += name_len + 1; - - // Point s to the end of the value - const char* s = (const char*)memchr(p, delimiter, (size_t)(end - p)); - if ( s == nullptr ) { s = end; } - assert(s >= p); + bool iterateURLQueries(string_view queries, char delimiter, function_ref callback) { + bool stop = false; + split(queries, string_view{&delimiter, 1}, [&](string_view query) { + if ( !stop ) { + string value; + if ( auto eq = query.find('='); eq != string::npos ) { + value = query.substr(eq + 1); + query = query.substr(0, eq); + } + stop = callback(query, value); + } + }); + return stop; + } - // Decode variable into destination buffer - return URLDecode(slice(p, s - p), true); + string getURLQueryParam(slice queries, string_view name, char delimiter, size_t occurrence) { + string value; + iterateURLQueries(queries, delimiter, [&](string_view k, string_view v) -> bool { + if ( name.size() == k.size() && 0 == strncasecmp(name.data(), k.data(), k.size()) && 0 == occurrence-- ) { + value = URLDecode(v); + return true; // stop iteration } - } - return dst; + return false; + }); + return value; } + std::string timestamp() { + char dateStr[100]; + time_t t = time(nullptr); + strftime(dateStr, sizeof(dateStr), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t)); // faster than date::format() + return string(dateStr); + } + + } // namespace litecore::REST /* Copyright (c) 2013-2017 the Civetweb developers diff --git a/REST/netUtils.hh b/REST/netUtils.hh index edf9b906b..0831cdc1e 100644 --- a/REST/netUtils.hh +++ b/REST/netUtils.hh @@ -11,6 +11,7 @@ // #pragma once +#include "fleece/function_ref.hh" #include "fleece/slice.hh" namespace litecore::REST { @@ -19,6 +20,21 @@ namespace litecore::REST { std::string URLEncode(fleece::slice str); - std::string getURLQueryParam(fleece::slice queries, const char* name, char delimiter = '&', size_t occurrence = 0); + /// Gets a URL query parameter by name. The returned value is URL-decoded. + std::string getURLQueryParam(fleece::slice queries, std::string_view name, char delimiter = '&', + size_t occurrence = 0); + + /// Calls the callback with the name and (raw) value of each query parameter. + /// @param queries The query string (without the initial '?') + /// @param delimiter The character separating queries, usually '&' + /// @param callback A function that will be passed the name and (raw) value. + /// You need to call \ref URLDecode to decode the value. + /// Return `false` to continue the iteration, `true` to stop. + /// @returns true if the callback stopped the iteration, else false. + bool iterateURLQueries(std::string_view queries, char delimiter, + fleece::function_ref callback); + + /// Returns the current date/time formatted per HTTP, for use in Date: header. + std::string timestamp(); } // namespace litecore::REST diff --git a/REST/tests/ListenerHarness.hh b/REST/tests/ListenerHarness.hh index 7fa7164d7..75f2f56c4 100644 --- a/REST/tests/ListenerHarness.hh +++ b/REST/tests/ListenerHarness.hh @@ -16,6 +16,8 @@ #include "CertHelper.hh" #include "c4Listener.h" +#ifdef COUCHBASE_ENTERPRISE + using namespace fleece; class ListenerHarness { @@ -29,9 +31,6 @@ class ListenerHarness { [[nodiscard]] C4Listener* listener() const { return _listener; } - -#ifdef COUCHBASE_ENTERPRISE - C4Cert* useServerIdentity(const Identity& id) { alloc_slice digest = c4keypair_publicKeyDigest(id.key); C4Log("Using %s server TLS cert %.*s for this test", @@ -86,31 +85,24 @@ class ListenerHarness { _tlsConfig.tlsCallbackContext = context; } -#endif // COUCHBASE_ENTERPRISE - void share(C4Database* dbToShare, slice name) { if ( _listener ) return; - auto missing = config.apis & ~c4listener_availableAPIs(); - if ( missing ) FAIL("Listener API " << missing << " is unavailable in this build"); - C4Error err; - _listener = c4listener_start(&config, &err); + _listener = c4listener_start(&config, ERROR_INFO()); REQUIRE(_listener); - REQUIRE(c4listener_shareDB(_listener, name, dbToShare, &err)); + REQUIRE(c4listener_shareDB(_listener, name, dbToShare, WITH_ERROR())); } void stop() { _listener = nullptr; } public: C4ListenerConfig config; -#ifdef COUCHBASE_ENTERPRISE - Identity serverIdentity, clientIdentity; -#endif + Identity serverIdentity, clientIdentity; private: c4::ref _listener; C4TLSConfig _tlsConfig = {}; -#ifdef COUCHBASE_ENTERPRISE - CertHelper _certHelper; -#endif + CertHelper _certHelper; }; + +#endif // COUCHBASE_ENTERPRISE diff --git a/REST/tests/RESTListenerTest.cc b/REST/tests/RESTListenerTest.cc deleted file mode 100644 index bc277eacb..000000000 --- a/REST/tests/RESTListenerTest.cc +++ /dev/null @@ -1,886 +0,0 @@ -// -// RESTListenerTest.cc -// -// Copyright 2017-Present Couchbase, Inc. -// -// Use of this software is governed by the Business Source License included -// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -// in that file, in accordance with the Business Source License, use of this -// software will be governed by the Apache License, Version 2.0, included in -// the file licenses/APL2.txt. -// - -#include "c4Test.hh" -#include "Error.hh" -#include "c4Collection.h" -#include "c4Database.h" -#include "c4Replicator.h" -#include "ListenerHarness.hh" -#include "FilePath.hh" -#include "Response.hh" -#include "NetworkInterfaces.hh" -#include "fleece/Mutable.hh" -#include "ReplicatorAPITest.hh" -#include "WebSocketInterface.hh" -#include -#include - -using namespace litecore; -using namespace litecore::net; -using namespace litecore::REST; -using namespace std; - - -#if !defined(_WIN32) || defined(_WIN64) -// These tests often hang in 32-bit Windows but let's cheekily ignore it since -// this not part of Couchbase Lite directly anyway, but used in the cblite CLI -// which is 64-bit only on Windows. - -static string to_str(FLSlice s) { return {(char*)s.buf, s.size}; } - -static string to_str(Value v) { return to_str(v.asString()); } - -# define TEST_PORT 0 - -class C4RESTTest - : public C4Test - , public ListenerHarness { - public: - C4RESTTest() : C4Test(0), ListenerHarness({TEST_PORT, nullslice, kC4RESTAPI}) { - std::call_once(ReplicatorAPITest::once, [&]() { - // Register the BuiltInWebSocket class as the C4Replicator's WebSocketImpl. - C4RegisterBuiltInWebSocket(); - }); - } - - void setUpDirectory() { - litecore::FilePath tempDir(TempDir() + "rest/"); - tempDir.delRecursive(); - tempDir.mkdir(); - directory = alloc_slice(tempDir.path().c_str()); - config.directory = directory; - config.allowCreateDBs = true; - } - - -# ifdef COUCHBASE_ENTERPRISE - void setupCertAuth() { - auto callback = [](C4Listener* listener, C4Slice clientCertData, void* context) -> bool { - auto self = (C4RESTTest*)context; - self->receivedCertAuth = clientCertData; - return self->allowClientCert; - }; - setCertAuthCallback(callback, this); - } -# endif - - - void setupHTTPAuth() { - config.callbackContext = this; - config.httpAuthCallback = [](C4Listener* listener, C4Slice authHeader, void* context) { - auto self = (C4RESTTest*)context; - self->receivedHTTPAuthFromListener = listener; - self->receivedHTTPAuthHeader = authHeader; - return self->allowHTTPConnection; - }; - } - - void forEachURL(C4Database* db, C4ListenerAPIs api, function_ref callback) { - MutableArray urls(c4listener_getURLs(listener(), db, api, nullptr)); - FLMutableArray_Release(urls); - REQUIRE(urls); - for ( Array::iterator i(urls); i; ++i ) callback(i->asString()); - } - - unique_ptr request(const string& method, const string& uri, const map& headersMap, - slice body, HTTPStatus expectedStatus) { - Encoder enc; - enc.beginDict(); - for ( auto& h : headersMap ) { - enc.writeKey(h.first); - enc.writeString(h.second); - } - enc.endDict(); - auto headers = enc.finishDoc(); - - share(db, "db"_sl); - - C4Log("---- %s %s", method.c_str(), uri.c_str()); - string scheme = config.tlsConfig ? "https" : "http"; - auto port = c4listener_getPort(listener()); - unique_ptr r; - unsigned totalAttempt = 5; - double timeout = 5; - for ( unsigned attemptCount = 0; attemptCount < totalAttempt; ++attemptCount ) { - r.reset(new Response(scheme, method, requestHostname, port, uri)); - r->setHeaders(headers).setBody(body); - if ( pinnedCert ) r->allowOnlyCert(pinnedCert); -# ifdef COUCHBASE_ENTERPRISE - if ( rootCerts ) r->setRootCerts(rootCerts); - if ( clientIdentity.cert ) r->setIdentity(clientIdentity.cert, clientIdentity.key); -# endif - r->setTimeout(timeout); - if ( !r->run() ) { - if ( r->error().code != websocket::kNetErrTimeout || attemptCount + 1 == totalAttempt ) { - if ( attemptCount > 0 ) - C4LogToAt(kC4DefaultLog, kC4LogWarning, "Error: %s after %d timeouts", - c4error_descriptionStr(r->error()), attemptCount); - else - C4LogToAt(kC4DefaultLog, kC4LogWarning, "Error: %s", c4error_descriptionStr(r->error())); - } - } else { - if ( attemptCount > 0 ) - C4LogToAt(kC4DefaultLog, kC4LogInfo, "Request has been timed out %d times", attemptCount); - break; - } - } - C4Log("Status: %d %s", static_cast(r->status()), r->statusMessage().c_str()); - string responseBody = r->body().asString(); - C4Log("Body: %s", responseBody.c_str()); - INFO("Error: " << c4error_descriptionStr(r->error())); - REQUIRE(r->status() == expectedStatus); - return r; - } - - unique_ptr request(string method, string uri, HTTPStatus expectedStatus) { - return request(std::move(method), std::move(uri), {}, nullslice, expectedStatus); - } - - bool wait(const std::shared_ptr& response, C4ReplicatorActivityLevel activityLevel, - unsigned timeoutSeconds) { - Dict b = response->bodyAsJSON().asDict(); - unsigned sid = (unsigned)b.get("session_id").asUnsigned(); - const unsigned pollIntervalSeconds = 1; - unsigned nextTime = 0; - while ( timeoutSeconds == 0 || nextTime < timeoutSeconds ) { - nextTime += pollIntervalSeconds; - std::this_thread::sleep_for(chrono::seconds(pollIntervalSeconds)); - auto r = request("GET", "/_active_tasks", {{"Content-Type", "application/json"}}, "", HTTPStatus::OK); - Array body = r->bodyAsJSON().asArray(); - Dict sess; - for ( Array::iterator iter(body); iter; ++iter ) { - Dict dict = iter.value().asDict(); - unsigned s = (unsigned)dict.get("session_id").asUnsigned(); - if ( s == sid ) { - sess = dict; - break; - } - } - if ( !sess ) { break; } - static slice const kStatusName[] = {"Stopped"_sl, "Offline"_sl, "Connecting"_sl, "Idle"_sl, "Active"_sl}; - if ( sess.get("status").asString() == kStatusName[activityLevel] ) { return true; } - } - return false; - } - - void testRootLevel() { - auto r = request("GET", "/", HTTPStatus::OK); - auto body = r->bodyAsJSON().asDict(); - REQUIRE(body); - CHECK(to_str(body["couchdb"]) == "Welcome"); - } - - alloc_slice directory; - string requestHostname{"localhost"}; - - C4Listener* receivedHTTPAuthFromListener = nullptr; - optional receivedHTTPAuthHeader; - bool allowHTTPConnection = true; - - alloc_slice pinnedCert; -# ifdef COUCHBASE_ENTERPRISE - c4::ref rootCerts; - - C4Listener* receivedCertAuthFromListener = nullptr; - optional receivedCertAuth; - bool allowClientCert = true; -# endif -}; - -# pragma mark - ROOT LEVEL: - -TEST_CASE_METHOD(C4RESTTest, "Network interfaces", "[Listener][C]") { - vector interfaces; - for ( auto& i : Interface::all() ) interfaces.push_back(i.name); - vector addresses; - for ( auto& addr : Interface::allAddresses() ) addresses.push_back(string(addr)); - vector primaryAddresses; - for ( auto& addr : Interface::primaryAddresses() ) primaryAddresses.push_back(string(addr)); - auto hostname = GetMyHostName(); - C4Log("Interface names = {%s}", join(interfaces, ", ").c_str()); - C4Log("IP addresses = {%s}", join(addresses, ", ").c_str()); - C4Log("Primary addrs = {%s}", join(primaryAddresses, ", ").c_str()); - C4Log("Hostname = %s", (hostname ? hostname->c_str() : "(unknown)")); - CHECK(!interfaces.empty()); - CHECK(!primaryAddresses.empty()); - CHECK(!addresses.empty()); -} - -TEST_CASE_METHOD(C4RESTTest, "Listener URLs", "[Listener][C]") { - share(db, "db"_sl); - auto configPortStr = to_string(c4listener_getPort(listener())); - string expectedSuffix = string(":") + configPortStr + "/"; - forEachURL(nullptr, kC4RESTAPI, [&expectedSuffix](string_view url) { - C4Log("Listener URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, "http://")); - CHECK(hasSuffix(url, expectedSuffix)); - }); - forEachURL(db, kC4RESTAPI, [&expectedSuffix](string_view url) { - C4Log("Database URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, "http://")); - CHECK(hasSuffix(url, expectedSuffix + "db")); - }); - - { - ExpectingExceptions x; - C4Error err; - FLMutableArray invalid = c4listener_getURLs(listener(), db, kC4SyncAPI, &err); - CHECK(!invalid); - CHECK(err.domain == LiteCoreDomain); - CHECK(err.code == kC4ErrorInvalidParameter); - } -} - -TEST_CASE_METHOD(C4RESTTest, "Listen on interface", "[Listener][C]") { - optional intf; - string intfAddress; - SECTION("All interfaces") { - C4Log("Here are all the IP interfaces and their addresses:"); - for ( auto& i : Interface::all() ) { - C4Log(" - %s (%.02x, routable=%d) :", i.name.c_str(), i.flags, i.isRoutable()); - for ( auto& addr : i.addresses ) C4Log(" - %s", string(addr).c_str()); - } - } - SECTION("Specific interface") { - intf = Interface::all()[0]; - intfAddress = string(intf->addresses[0]); - SECTION("Use interface name") { - C4Log("Will listen on interface %s", intf->name.c_str()); - config.networkInterface = slice(intf->name); - } - SECTION("Use interface address") { - C4Log("Will listen on address %s", intfAddress.c_str()); - config.networkInterface = slice(intfAddress); - } - } - - share(db, "db"_sl); - - // Check that the listener's reported URLs contain the interface address: - forEachURL(db, kC4RESTAPI, [&](string_view url) { - C4Log("Checking URL <%.*s>", SPLAT(slice(url))); - C4Address address; - C4String dbName; - INFO("URL is <" << url << ">"); - CHECK(c4address_fromURL(slice(url), &address, &dbName)); - CHECK(address.port == c4listener_getPort(listener())); - CHECK(dbName == "db"_sl); - - if ( intf ) { - requestHostname = string(slice(address.hostname)); - bool foundAddrInInterface = false; - for ( auto& addr : intf->addresses ) { - if ( string(addr) == requestHostname ) { - foundAddrInInterface = true; - break; - } - } - CHECK(foundAddrInInterface); - } - - testRootLevel(); - }); -} - -TEST_CASE_METHOD(C4RESTTest, "Listener Auto-Select Port", "[Listener][C]") { - share(db, "db"_sl); - const auto port = c4listener_getPort(listener()); - C4Log("System selected port %u", port); - CHECK(port != 0); -} - -TEST_CASE_METHOD(C4RESTTest, "No Listeners on Same Port", "[Listener][C]") { - share(db, "db"_sl); - config.port = c4listener_getPort(listener()); - C4Error err; - - ExpectingExceptions x; - auto listener2 = c4listener_start(&config, &err); - CHECK(!listener2); - CHECK(err.domain == POSIXDomain); - CHECK(err.code == EADDRINUSE); -} - -TEST_CASE_METHOD(C4RESTTest, "REST root level", "[REST][Listener][C]") { testRootLevel(); } - -TEST_CASE_METHOD(C4RESTTest, "REST _all_databases", "[REST][Listener][C]") { - auto r = request("GET", "/_all_dbs", HTTPStatus::OK); - auto body = r->bodyAsJSON().asArray(); - REQUIRE(body.count() == 1); - CHECK(to_str(body[0]) == "db"); -} - -TEST_CASE_METHOD(C4RESTTest, "REST unknown special top-level", "[REST][Listener][C]") { - request("GET", "/_foo", HTTPStatus::NotFound); - request("GET", "/_", HTTPStatus::NotFound); -} - -# pragma mark - DATABASE: - -TEST_CASE_METHOD(C4RESTTest, "REST GET database", "[REST][Listener][C]") { - unique_ptr r; - SECTION("No slash") { r = request("GET", "/db", HTTPStatus::OK); } - SECTION("URL-encoded") { r = request("GET", "/%64%62", HTTPStatus::OK); } - SECTION("With slash") { r = request("GET", "/db/", HTTPStatus::OK); } - auto body = r->bodyAsJSON().asDict(); - REQUIRE(body); - CHECK(to_str(body["db_name"]) == "db"); - CHECK(to_str(body["collection_name"]) == "_default"); - CHECK(to_str(body["scope_name"]) == "_default"); - CHECK(body["db_uuid"].type() == kFLString); - CHECK(body["db_uuid"].asString().size >= 32); - CHECK(body["doc_count"].type() == kFLNumber); - CHECK(body["doc_count"].asInt() == 0); - CHECK(body["update_seq"].type() == kFLNumber); - CHECK(body["update_seq"].asInt() == 0); -} - -TEST_CASE_METHOD(C4RESTTest, "REST DELETE database", "[REST][Listener][C]") { - unique_ptr r; - SECTION("Disallowed") { r = request("DELETE", "/db", HTTPStatus::Forbidden); } - SECTION("Allowed") { - config.allowDeleteDBs = true; - r = request("DELETE", "/db", HTTPStatus::OK); - r = request("GET", "/db", HTTPStatus::NotFound); - REQUIRE(!FilePath(string(databasePath())).exists()); - // This is the easiest cross-platform way to check that the db was deleted: - //REQUIRE(remove(string(databasePath()).c_str()) != 0); - REQUIRE(errno == ENOENT); - } -} - -TEST_CASE_METHOD(C4RESTTest, "REST PUT database", "[REST][Listener][C]") { - unique_ptr r; - SECTION("Disallowed") { - r = request("PUT", "/db", HTTPStatus::Forbidden); - r = request("PUT", "/otherdb", HTTPStatus::Forbidden); - // r = request("PUT", "/and%2For", HTTPStatus::Forbidden); // that's a slash. This is a legal db name. - } - SECTION("Allowed") { - setUpDirectory(); - SECTION("Duplicate") { r = request("PUT", "/db", HTTPStatus::PreconditionFailed); } - SECTION("New DB") { - r = request("PUT", "/otherdb", HTTPStatus::Created); - r = request("GET", "/otherdb", HTTPStatus::OK); - - SECTION("Test _all_dbs again") { - r = request("GET", "/_all_dbs", HTTPStatus::OK); - auto body = r->bodyAsJSON().asArray(); - REQUIRE(body.count() == 2); - CHECK(to_str(body[0]) == "db"); - CHECK(to_str(body[1]) == "otherdb"); - } - } - } -} - -# pragma mark - COLLECTIONS: - -TEST_CASE_METHOD(C4RESTTest, "REST GET database with collections", "[REST][Listener][C]") { - REQUIRE(c4db_createCollection(db, {"guitars"_sl, "stuff"_sl}, ERROR_INFO())); - REQUIRE(c4db_createCollection(db, {"synths"_sl, "stuff"_sl}, ERROR_INFO())); - - unique_ptr r; - r = request("GET", "/db", HTTPStatus::OK); - auto body = r->bodyAsJSON().asDict(); - REQUIRE(body); - Log("%.*s", SPLAT(r->body())); - Dict scopes = body["scopes"].asDict(); - REQUIRE(scopes); - CHECK(scopes.count() == 2); - Dict stuff = scopes["stuff"].asDict(); - REQUIRE(stuff); - Dict guitars = stuff["guitars"].asDict(); - REQUIRE(guitars); - CHECK(guitars["doc_count"].type() == kFLNumber); - CHECK(guitars["doc_count"].asInt() == 0); - CHECK(guitars["update_seq"].type() == kFLNumber); - CHECK(guitars["update_seq"].asInt() == 0); - CHECK(stuff["synths"].asDict()); - - Dict dflt = scopes["_default"].asDict(); - REQUIRE(dflt); - CHECK(dflt.count() == 1); - Dict dfltColl = dflt["_default"].asDict(); - REQUIRE(dfltColl); - CHECK(dfltColl["doc_count"].type() == kFLNumber); - CHECK(dfltColl["doc_count"].asInt() == 0); - CHECK(dfltColl["update_seq"].type() == kFLNumber); - CHECK(dfltColl["update_seq"].asInt() == 0); -} - -TEST_CASE_METHOD(C4RESTTest, "REST GET collection", "[REST][Listener][C]") { - REQUIRE(c4db_createCollection(db, {"guitars"_sl, "stuff"_sl}, ERROR_INFO())); - - unique_ptr r; - r = request("GET", "/db.guitars/", HTTPStatus::NotFound); - - { - ExpectingExceptions x; - r = request("GET", "/./", HTTPStatus::BadRequest); - r = request("GET", "/db./", HTTPStatus::BadRequest); - r = request("GET", "/.db./", HTTPStatus::BadRequest); - r = request("GET", "/db../", HTTPStatus::BadRequest); - r = request("GET", "/db.stuff.guitars./", HTTPStatus::BadRequest); - } - - r = request("GET", "/db.foo/", HTTPStatus::NotFound); - r = request("GET", "/db.foo.bar/", HTTPStatus::NotFound); - - r = request("GET", "/db.stuff.guitars/", HTTPStatus::OK); - auto body = r->bodyAsJSON().asDict(); - REQUIRE(body); - CHECK(to_str(body["db_name"]) == "db"); - CHECK(to_str(body["collection_name"]) == "guitars"); - CHECK(to_str(body["scope_name"]) == "stuff"); - CHECK(body["db_uuid"].type() == kFLString); - CHECK(body["db_uuid"].asString().size >= 32); - CHECK(body["doc_count"].type() == kFLNumber); - CHECK(body["doc_count"].asInt() == 0); - CHECK(body["update_seq"].type() == kFLNumber); - CHECK(body["update_seq"].asInt() == 0); -} - -TEST_CASE_METHOD(C4RESTTest, "REST DELETE collection", "[REST][Listener][C]") { - REQUIRE(c4db_createCollection(db, {"guitars"_sl, "stuff"_sl}, ERROR_INFO())); - - unique_ptr r; - SECTION("Disallowed") { r = request("DELETE", "/db.stuff.guitars", HTTPStatus::Forbidden); } - SECTION("Allowed") { - config.allowDeleteCollections = true; - r = request("DELETE", "/db.stuff.guitars", HTTPStatus::OK); - r = request("GET", "/db.stuff.guitars", HTTPStatus::NotFound); - } -} - -TEST_CASE_METHOD(C4RESTTest, "REST PUT collection", "[REST][Listener][C]") { - unique_ptr r; - SECTION("Disallowed") { - r = request("PUT", "/db.foo", HTTPStatus::Forbidden); - r = request("PUT", "/db.foo.bar", HTTPStatus::Forbidden); - } - SECTION("Allowed") { - config.allowCreateCollections = true; - SECTION("Duplicate") { r = request("PUT", "/db._default._default", HTTPStatus::PreconditionFailed); } - SECTION("New Collection") { - r = request("PUT", "/db.guitars", HTTPStatus::Created); - r = request("GET", "/db.guitars", HTTPStatus::OK); - } - } -} - -# pragma mark - DOCUMENTS: - -TEST_CASE_METHOD(C4RESTTest, "REST CRUD", "[REST][Listener][C]") { - unique_ptr r; - Dict body; - alloc_slice docID; - - string dbPath; - C4Collection* coll; - bool inCollection = GENERATE(false, true); - if ( inCollection ) { - Log("---- Using collection 'coll'"); - coll = c4db_createCollection(db, {"coll"_sl}, ERROR_INFO()); - REQUIRE(coll); - dbPath = "/db.coll"; - } else { - coll = c4db_getDefaultCollection(db, nullptr); - dbPath = "/db"; - } - - SECTION("POST") { - r = request("POST", dbPath, {{"Content-Type", "application/json"}}, R"({"year": 1964})", HTTPStatus::Created); - body = r->bodyAsJSON().asDict(); - docID = body["id"].asString(); - CHECK(docID.size >= 20); - } - - SECTION("PUT") { - r = request("PUT", dbPath + "/mydocument", {{"Content-Type", "application/json"}}, R"({"year": 1964})", - HTTPStatus::Created); - body = r->bodyAsJSON().asDict(); - docID = body["id"].asString(); - CHECK(docID == "mydocument"_sl); - - request("PUT", dbPath + "/mydocument", {{"Content-Type", "application/json"}}, R"({"year": 1977})", - HTTPStatus::Conflict); - request("PUT", dbPath + "/mydocument", {{"Content-Type", "application/json"}}, - R"({"year": 1977, "_rev":"1-ffff"})", HTTPStatus::Conflict); - } - - CHECK(body["ok"].asBool() == true); - alloc_slice revID(body["rev"].asString()); - CHECK(revID.size > 0); - - { - c4::ref doc = c4coll_getDoc(coll, docID, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK(doc->revID == revID); - body = c4doc_getProperties(doc); - CHECK(body["year"].asInt() == 1964); - CHECK(body.count() == 1); // i.e. no _id or _rev properties - } - - r = request("GET", dbPath + "/" + docID.asString(), HTTPStatus::OK); - body = r->bodyAsJSON().asDict(); - CHECK(body["_id"].asString() == docID); - CHECK(body["_rev"].asString() == revID); - CHECK(body["year"].asInt() == 1964); - - r = request("DELETE", dbPath + "/" + docID.asString() + "?rev=" + revID.asString(), HTTPStatus::OK); - body = r->bodyAsJSON().asDict(); - CHECK(body["ok"].asBool() == true); - revID = body["rev"].asString(); - - { - c4::ref doc = c4coll_getDoc(coll, docID, true, kDocGetAll, ERROR_INFO()); - REQUIRE(doc); - CHECK((doc->flags & kDocDeleted) != 0); - CHECK(doc->revID == revID); - body = c4doc_getProperties(doc); - CHECK(body.count() == 0); - } - - r = request("GET", dbPath + "/" + docID.asString(), HTTPStatus::NotFound); -} - -TEST_CASE_METHOD(C4RESTTest, "REST _all_docs", "[REST][Listener][C]") { - auto r = request("GET", "/db/_all_docs", HTTPStatus::OK); - auto body = r->bodyAsJSON().asDict(); - auto rows = body["rows"].asArray(); - CHECK(rows); - CHECK(rows.count() == 0); - - request("PUT", "/db/mydocument", {{"Content-Type", "application/json"}}, R"({"year": 1964})", HTTPStatus::Created); - request("PUT", "/db/foo", {{"Content-Type", "application/json"}}, R"({"age": 17})", HTTPStatus::Created); - - r = request("GET", "/db/_all_docs", HTTPStatus::OK); - body = r->bodyAsJSON().asDict(); - rows = body["rows"].asArray(); - CHECK(rows); - CHECK(rows.count() == 2); - auto row = rows[0].asDict(); - CHECK(row["key"].asString() == "foo"_sl); - row = rows[1].asDict(); - CHECK(row["key"].asString() == "mydocument"_sl); -} - -TEST_CASE_METHOD(C4RESTTest, "REST _bulk_docs", "[REST][Listener][C]") { - unique_ptr r; - r = request("POST", "/db/_bulk_docs", {{"Content-Type", "application/json"}}, - json5("{docs:[{year:1962}, " - "{_id:'jens', year:1964}, " - "{_id:'bob', _rev:'1-eeee', year:1900}]}"), - HTTPStatus::OK); - Array body = r->bodyAsJSON().asArray(); - CHECK(body.count() == 3); - - Dict doc = body[0].asDict(); - CHECK(doc); - CHECK(doc["ok"].asBool()); - CHECK(doc["id"].asString().size > 0); - CHECK(doc["rev"].asString().size > 0); - - doc = body[1].asDict(); - CHECK(doc); - CHECK(doc["ok"].asBool()); - CHECK(doc["id"].asString() == "jens"_sl); - CHECK(doc["rev"].asString().size > 0); - - doc = body[2].asDict(); - CHECK(doc); - CHECK(!doc["ok"]); - CHECK(!doc["id"]); - CHECK(!doc["rev"]); - CHECK(doc["status"].asInt() == 404); - CHECK(doc["error"].asString() == "Not Found"_sl); -} - -# pragma mark - HTTP AUTH: - -TEST_CASE_METHOD(C4RESTTest, "REST HTTP auth missing", "[REST][Listener][C]") { - setupHTTPAuth(); - allowHTTPConnection = false; - auto r = request("GET", "/", HTTPStatus::Unauthorized); - CHECK(r->header("WWW-Authenticate") == "Basic charset=\"UTF-8\""); - CHECK(receivedHTTPAuthFromListener == listener()); - CHECK(receivedHTTPAuthHeader == nullslice); -} - -TEST_CASE_METHOD(C4RESTTest, "REST HTTP auth incorrect", "[REST][Listener][C]") { - setupHTTPAuth(); - allowHTTPConnection = false; - auto r = request("GET", "/", {{"Authorization", "Basic xxxx"}}, nullslice, HTTPStatus::Unauthorized); - CHECK(r->header("WWW-Authenticate") == "Basic charset=\"UTF-8\""); - CHECK(receivedHTTPAuthFromListener == listener()); - CHECK(receivedHTTPAuthHeader == "Basic xxxx"); -} - -TEST_CASE_METHOD(C4RESTTest, "REST HTTP auth correct", "[REST][Listener][C]") { - setupHTTPAuth(); - allowHTTPConnection = true; - auto r = request("GET", "/", {{"Authorization", "Basic xxxx"}}, nullslice, HTTPStatus::OK); - CHECK(receivedHTTPAuthFromListener == listener()); - CHECK(receivedHTTPAuthHeader == "Basic xxxx"); -} - -# pragma mark - TLS: - - -# ifdef COUCHBASE_ENTERPRISE - -TEST_CASE_METHOD(C4RESTTest, "TLS REST URLs", "[REST][Listener][C]") { - (void)useServerTLSWithTemporaryKey(); - share(db, "db"_sl); - auto configPortStr = to_string(c4listener_getPort(listener())); - string expectedSuffix = string(":") + configPortStr + "/"; - forEachURL(nullptr, kC4RESTAPI, [&expectedSuffix](string_view url) { - C4Log("Listener URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, "https://")); - CHECK(hasSuffix(url, expectedSuffix)); - }); - forEachURL(db, kC4RESTAPI, [&expectedSuffix](string_view url) { - C4Log("Database URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, "https://")); - CHECK(hasSuffix(url, expectedSuffix + "db")); - }); - - { - ExpectingExceptions x; - C4Error err; - FLMutableArray invalid = c4listener_getURLs(listener(), db, kC4SyncAPI, &err); - CHECK(!invalid); - CHECK(err.domain == LiteCoreDomain); - CHECK(err.code == kC4ErrorInvalidParameter); - } -} - -TEST_CASE_METHOD(C4RESTTest, "TLS REST untrusted cert", "[REST][Listener][TLS][C]") { - (void)useServerTLSWithTemporaryKey(); - - gC4ExpectExceptions = true; - auto r = request("GET", "/", HTTPStatus::undefined); - CHECK(r->error() == (C4Error{NetworkDomain, kC4NetErrTLSCertUnknownRoot})); -} - -TEST_CASE_METHOD(C4RESTTest, "TLS REST pinned cert", "[REST][Listener][TLS][C]") { - { - ExpectingExceptions x; - pinnedCert = useServerTLSWithTemporaryKey(); - } - testRootLevel(); -} - - -# ifdef PERSISTENT_PRIVATE_KEY_AVAILABLE -TEST_CASE_METHOD(C4RESTTest, "TLS REST pinned cert persistent key", "[REST][Listener][TLS][C]") { - { - ExpectingExceptions x; - pinnedCert = useServerTLSWithPersistentKey(); - } - testRootLevel(); -} -# endif - - -TEST_CASE_METHOD(C4RESTTest, "TLS REST client cert", "[REST][Listener][TLS][C]") { - pinnedCert = useServerTLSWithTemporaryKey(); - useClientTLSWithTemporaryKey(); - testRootLevel(); -} - -TEST_CASE_METHOD(C4RESTTest, "TLS REST client cert w/auth callback", "[REST][Listener][TLS][C]") { - pinnedCert = useServerTLSWithTemporaryKey(); - useClientTLSWithTemporaryKey(); - - setupCertAuth(); - config.tlsConfig->requireClientCerts = true; - allowClientCert = false; - - auto r = request("GET", "/", HTTPStatus::undefined); - CHECK(r->error() == (C4Error{NetworkDomain, kC4NetErrTLSCertRejectedByPeer})); -} - -TEST_CASE_METHOD(C4RESTTest, "TLS REST cert chain", "[REST][Listener][TLS][C]") { - Identity ca = CertHelper::createIdentity(false, kC4CertUsage_TLS_CA, "Test CA", nullptr, nullptr, true); - useServerIdentity(CertHelper::createIdentity(false, kC4CertUsage_TLSServer, "localhost", nullptr, &ca)); - auto summary = alloc_slice(c4cert_summary(serverIdentity.cert)); - useClientIdentity(CertHelper::createIdentity(false, kC4CertUsage_TLSClient, "Test Client", nullptr, &ca)); - setListenerRootClientCerts(ca.cert); - rootCerts = c4cert_retain(ca.cert); - testRootLevel(); -} - -TEST_CASE_METHOD(C4RESTTest, "Sync Listener URLs", "[REST][Listener][TLS][C]") { - bool expectErrorForREST = false; - string restScheme = "http"; - string syncScheme = "ws"; - - config.allowPull = true; - config.allowPush = true; - SECTION("Plain") { - SECTION("With REST") { config.apis = kC4RESTAPI | kC4SyncAPI; } - - SECTION("Without REST") { - expectErrorForREST = true; - config.apis = kC4SyncAPI; - } - } - - SECTION("TLS") { - (void)useServerTLSWithTemporaryKey(); - syncScheme = "wss"; - SECTION("With REST") { - restScheme = "https"; - config.apis = kC4RESTAPI | kC4SyncAPI; - } - - SECTION("Without REST") { - expectErrorForREST = true; - config.apis = kC4SyncAPI; - } - } - - share(db, "db"); - auto configPortStr = to_string(c4listener_getPort(listener())); - string expectedSuffix = string(":") + configPortStr + "/"; - if ( expectErrorForREST ) { - C4Error err; - ExpectingExceptions e; - FLMutableArray invalid = c4listener_getURLs(listener(), db, kC4RESTAPI, &err); - CHECK(!invalid); - CHECK(err.domain == LiteCoreDomain); - CHECK(err.code == kC4ErrorInvalidParameter); - } else { - forEachURL(db, kC4RESTAPI, [&expectedSuffix, &restScheme](string_view url) { - C4Log("Database URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, restScheme)); - CHECK(hasSuffix(url, expectedSuffix + "db")); - }); - } - - forEachURL(db, kC4SyncAPI, [&expectedSuffix, &syncScheme](string_view url) { - C4Log("Database URL = <%.*s>", SPLAT(slice(url))); - CHECK(hasPrefix(url, syncScheme)); - CHECK(hasSuffix(url, expectedSuffix + "db")); - }); -} - -// Following test cases that are tagged with [.SyncServer] are not exercised in the automatic build. -// They require special set-up of the Sync Gateway server, c.f. ReplicatorSGTest.cc. - -// Continuous replication without authentication -TEST_CASE_METHOD(C4RESTTest, "REST HTTP Replicate, Continuous", "[REST][Listener][C][.SyncServer]") { - importJSONLines(sFixturesDir + "names_100.json"); - std::stringstream body; - string targetDb = slice(ReplicatorAPITest::kScratchDBName).asString(); - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "continuous: true}"; - shared_ptr r = - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Idle, 5)); - - std::stringstream newss; - body.swap(newss); - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "cancel: true}"; - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Stopped, 5)); -} - -// OneShot replication. -TEST_CASE_METHOD(C4RESTTest, "REST HTTP Replicate, OneShot", "[REST][Listener][C][.SyncServer]") { - importJSONLines(sFixturesDir + "names_100.json"); - std::stringstream body; - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << slice(ReplicatorAPITest::kScratchDBName).asString() << "'," - << "continuous: false}"; - - auto r = request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(r->status() == HTTPStatus::OK); -} - -// Continuous replication with authentication -TEST_CASE_METHOD(C4RESTTest, "REST HTTP Replicate Continuous, Auth", "[REST][Listener][C][.SyncServer]") { - importJSONLines(sFixturesDir + "names_100.json"); - std::stringstream body; - string targetDb = slice(ReplicatorAPITest::kProtectedDBName).asString(); - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "user: 'pupshaw'," - << "password: 'frank'," - << "continuous: true}"; - shared_ptr r = - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Idle, 5)); - - std::stringstream newss; - body.swap(newss); - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "cancel: true}"; - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Stopped, 5)); -} - -// This test uses SG set up for [.SyncServerCollection], but without TLS. c.f ReplicatorCollectionSG.cc. -TEST_CASE_METHOD(C4RESTTest, "REST HTTP Replicate Continuous (collections)", - "[REST][Listener][C][.SyncServerCollectionHTTP]") { - constexpr C4CollectionSpec Roses = {"roses"_sl, "flowers"_sl}; - constexpr C4CollectionSpec Tulips = {"tulips"_sl, "flowers"_sl}; - std::array collections{db->createCollection(Roses), db->createCollection(Tulips)}; - string idPrefix = timePrefix(); - for ( auto& coll : collections ) { importJSONLines(sFixturesDir + "names_100.json", coll, 0, false, 0, idPrefix); } - - std::stringstream body; - string targetDb = "scratch"; - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "user: 'pupshaw'," - << "password: 'frank'," - << "continuous: true," - << "collections: ['flowers.roses','flowers.tulips']}"; - - shared_ptr r = - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Idle, 5)); - - std::stringstream newss; - body.swap(newss); - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << targetDb << "'," - << "cancel: true}"; - request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(wait(r, kC4Stopped, 5)); -} - -// OneShot replication with authentication. -TEST_CASE_METHOD(C4RESTTest, "REST HTTP Replicate Oneshot, Auth", "[REST][Listener][C][.SyncServer]") { - importJSONLines(sFixturesDir + "names_100.json"); - std::stringstream body; - body << "{source: 'db'," - << "target: 'ws://localhost:4984/" << slice(ReplicatorAPITest::kProtectedDBName).asString() << "'," - << "user: 'pupshaw'," - << "password: 'frank'," - "continuous: false}"; - - auto r = request("POST", "/_replicate", {{"Content-Type", "application/json"}}, json5(body.str()), HTTPStatus::OK); - CHECK(r->status() == HTTPStatus::OK); -} - -# endif // COUCHBASE_ENTERPRISE - -#endif diff --git a/REST/tests/SyncListenerTest.cc b/REST/tests/SyncListenerTest.cc index 8c3403ada..ed97c88e8 100644 --- a/REST/tests/SyncListenerTest.cc +++ b/REST/tests/SyncListenerTest.cc @@ -16,6 +16,7 @@ #include "Server.hh" #include "NetworkInterfaces.hh" #include "c4Replicator.h" +#include "TCPSocket.hh" #include using namespace litecore::REST; @@ -40,12 +41,10 @@ class C4SyncListenerTest _sg.remoteDBName = C4STR("db2"); } - static constexpr C4ListenerConfig kConfig = [] { - C4ListenerConfig config = {}; - config.apis = kC4SyncAPI; - config.allowPush = config.allowPull = true; - return config; - }(); + static constexpr C4ListenerConfig kConfig = { + .allowPush = true, + .allowPull = true, + }; void run(bool expectSuccess = true) { ReplicatorAPITest::importJSONLines(sFixturesDir + "names_100.json"); @@ -234,6 +233,8 @@ TEST_CASE_METHOD(C4SyncListenerTest, "P2P Sync connection count", "[Listener][C] } TEST_CASE_METHOD(C4SyncListenerTest, "P2P ReadOnly Sync", "[Push][Pull][Listener][C]") { + // This method tests disabling push or pull in the listener. + // All these replications are expected to fail because the listener prevents them. C4ReplicatorMode pushMode = kC4Disabled; C4ReplicatorMode pullMode = kC4Disabled; SECTION("Push") { @@ -261,7 +262,13 @@ TEST_CASE_METHOD(C4SyncListenerTest, "P2P ReadOnly Sync", "[Push][Pull][Listener } TEST_CASE_METHOD(C4SyncListenerTest, "P2P Server Addresses", "[Listener]") { - fleece::Retained s(new Server()); + class FakeDelegate : public Server::Delegate { + public: + void handleConnection(std::unique_ptr) override {} + }; + + FakeDelegate delegate; + fleece::Retained s(new Server(delegate)); s->start(0); auto addresses = s->addresses(); s->stop(); @@ -310,6 +317,7 @@ TEST_CASE_METHOD(C4SyncListenerTest, "Listener stops replicators", "[Listener]") _sg.address.port = c4listener_getPort(listener()); REQUIRE(startReplicator(kC4Continuous, kC4Continuous, WITH_ERROR())); waitForStatus(kC4Idle); + C4Log(" >>> Replicator is idle; stopping"); stop(); waitForStatus(kC4Stopped); } diff --git a/Replicator/ChangesFeed.cc b/Replicator/ChangesFeed.cc index 34303e545..d7aaa4986 100644 --- a/Replicator/ChangesFeed.cc +++ b/Replicator/ChangesFeed.cc @@ -38,13 +38,10 @@ namespace litecore::repl { , _delegate(delegate) , _options(options) , _db(db) + , _collectionSpec(checkpointer->collectionSpec()) + , _collectionIndex(CollectionIndex(_options->collectionSpecToIndex().at(checkpointer->collectionSpec()))) , _checkpointer(checkpointer) , _skipDeleted(_options->skipDeleted()) { - DebugAssert(_checkpointer); - - // JIM: This breaks tons of encapsulation, and should be reworked - _collectionIndex = - (CollectionIndex)_options->collectionSpecToIndex().at(_checkpointer->collection()->getSpec()); _continuous = _options->push(_collectionIndex) == kC4Continuous; filterByDocIDs(_options->docIDs(_collectionIndex)); } @@ -72,8 +69,8 @@ namespace litecore::repl { // Start the observer immediately, before querying historical changes, to avoid any // gaps between the history and notifications. But do not set `_notifyOnChanges` yet. logVerbose("Starting DB observer"); - _changeObserver = C4DatabaseObserver::create(_checkpointer->collection(), - [this](C4DatabaseObserver*) { this->_dbChanged(); }); + BorrowedCollection coll(_db.useWriteable(), _collectionSpec); + _changeObserver = C4DatabaseObserver::create(coll, [this](C4DatabaseObserver*) { this->_dbChanged(); }); } Changes changes = {}; @@ -94,25 +91,23 @@ namespace litecore::repl { // Run a by-sequence enumerator to find the changed docs: C4EnumeratorOptions options = kC4DefaultEnumeratorOptions; - // TBD: pushFilter should be collection-aware. + // TODO: pushFilter should be collection-aware. if ( !_getForeignAncestors && !_options->pushFilter(_collectionIndex) ) options.flags &= ~kC4IncludeBodies; if ( !_skipDeleted ) options.flags |= kC4IncludeDeleted; if ( _db.usingVersionVectors() ) options.flags |= kC4IncludeRevHistory; try { - _db.useLocked([&](C4Database* db) { - Assert(db == _checkpointer->collection()->getDatabase()); - C4DocEnumerator e(_checkpointer->collection(), _maxSequence, options); - changes.revs.reserve(limit); - while ( e.next() && limit > 0 ) { - C4DocumentInfo info = e.documentInfo(); - auto rev = makeRevToSend(info, &e); - if ( rev ) { - changes.revs.push_back(rev); - --limit; - } + BorrowedCollection collection = _db.useCollection(_collectionSpec); + C4DocEnumerator e(collection, _maxSequence, options); + changes.revs.reserve(limit); + while ( e.next() && limit > 0 ) { + C4DocumentInfo info = e.documentInfo(); + auto rev = makeRevToSend(info, &e); + if ( rev ) { + changes.revs.push_back(rev); + --limit; } - }); + } } catch ( ... ) { changes.err = C4Error::fromCurrentException(); } if ( limit > 0 && !_caughtUp ) { @@ -137,13 +132,6 @@ namespace litecore::repl { uint32_t nChanges = nextObservation.numChanges; if ( nChanges == 0 ) break; - if ( !nextObservation.external && !_echoLocalChanges ) { - logDebug("Observed %u of my own db changes #%" PRIu64 " ... #%" PRIu64 " (ignoring)", nChanges, - static_cast(c4changes[0].sequence), - static_cast(c4changes[nChanges - 1].sequence)); - _maxSequence = c4changes[nChanges - 1].sequence; - continue; // ignore changes I made myself - } logVerbose("Observed %u db changes #%" PRIu64 " ... #%" PRIu64, nChanges, (uint64_t)c4changes[0].sequence, (uint64_t)c4changes[nChanges - 1].sequence); @@ -212,7 +200,7 @@ namespace litecore::repl { (_docIDs != nullptr && _docIDs->find(slice(info.docID).asString()) == _docIDs->end()) ) { return nullptr; } else { - auto rev = make_retained(info, _checkpointer->collection()->getSpec(), + auto rev = make_retained(info, _checkpointer->collectionSpec(), _options->collectionCallbackContext(_collectionIndex)); return shouldPushRev(rev, e) ? rev : nullptr; } @@ -227,13 +215,11 @@ namespace litecore::repl { C4Error error; Retained doc; try { - _db.useLocked([&](C4Database* db) { - if ( e ) doc = e->getDocument(); - else - doc = _checkpointer->collection()->getDocument( - rev->docID, true, (needRemoteRevID ? kDocGetAll : kDocGetCurrentRev)); - if ( !doc ) error = C4Error::make(LiteCoreDomain, kC4ErrorNotFound); - }); + if ( e ) doc = e->getDocument(); + else + doc = _db.useCollection(_collectionSpec) + ->getDocument(rev->docID, true, (needRemoteRevID ? kDocGetAll : kDocGetCurrentRev)); + if ( !doc ) error = C4Error::make(LiteCoreDomain, kC4ErrorNotFound); } catch ( ... ) { error = C4Error::fromCurrentException(); } if ( !doc ) { _delegate.failedToGetChange(rev, error, false); @@ -249,7 +235,7 @@ namespace litecore::repl { } if ( _options->pushFilter(_collectionIndex) ) { // If there's a push filter, ask it whether to push the doc: - if ( !_options->pushFilter(_collectionIndex)(_checkpointer->collection()->getSpec(), doc->docID(), + if ( !_options->pushFilter(_collectionIndex)(_checkpointer->collectionSpec(), doc->docID(), doc->selectedRev().revID, doc->selectedRev().flags, doc->getProperties(), _options->collectionCallbackContext(_collectionIndex)) ) { diff --git a/Replicator/ChangesFeed.hh b/Replicator/ChangesFeed.hh index aa855b893..052000fbd 100644 --- a/Replicator/ChangesFeed.hh +++ b/Replicator/ChangesFeed.hh @@ -56,8 +56,6 @@ namespace litecore::repl { void setLastSequence(C4SequenceNumber s) { _maxSequence = s; } - void setEchoLocalChanges(bool echo) { _echoLocalChanges = echo; } - void setSkipDeletedDocs(bool skip) { _skipDeleted = skip; } void setCheckpointValid(bool valid) { _isCheckpointValid = valid; } @@ -102,20 +100,19 @@ namespace litecore::repl { Delegate& _delegate; RetainedConst _options; DBAccess& _db; + C4CollectionSpec const _collectionSpec; + CollectionIndex const _collectionIndex; bool _getForeignAncestors{false}; // True in propose-changes mode private: Checkpointer* _checkpointer; - DocIDSet _docIDs; // Doc IDs to filter to, or null - std::unique_ptr _changeObserver; // Used in continuous push mode - C4SequenceNumber _maxSequence{0}; // Latest sequence I've read - bool _continuous; // Continuous mode - bool _echoLocalChanges{false}; // True if including changes made by _db - bool _skipDeleted{false}; // True if skipping tombstones + DocIDSet _docIDs; // Doc IDs to filter to, or null + std::unique_ptr _changeObserver; // Used in continuous push mode + C4SequenceNumber _maxSequence{0}; // Latest sequence I've read + bool _continuous; // Continuous mode + bool _skipDeleted{false}; // True if skipping tombstones bool _isCheckpointValid{true}; bool _caughtUp{false}; // Delivered all historical changes std::atomic _notifyOnChanges{false}; // True if expecting change notification - CollectionIndex - _collectionIndex; // Identifies the collection index (in the replicator) of the collection being used }; class ReplicatorChangesFeed final : public ChangesFeed { diff --git a/Replicator/Checkpointer.cc b/Replicator/Checkpointer.cc index 95bcbd2f0..ac894e770 100644 --- a/Replicator/Checkpointer.cc +++ b/Replicator/Checkpointer.cc @@ -33,8 +33,8 @@ namespace litecore::repl { #pragma mark - CHECKPOINT ACCESSORS: - Checkpointer::Checkpointer(const Options* opt, fleece::slice remoteURL, C4Collection* collection) - : _options(opt), _remoteURL(remoteURL), _collection(collection) {} + Checkpointer::Checkpointer(const Options* opt, fleece::slice remoteURL, C4CollectionSpec const& spec) + : _options(opt), _collectionSpec(spec), _remoteURL(remoteURL) {} Checkpointer::~Checkpointer() = default; @@ -198,11 +198,10 @@ namespace litecore::repl { // checkpointers would be inaccessible. For this reason, the default // collection must remain unchanged. bool useSha1 = true; - if ( _collection != nullptr && _collection->getSpec() != kC4DefaultCollectionSpec ) { - auto spec = _collection->getSpec(); - auto index = _options->collectionSpecToIndex().at(spec); - enc.writeString(spec.name); - enc.writeString(spec.scope); + if ( _collectionSpec != kC4DefaultCollectionSpec ) { + auto index = _options->collectionSpecToIndex().at(_collectionSpec); + enc.writeString(_collectionSpec.name); + enc.writeString(_collectionSpec.scope); // CBL-501: Push only and pull only checkpoints create conflict // So include them in the derivation @@ -310,7 +309,7 @@ namespace litecore::repl { CollectionIndex i = collectionIndex(); return isDocumentIDAllowed(doc->docID()) && (!_options->pushFilter(i) - || _options->pushFilter(i)(_collection->getSpec(), doc->docID(), doc->selectedRev().revID, + || _options->pushFilter(i)(_collectionSpec, doc->docID(), doc->selectedRev().revID, doc->selectedRev().flags, doc->getProperties(), _options->collectionCallbackContext(collectionIndex()))); } @@ -329,8 +328,9 @@ namespace litecore::repl { } read(db, false); - const auto dbLastSequence = collection()->getLastSequence(); - const auto replLastSequence = this->localMinSequence(); + C4Collection* collection = db->getCollection(_collectionSpec); + const auto dbLastSequence = collection->getLastSequence(); + const auto replLastSequence = this->localMinSequence(); if ( replLastSequence >= dbLastSequence ) { // No changes since the last checkpoint return; @@ -344,7 +344,7 @@ namespace litecore::repl { opts.flags |= kC4IncludeBodies; } - C4DocEnumerator e(collection(), replLastSequence, opts); + C4DocEnumerator e(collection, replLastSequence, opts); while ( e.next() ) { C4DocumentInfo info = e.documentInfo(); @@ -379,7 +379,8 @@ namespace litecore::repl { } read(db, false); - Retained doc = collection()->getDocument(docId, false, kDocGetCurrentRev); + C4Collection* collection = db->getCollection(_collectionSpec); + Retained doc = collection->getDocument(docId, false, kDocGetCurrentRev); return doc && !_checkpoint->isSequenceCompleted(doc->sequence()) && isDocumentAllowed(doc); } diff --git a/Replicator/Checkpointer.hh b/Replicator/Checkpointer.hh index dae2d2fcf..24ac605a4 100644 --- a/Replicator/Checkpointer.hh +++ b/Replicator/Checkpointer.hh @@ -45,7 +45,7 @@ namespace litecore::repl { Replicator, Pusher and Puller. */ class Checkpointer { public: - Checkpointer(const Options* NONNULL, fleece::slice remoteURL, C4Collection*); + Checkpointer(const Options* NONNULL, fleece::slice remoteURL, C4CollectionSpec const&); ~Checkpointer(); @@ -161,7 +161,7 @@ namespace litecore::repl { alloc_slice& newRevID); /// The collection used internally during the operation of Replicator. - C4Collection* collection() const { return _collection; } + C4CollectionSpec collectionSpec() const { return _collectionSpec; } private: void checkpointIsInvalid(); @@ -172,11 +172,12 @@ namespace litecore::repl { void saveSoon(); CollectionIndex collectionIndex() const { - return fleece::narrow_cast(_options->collectionSpecToIndex().at(_collection->getSpec())); + return fleece::narrow_cast(_options->collectionSpecToIndex().at(_collectionSpec)); } Logging* _logger{}; RetainedConst _options; + C4CollectionSpec _collectionSpec; alloc_slice const _remoteURL; std::unordered_set _docIDs; @@ -196,7 +197,6 @@ namespace litecore::repl { std::unique_ptr _timer; SaveCallback _saveCallback; duration _saveTime{}; - C4Collection* const _collection; }; } // namespace litecore::repl diff --git a/Replicator/DBAccess.cc b/Replicator/DBAccess.cc index 4a28809e2..f9f8538c0 100644 --- a/Replicator/DBAccess.cc +++ b/Replicator/DBAccess.cc @@ -12,6 +12,7 @@ #include "DBAccess.hh" #include "DatabaseImpl.hh" +#include "DatabasePool.hh" #include "ReplicatedRev.hh" #include "ReplicatorTuning.hh" #include "Error.hh" @@ -30,36 +31,36 @@ namespace litecore::repl { using namespace std; using namespace fleece; - DBAccess::DBAccess(C4Database* db, bool disableBlobSupport) - : access_lock(db) - , Logging(SyncLog) - , _blobStore(&db->getBlobStore()) + DBAccess::DBAccess(DatabasePool* pool, bool disableBlobSupport) + : Logging(SyncLog) + , _pool(pool) , _disableBlobSupport(disableBlobSupport) - , _revsToMarkSynced(bind(&DBAccess::markRevsSyncedNow, this), bind(&DBAccess::markRevsSyncedLater, this), + , _revsToMarkSynced([this](int) { markRevsSyncedNow(); }, bind(&DBAccess::markRevsSyncedLater, this), tuning::kInsertionDelay) , _timer([this] { markRevsSyncedNow(); }) - , _usingVersionVectors((db->getConfiguration().flags & kC4DB_VersionVectors) != 0) {} - - AccessLockedDB& DBAccess::insertionDB() { - if ( !_insertionDB ) { - useLocked([&](C4Database* db) { - if ( !_insertionDB ) { - Retained idb; - try { - idb = db->openAgain(); - DatabaseImpl* impl = asInternal(idb); - logInfo("InsertionDB=%s", impl->dataFile()->loggingName().c_str()); - _c4db_setDatabaseTag(idb, DatabaseTag_DBAccess); - } catch ( const exception& x ) { - C4Error error = C4Error::fromException(x); - logError("Couldn't open new db connection: %s", error.description().c_str()); - idb = db; - } - _insertionDB.emplace(std::move(idb)); - } + , _usingVersionVectors((pool->getConfiguration().flags & kC4DB_VersionVectors) != 0) { + // There are a couple of read-only operations on a C4Database that may require write access + // the first time. Take care of those now so a read-only instance doesn't trigger them and + // cause a write-access exception: + BorrowedDatabase bdb = _pool->borrowWriteable(); + + // KeyStores' sequence indexes are created lazily. Force them to be created now: + bdb->forEachScope([&](slice scope) { + bdb->forEachCollection(scope, [&](C4CollectionSpec const& spec) { + (void)bdb->getCollection(spec)->getDocumentBySequence(9999999_seq); }); - } - return *_insertionDB; + }); + + // The SourceID may need to be generated and stored into the db: + if ( _usingVersionVectors ) _mySourceID = string(bdb->getSourceID()); + + // The BlobStore is thread-safe, so it can be used later without needing to borrow a db. + _blobStore = &bdb->getBlobStore(); + } + + DBAccess::DBAccess(C4Database* db, bool disableBlobSupport) : DBAccess(new DatabasePool(db), disableBlobSupport) { + _ownsPool = true; + _pool->setCapacity(2); } DBAccess::~DBAccess() { close(); } @@ -67,44 +68,26 @@ namespace litecore::repl { void DBAccess::close() { if ( _closed.test_and_set() ) { return; } _timer.stop(); - useLocked([this](Retained& db) { - // Any use of the class after this will result in a crash that - // should be easily identifiable, so forgo asserting if the pointer - // is null in other areas. - db = nullptr; - this->_sentry = &DBAccess::AssertDBOpen; - if ( this->_insertionDB ) { - this->_insertionDB->useLocked([](Retained& idb) { idb = nullptr; }); - this->_insertionDB.reset(); - } - }); + if ( _ownsPool ) _pool->close(); } - UseCollection DBAccess::useCollection(C4Collection* coll) { return {*this, coll}; } - - UseCollection DBAccess::useCollection(C4Collection* coll) const { return {*const_cast(this), coll}; } - string DBAccess::convertVersionToAbsolute(slice revID) { string version(revID); - if ( _usingVersionVectors ) { - if ( _mySourceID.empty() ) { - useLocked([&](C4Database* c4db) { - if ( _mySourceID.empty() ) _mySourceID = string(c4db->getSourceID()); - }); - } - replace(version, "*", _mySourceID); - } + if ( _usingVersionVectors ) replace(version, "*", _mySourceID); return version; } C4RemoteID DBAccess::lookUpRemoteDBID(slice key) { Assert(_remoteDBID == 0); - _remoteDBID = useLocked()->getRemoteDBID(key, true); + // (Needs useWriteable because getRemoteDBID may write to the database) + auto db = useWriteable(); + _remoteDBID = db->getRemoteDBID(key, true); return _remoteDBID; } - Retained DBAccess::getDoc(C4Collection* collection, slice docID, C4DocContentLevel content) const { - return useCollection(collection)->getDocument(docID, true, content); + Retained DBAccess::getDoc(C4CollectionSpec const& spec, slice docID, C4DocContentLevel content) const { + auto coll = useCollection(spec); + return coll->getDocument(docID, true, content); } alloc_slice DBAccess::getDocRemoteAncestor(C4Document* doc) const { @@ -113,20 +96,18 @@ namespace litecore::repl { return {}; } - void DBAccess::setDocRemoteAncestor(C4Collection* coll, slice docID, slice revID) { + void DBAccess::setDocRemoteAncestor(C4CollectionSpec const& spec, slice docID, slice revID) { if ( !_remoteDBID ) return; logInfo("Updating remote #%u's rev of '%.*s' to %.*s of collection %.*s.%.*s", _remoteDBID, SPLAT(docID), - SPLAT(revID), SPLAT(coll->getSpec().scope), SPLAT(coll->getSpec().name)); + SPLAT(revID), SPLAT(spec.scope), SPLAT(spec.name)); try { - useLocked([&](C4Database* db) { - Assert(db == coll->getDatabase()); - C4Database::Transaction t(db); - Retained doc = coll->getDocument(docID, true, kDocGetAll); - if ( !doc ) error::_throw(error::NotFound); - doc->setRemoteAncestorRevID(_remoteDBID, revID); - doc->save(); - t.commit(); - }); + BorrowedCollection coll(useWriteable(), spec); + C4Database::Transaction t(coll->getDatabase()); + Retained doc = coll->getDocument(docID, true, kDocGetAll); + if ( !doc ) error::_throw(error::NotFound); + doc->setRemoteAncestorRevID(_remoteDBID, revID); + doc->save(); + t.commit(); } catch ( const exception& x ) { C4Error error = C4Error::fromException(x); warn("Failed to update remote #%u's rev of '%.*s' to %.*s: %d/%d", _remoteDBID, SPLAT(docID), SPLAT(revID), @@ -134,16 +115,13 @@ namespace litecore::repl { } } - unique_ptr DBAccess::unresolvedDocsEnumerator(C4Collection* coll, bool orderByID) { + unique_ptr DBAccess::unresolvedDocsEnumerator(C4Collection* collection, bool orderByID) { C4EnumeratorOptions options = kC4DefaultEnumeratorOptions; options.flags &= ~kC4IncludeBodies; options.flags &= ~kC4IncludeNonConflicted; options.flags |= kC4IncludeDeleted; if ( !orderByID ) options.flags |= kC4Unsorted; - return useLocked>([&](const Retained& db) { - DebugAssert(db.get() == coll->getDatabase()); - return make_unique(coll, options); - }); + return make_unique(collection, options); } static bool containsAttachmentsProperty(slice json) { @@ -273,40 +251,38 @@ namespace litecore::repl { } SharedKeys DBAccess::updateTempSharedKeys() { - auto& db = _insertionDB ? *_insertionDB : *this; - SharedKeys result; - return db.useLocked([&](C4Database* idb) { - SharedKeys dbsk = idb->getFleeceSharedKeys(); - lock_guard lock(_tempSharedKeysMutex); - if ( !_tempSharedKeys || _tempSharedKeysInitialCount < dbsk.count() ) { - // Copy database's sharedKeys: - _tempSharedKeys = SharedKeys::create(dbsk.stateData()); - _tempSharedKeysInitialCount = dbsk.count(); - int retryCount = 0; - while ( _usuallyFalse(_tempSharedKeys.count() != dbsk.count() && retryCount++ < 10) ) { - // CBL-4288: Possible compiler optimization issue? If these two counts - // are not equal then the shared keys creation process has been corrupted - // and we must not continue as-is because then we will have data corruption - - // This really should not be the solution, but yet it reliably seems to stop - // this weirdness from happening - Warn("CBL-4288: Shared keys creation process failed, retrying..."); - _tempSharedKeys = SharedKeys::create(dbsk.stateData()); - } - - if ( _usuallyFalse(_tempSharedKeys.count() != dbsk.count()) ) { - // The above loop failed, so force an error condition to prevent a bad write - // Note: I have never seen this happen, it is here just because the alternative - // is data corruption, which is absolutely unacceptable - WarnError("CBL-4288: Retrying 10 times did not solve the issue, aborting document encode..."); - _tempSharedKeys = SharedKeys(); - } + SharedKeys result; + auto idb = _pool->borrow(); + SharedKeys dbsk = idb->getFleeceSharedKeys(); + lock_guard lock(_tempSharedKeysMutex); + if ( !_tempSharedKeys || _tempSharedKeysInitialCount < dbsk.count() ) { + // Copy database's sharedKeys: + _tempSharedKeys = SharedKeys::create(dbsk.stateData()); + _tempSharedKeysInitialCount = dbsk.count(); + int retryCount = 0; + while ( _usuallyFalse(_tempSharedKeys.count() != dbsk.count() && retryCount++ < 10) ) { + // CBL-4288: Possible compiler optimization issue? If these two counts + // are not equal then the shared keys creation process has been corrupted + // and we must not continue as-is because then we will have data corruption + + // This really should not be the solution, but yet it reliably seems to stop + // this weirdness from happening + Warn("CBL-4288: Shared keys creation process failed, retrying..."); + _tempSharedKeys = SharedKeys::create(dbsk.stateData()); + } - assert(_tempSharedKeys); + if ( _usuallyFalse(_tempSharedKeys.count() != dbsk.count()) ) { + // The above loop failed, so force an error condition to prevent a bad write + // Note: I have never seen this happen, it is here just because the alternative + // is data corruption, which is absolutely unacceptable + WarnError("CBL-4288: Retrying 10 times did not solve the issue, aborting document encode..."); + _tempSharedKeys = SharedKeys(); } - _tempSharedKeys.disableCaching(); - return _tempSharedKeys; - }); + + assert(_tempSharedKeys); + } + _tempSharedKeys.disableCaching(); + return _tempSharedKeys; } Doc DBAccess::tempEncodeJSON(slice jsonBody, FLError* err) { @@ -334,7 +310,7 @@ namespace litecore::repl { return doc; } - alloc_slice DBAccess::reEncodeForDatabase(Doc doc) { + alloc_slice DBAccess::reEncodeForDatabase(Doc doc, C4Database* idb) { bool reEncode; { lock_guard lock(_tempSharedKeysMutex); @@ -343,14 +319,11 @@ namespace litecore::repl { } if ( reEncode ) { // Re-encode with database's current sharedKeys: - // insertionDB() asserts DB open, no need to do it here - return insertionDB().useLocked([&](C4Database* idb) { - SharedEncoder enc(idb->sharedFleeceEncoder()); - enc.writeValue(doc.root()); - alloc_slice data = enc.finish(); - enc.reset(); - return data; - }); + SharedEncoder enc(idb->sharedFleeceEncoder()); + enc.writeValue(doc.root()); + alloc_slice data = enc.finish(); + enc.reset(); + return data; } else { // _tempSharedKeys is still compatible with database's sharedKeys, so no re-encoding. // But we do need to copy the data, because the data in doc is tagged with the temp @@ -359,14 +332,14 @@ namespace litecore::repl { } } - Doc DBAccess::applyDelta(C4Document* doc, slice deltaJSON, bool useDBSharedKeys) { + Doc DBAccess::applyDelta(C4Document* doc, slice deltaJSON, C4Database* db) { Dict srcRoot = doc->getProperties(); if ( !srcRoot ) error::_throw(error::CorruptRevisionData, "DBAccess applyDelta error getting document's properties"); bool useLegacyAttachments = !_disableBlobSupport && containsAttachmentsProperty(deltaJSON); Doc reEncodedDoc; - if ( useLegacyAttachments || !useDBSharedKeys ) { + if ( useLegacyAttachments || !db ) { Encoder enc; enc.setSharedKeys(tempSharedKeys()); if ( useLegacyAttachments ) { @@ -388,13 +361,10 @@ namespace litecore::repl { flErr = kFLInvalidData; } else { #endif - if ( useDBSharedKeys ) { - // insertionDB() asserts DB open, no need to do it here - insertionDB().useLocked([&](C4Database* idb) { - SharedEncoder enc(idb->sharedFleeceEncoder()); - JSONDelta::apply(srcRoot, deltaJSON, enc); - result = enc.finishDoc(&flErr); - }); + if ( db ) { + SharedEncoder enc(db->sharedFleeceEncoder()); + JSONDelta::apply(srcRoot, deltaJSON, enc); + result = enc.finishDoc(&flErr); } else { Encoder enc; enc.setSharedKeys(tempSharedKeys()); @@ -414,57 +384,59 @@ namespace litecore::repl { return result; } - Doc DBAccess::applyDelta(C4Collection* collection, slice docID, slice baseRevID, slice deltaJSON) { - Retained doc = getDoc(collection, docID, kDocGetAll); + Doc DBAccess::applyDelta(C4CollectionSpec const& spec, slice docID, slice baseRevID, slice deltaJSON) { + Retained doc = getDoc(spec, docID, kDocGetAll); if ( !doc ) error::_throw(error::NotFound); if ( !doc->selectRevision(baseRevID, true) || !doc->loadRevisionBody() ) return nullptr; - return applyDelta(doc, deltaJSON, false); + return applyDelta(doc, deltaJSON, nullptr); } void DBAccess::markRevSynced(ReplicatedRev* rev NONNULL) { _revsToMarkSynced.push(rev); } - // Mark all the queued revisions as synced to the server. void DBAccess::markRevsSyncedNow() { + BorrowedDatabase db = useWriteable(); + markRevsSyncedNow(db); + } + + // Mark all the queued revisions as synced to the server. + void DBAccess::markRevsSyncedNow(C4Database* db) { _timer.stop(); auto revs = _revsToMarkSynced.pop(); if ( !revs ) return; Stopwatch st; - // insertionDB() asserts DB open, no need to do it here - insertionDB().useLocked([&](C4Database* idb) { - try { - C4Database::Transaction transaction(idb); - for ( ReplicatedRev* rev : *revs ) { - C4CollectionSpec coll = rev->collectionSpec; - C4Collection* collection = idb->getCollection(coll); - if ( collection == nullptr ) { - C4Error::raise(LiteCoreDomain, kC4ErrorNotOpen, "%s", - stringprintf("Failed to find collection '%*s.%*s'.", SPLAT(coll.scope), - SPLAT(coll.name)) - .c_str()); - } - logDebug("Marking rev '%.*s'.%.*s '%.*s' %.*s (#%" PRIu64 ") as synced to remote db %u", - SPLAT(coll.scope), SPLAT(coll.name), SPLAT(rev->docID), SPLAT(rev->revID), - static_cast(rev->sequence), remoteDBID()); - try { - collection->markDocumentSynced(rev->docID, rev->revID, rev->sequence, - rev->rejectedByRemote ? 0 : remoteDBID()); - } catch ( const exception& x ) { - C4Error error = C4Error::fromException(x); - warn("Unable to mark '%.*s'.%.*s '%.*s' %.*s (#%" PRIu64 ") as synced; error %d/%d", - SPLAT(coll.scope), SPLAT(coll.name), SPLAT(rev->docID), SPLAT(rev->revID), - (uint64_t)rev->sequence, error.domain, error.code); - } + try { + C4Database::Transaction transaction(db); + for ( ReplicatedRev* rev : *revs ) { + C4CollectionSpec coll = rev->collectionSpec; + C4Collection* collection = db->getCollection(coll); + if ( collection == nullptr ) { + C4Error::raise( + LiteCoreDomain, kC4ErrorNotOpen, "%s", + stringprintf("Failed to find collection '%*s.%*s'.", SPLAT(coll.scope), SPLAT(coll.name)) + .c_str()); + } + logDebug("Marking rev '%.*s'.%.*s '%.*s' %.*s (#%" PRIu64 ") as synced to remote db %u", + SPLAT(coll.scope), SPLAT(coll.name), SPLAT(rev->docID), SPLAT(rev->revID), + static_cast(rev->sequence), remoteDBID()); + try { + collection->markDocumentSynced(rev->docID, rev->revID, rev->sequence, + rev->rejectedByRemote ? 0 : remoteDBID()); + } catch ( const exception& x ) { + C4Error error = C4Error::fromException(x); + warn("Unable to mark '%.*s'.%.*s '%.*s' %.*s (#%" PRIu64 ") as synced; error %d/%d", + SPLAT(coll.scope), SPLAT(coll.name), SPLAT(rev->docID), SPLAT(rev->revID), + (uint64_t)rev->sequence, error.domain, error.code); } - transaction.commit(); - double t = st.elapsed(); - logVerbose("Marked %zu revs as synced-to-server in %.2fms (%.0f/sec)", revs->size(), t * 1000, - (double)revs->size() / t); - } catch ( const exception& x ) { - C4Error error = C4Error::fromException(x); - warn("Error marking %zu revs as synced: %d/%d", revs->size(), error.domain, error.code); } - }); + transaction.commit(); + double t = st.elapsed(); + logVerbose("Marked %zu revs as synced-to-server in %.2fms (%.0f/sec)", revs->size(), t * 1000, + (double)revs->size() / t); + } catch ( const exception& x ) { + C4Error error = C4Error::fromException(x); + warn("Error marking %zu revs as synced: %d/%d", revs->size(), error.domain, error.code); + } } void DBAccess::markRevsSyncedLater() { _timer.fireAfter(tuning::kInsertionDelay); } diff --git a/Replicator/DBAccess.hh b/Replicator/DBAccess.hh index dac68230c..4fd855d59 100644 --- a/Replicator/DBAccess.hh +++ b/Replicator/DBAccess.hh @@ -15,6 +15,7 @@ #include "c4Database.hh" #include "c4Document.hh" #include "Batcher.hh" +#include "DatabasePool.hh" #include "Error.hh" #include "Logging.hh" #include "Timer.hh" @@ -32,34 +33,37 @@ namespace litecore::repl { class ReplicatedRev; class UseCollection; - using AccessLockedDB = access_lock>; - /** Thread-safe access to a C4Database. */ - class DBAccess - : public AccessLockedDB - , public Logging { + class DBAccess final : public Logging { public: using slice = fleece::slice; using alloc_slice = fleece::alloc_slice; using Dict = fleece::Dict; - DBAccess(C4Database* db, bool disableBlobSupport); + DBAccess(DatabasePool*, bool disableBlobSupport); + DBAccess(C4Database*, bool disableBlobSupport); ~DBAccess() override; - static void AssertDBOpen(const Retained& db) { - if ( !db ) { - litecore::error::_throw(litecore::error::Domain::LiteCore, litecore::error::LiteCoreError::NotOpen); - } + /// Returns a temporary object convertible to C4Database*. Use it only briefly. + BorrowedDatabase useLocked() const { return _pool->borrow(); } + + /// Returns a temporary object convertible to C4Collection*. Use it only briefly. + BorrowedCollection useCollection(C4CollectionSpec const& spec) const { + return BorrowedCollection(_pool->borrow(), spec); + } + + auto useLocked(auto callback) const { + BorrowedDatabase db = _pool->borrow(); + return callback(db.get()); } + /// Returns a writeable database. Use only when you need to write. + BorrowedDatabase useWriteable() { return _pool->borrowWriteable(); } + /** Shuts down the DBAccess and makes further use of it invalid. Any attempt to use it after this point is considered undefined behavior. */ void close(); - /** Check the C4Collection inside the lock and returns a holder object that hodes this->useLocked().*/ - UseCollection useCollection(C4Collection*); - UseCollection useCollection(C4Collection*) const; - /** Looks up the remote DB identifier of this replication. */ C4RemoteID lookUpRemoteDBID(slice key); @@ -70,21 +74,19 @@ namespace litecore::repl { std::string convertVersionToAbsolute(slice revID); - // (The "use" method is inherited from access_lock) - //////// DOCUMENTS: /** Gets a document by ID */ - Retained getDoc(C4Collection* NONNULL, slice docID, C4DocContentLevel content) const; + Retained getDoc(C4CollectionSpec const&, slice docID, C4DocContentLevel content) const; /** Returns the remote ancestor revision ID of a document. */ alloc_slice getDocRemoteAncestor(C4Document* doc NONNULL) const; /** Updates the remote ancestor revision ID of a document, to an existing revision. */ - void setDocRemoteAncestor(C4Collection* NONNULL, slice docID, slice revID); + void setDocRemoteAncestor(C4CollectionSpec const&, slice docID, slice revID); /** Returns the document enumerator for all unresolved docs present in the DB */ - std::unique_ptr unresolvedDocsEnumerator(C4Collection* NONNULL, bool orderByID); + static std::unique_ptr unresolvedDocsEnumerator(C4Collection*, bool orderByID); /** Mark this revision as synced (i.e. the server's current revision) soon. NOTE: While this is queued, calls to C4Document::getRemoteAncestor() for this doc won't @@ -95,17 +97,19 @@ namespace litecore::repl { /** Synchronously fulfills all markRevSynced requests. */ void markRevsSyncedNow(); + void markRevsSyncedNow(C4Database* db); //////// DELTAS: /** Applies a delta to an existing revision. Never returns NULL; - errors decoding or applying the delta are thrown as Fleece exceptions. */ - fleece::Doc applyDelta(C4Document* doc NONNULL, slice deltaJSON, bool useDBSharedKeys); + errors decoding or applying the delta are thrown as Fleece exceptions. + If `db` is non-null, the doc will be re-encoded with its SharedKeys. */ + fleece::Doc applyDelta(C4Document* doc NONNULL, slice deltaJSON, C4Database* db); /** Reads a document revision and applies a delta to it. Returns NULL if the baseRevID no longer exists or its body is not known. Other errors (including doc-not-found) are thrown as exceptions. */ - fleece::Doc applyDelta(C4Collection* NONNULL, slice docID, slice baseRevID, slice deltaJSON); + fleece::Doc applyDelta(C4CollectionSpec const&, slice docID, slice baseRevID, slice deltaJSON); //////// BLOBS / ATTACHMENTS: @@ -133,26 +137,23 @@ namespace litecore::repl { /** Takes a document produced by tempEncodeJSON and re-encodes it if necessary with the database's real SharedKeys, so it's suitable for saving. This can only be called inside a transaction. */ - alloc_slice reEncodeForDatabase(fleece::Doc); - - /** A separate C4Database instance used for insertions, to avoid blocking the main - C4Database. */ - AccessLockedDB& insertionDB(); + alloc_slice reEncodeForDatabase(fleece::Doc, C4Database*); - /** Manages a transaction safely. The begin() method calls beginTransaction, then commit() - or abort() end it. If the object exits scope when it's been begun but not yet - ended, it aborts the transaction. */ + /** Manages a transaction safely. Call commit() to commit, abort() to abort. + If the object exits scope when it's been begun but not yet ended, it aborts the transaction. */ class Transaction { public: - explicit Transaction(AccessLockedDB& dba) : _dba(dba.useLocked()), _t(_dba) {} + explicit Transaction(DBAccess& dba) : _db(dba.useWriteable()), _t(_db) {} + + C4Database* db() { return _db.get(); } void commit() { _t.commit(); } void abort() { _t.abort(); } private: - AccessLockedDB::access&> _dba; - C4Database::Transaction _t; + BorrowedDatabase _db; + C4Database::Transaction _t; }; static std::atomic gNumDeltasApplied; // For unit tests only @@ -165,7 +166,8 @@ namespace litecore::repl { fleece::SharedKeys tempSharedKeys(); fleece::SharedKeys updateTempSharedKeys(); - C4BlobStore* const _blobStore; // Database's BlobStore + Retained _pool; // Pool of C4Databases + C4BlobStore* _blobStore{}; // Database's BlobStore fleece::SharedKeys _tempSharedKeys; // Keys used in tempEncodeJSON() std::mutex _tempSharedKeysMutex; // Mutex for replacing _tempSharedKeys unsigned _tempSharedKeysInitialCount{0}; // Count when copied from db's keys @@ -174,25 +176,10 @@ namespace litecore::repl { bool const _disableBlobSupport; // Does replicator support blobs? actor::Batcher _revsToMarkSynced; // Pending revs to be marked as synced actor::Timer _timer; // Implements Batcher delay - std::optional _insertionDB; // DB handle to use for insertions - std::string _mySourceID; - const bool _usingVersionVectors; // True if DB uses version vectors - std::atomic_flag _closed = ATOMIC_FLAG_INIT; + std::string _mySourceID; // Version vector sourceID + const bool _usingVersionVectors; // True if DB uses version vectors + bool _ownsPool = false; // True if I created _pool + std::atomic_flag _closed = ATOMIC_FLAG_INIT; // True after closed }; - class UseCollection { - DBAccess& _dbAccess; - decltype(_dbAccess.useLocked()) _access; - C4Collection* _collection; - - public: - UseCollection(DBAccess& db_, C4Collection* collection) - : _dbAccess(db_), _access(_dbAccess.useLocked()), _collection(collection) { - Assert(_access.get() == _collection->getDatabase()); - } - - C4Collection* operator->() { return _collection; } - - const C4Collection* operator->() const { return _collection; } - }; } // namespace litecore::repl diff --git a/Replicator/IncomingRev+Blobs.cc b/Replicator/IncomingRev+Blobs.cc index 48c091816..da108cd16 100644 --- a/Replicator/IncomingRev+Blobs.cc +++ b/Replicator/IncomingRev+Blobs.cc @@ -132,12 +132,11 @@ namespace litecore::repl { // Sends periodic notifications to the Replicator if desired. void IncomingRev::notifyBlobProgress(bool always) { if ( progressNotificationLevel() < 2 ) return; - auto collSpec = getCollection()->getSpec(); - auto now = actor::Timer::clock::now(); + auto now = actor::Timer::clock::now(); if ( always || now - _lastNotifyTime > 250ms ) { _lastNotifyTime = now; Replicator::BlobProgress prog{Dir::kPulling, - collSpec, + collectionSpec(), _blob->docID, _blob->docProperty, _blob->key, diff --git a/Replicator/IncomingRev.cc b/Replicator/IncomingRev.cc index 95d6b71ee..915f7329e 100644 --- a/Replicator/IncomingRev.cc +++ b/Replicator/IncomingRev.cc @@ -61,7 +61,7 @@ namespace litecore::repl { _rev = new RevToInsert(this, _revMessage->property("id"_sl), _revMessage->property("rev"_sl), _revMessage->property("history"_sl), _revMessage->boolProperty("deleted"_sl), _revMessage->boolProperty("noconflicts"_sl) || _options->noIncomingConflicts(), - getCollection()->getSpec(), _options->collectionCallbackContext(collectionIndex())); + collectionSpec(), _options->collectionCallbackContext(collectionIndex())); _rev->deltaSrcRevID = _revMessage->property("deltaSrc"_sl); slice sequenceStr = _revMessage->property(slice("sequence")); _remoteSequence = RemoteSequence(sequenceStr); @@ -222,7 +222,7 @@ namespace litecore::repl { // have properties to decrypt. logVerbose("Need to apply delta immediately for '%.*s' #%.*s ...", SPLAT(_rev->docID), SPLAT(_rev->revID)); try { - fleeceDoc = _db->applyDelta(getCollection(), _rev->docID, _rev->deltaSrcRevID, jsonBody); + fleeceDoc = _db->applyDelta(collectionSpec(), _rev->docID, _rev->deltaSrcRevID, jsonBody); if ( !fleeceDoc ) { // Don't have the body of the source revision. This might be because I'm in // no-conflict mode and the peer is trying to push me a now-obsolete revision. @@ -372,8 +372,7 @@ namespace litecore::repl { // Calls the custom pull validator if available. bool IncomingRev::performPullValidation(Dict body) { if ( _options->pullFilter(collectionIndex()) ) { - if ( !_options->pullFilter(collectionIndex())(getCollection()->getSpec(), _rev->docID, _rev->revID, - _rev->flags, body, + if ( !_options->pullFilter(collectionIndex())(collectionSpec(), _rev->docID, _rev->revID, _rev->flags, body, _options->collectionCallbackContext(collectionIndex())) ) { failWithError(WebSocketDomain, 403, "rejected by validation function"_sl); return false; diff --git a/Replicator/Inserter.cc b/Replicator/Inserter.cc index 414470059..d75a2fb49 100644 --- a/Replicator/Inserter.cc +++ b/Replicator/Inserter.cc @@ -46,14 +46,15 @@ namespace litecore::repl { C4Error transactionErr = {}; try { - DBAccess::Transaction transaction(_db->insertionDB()); + DBAccess::Transaction transaction(*_db); + C4Collection* collection = transaction.db()->getCollection(collectionSpec()); // Before updating docs, write all pending changes to remote ancestors, in case any // of them apply to the docs we're updating: - _db->markRevsSyncedNow(); + _db->markRevsSyncedNow(transaction.db()); for ( RevToInsert* rev : *revs ) { C4Error docErr; - bool docSaved = insertRevisionNow(rev, &docErr); + bool docSaved = insertRevisionNow(rev, collection, &docErr); rev->trimBody(); // don't need body any more if ( docSaved ) { rev->owner->revisionProvisionallyInserted(rev->revocationMode != RevocationMode::kNone); @@ -95,13 +96,12 @@ namespace litecore::repl { } // Inserts one revision. Returns only C4Errors, never throws exceptions. - bool Inserter::insertRevisionNow(RevToInsert* rev, C4Error* outError) { + bool Inserter::insertRevisionNow(RevToInsert* rev, C4Collection* collection, C4Error* outError) { try { if ( rev->flags & kRevPurged ) { // Server says the document is no longer accessible, i.e. it's been // removed from all channels the client has access to. Purge it. - auto locked = _db->insertionDB().useLocked(); - if ( insertionCollection()->purgeDocument(rev->docID) ) { + if ( collection->purgeDocument(rev->docID) ) { auto collPath = _options->collectionPath(collectionIndex()); logVerbose(" {'%.*s (%.*s)' removed (purged)}", SPLAT(rev->docID), SPLAT(collPath)); } @@ -127,24 +127,24 @@ namespace litecore::repl { put.deltaCB = [](void* context, C4Document* doc, C4Slice delta, C4RevisionFlags* revFlags, C4Error* outError) -> C4SliceResult { try { - return ((Inserter*)context)->applyDeltaCallback(doc, delta, revFlags, outError); + auto self = (Inserter*)context; + return self->applyDeltaCallback(doc, delta, revFlags, outError); } catch ( ... ) { *outError = C4Error::fromCurrentException(); return {}; } }; - put.deltaCBContext = this; + put.deltaCBContext = this; + _callbackCollection = collection; } else { // If not a delta, encode doc body using database's real sharedKeys: - bodyForDB = _db->reEncodeForDatabase(rev->doc); + bodyForDB = _db->reEncodeForDatabase(rev->doc, collection->getDatabase()); rev->doc = nullptr; } put.allocedBody = {(void*)bodyForDB.buf, bodyForDB.size}; // The save!! - auto doc = _db->insertionDB().useLocked>([outError, &put, this](C4Database* db) { - return insertionCollection()->putDocument(put, nullptr, outError); - }); + auto doc = collection->putDocument(put, nullptr, outError); if ( !doc ) return false; auto collPath = _options->collectionPath(collectionIndex()); logVerbose(" {'%.*s (%.*s)' #%.*s <- %.*s} seq %" PRIu64, SPLAT(rev->docID), SPLAT(collPath), @@ -169,7 +169,8 @@ namespace litecore::repl { // Callback from c4doc_put() that applies a delta, during _insertRevisionsNow() C4SliceResult Inserter::applyDeltaCallback(C4Document* c4doc, C4Slice deltaJSON, C4RevisionFlags* revFlags, C4Error* outError) { - Doc doc = _db->applyDelta(c4doc, deltaJSON, true); + C4Database* db = _callbackCollection->getDatabase(); + Doc doc = _db->applyDelta(c4doc, deltaJSON, db); alloc_slice body = doc.allocedData(); Dict root = doc.root().asDict(); FLSharedKeys sk = nullptr; @@ -181,7 +182,7 @@ namespace litecore::repl { if ( C4Document::hasOldMetaProperties(root) ) { body = nullslice; try { - sk = _db->insertionDB().useLocked()->getFleeceSharedKeys(); + sk = db->getFleeceSharedKeys(); body = C4Document::encodeStrippingOldMetaProperties(root, sk); bodyChanged = true; } @@ -205,14 +206,4 @@ namespace litecore::repl { return C4SliceResult(body); } - C4Collection* Inserter::insertionCollection() { - if ( _insertionCollection ) return _insertionCollection; - - auto c4db = _db->insertionDB().useLocked(); - auto coll = c4db->getCollection(getCollection()->getSpec()); - if ( !coll ) C4Error::raise({LiteCoreDomain, kC4ErrorNotOpen}); - _insertionCollection = coll; - return _insertionCollection; - } - } // namespace litecore::repl diff --git a/Replicator/Inserter.hh b/Replicator/Inserter.hh index cfc0c5ce7..cb190e0a2 100644 --- a/Replicator/Inserter.hh +++ b/Replicator/Inserter.hh @@ -31,15 +31,13 @@ namespace litecore::repl { std::string loggingClassName() const override { return "Inserter"; } private: - C4Collection* insertionCollection(); // Get the collection from the insertionDB - void _insertRevisionsNow(int gen); - bool insertRevisionNow(RevToInsert* NONNULL, C4Error*); + bool insertRevisionNow(RevToInsert* NONNULL, C4Collection*, C4Error*); C4SliceResult applyDeltaCallback(C4Document* doc NONNULL, C4Slice deltaJSON, C4RevisionFlags* revFlags, C4Error* outError); - actor::ActorBatcher _revsToInsert; // Pending revs to be added to db - C4Collection* _insertionCollection{nullptr}; + actor::ActorBatcher _revsToInsert; // Pending revs to be added to db + C4Collection* _callbackCollection{}; // A kludge used by the delta callback }; } // namespace litecore::repl diff --git a/Replicator/Pusher+Attachments.cc b/Replicator/Pusher+Attachments.cc index 7feb9deb1..181a5cba6 100644 --- a/Replicator/Pusher+Attachments.cc +++ b/Replicator/Pusher+Attachments.cc @@ -102,7 +102,7 @@ namespace litecore::repl { auto collIndex = Worker::getCollectionIndex(*req); if ( collIndex != kNotCollectionIndex ) { - auto collSpec = repl->collection(collIndex)->getSpec(); + auto collSpec = repl->collectionSpec(collIndex); progress.collSpec = collSpec; } diff --git a/Replicator/Pusher+Revs.cc b/Replicator/Pusher+Revs.cc index 558a5e39c..926f896e7 100644 --- a/Replicator/Pusher+Revs.cc +++ b/Replicator/Pusher+Revs.cc @@ -69,9 +69,9 @@ namespace litecore::repl { // Get the document & revision: C4Error c4err = {}; Dict root; - auto collection = getCollection(); slice replacementRevID = nullslice; - Retained doc = _db->useCollection(collection)->getDocument(request->docID, true, kDocGetAll); + auto coll = _db->useCollection(collectionSpec()); + Retained doc = coll->getDocument(request->docID, true, kDocGetAll); if ( doc ) { if ( doc->selectRevision(request->revID, true) ) root = doc->getProperties(); if ( root ) request->flags = doc->selectedRev().flags; diff --git a/Replicator/Pusher.cc b/Replicator/Pusher.cc index c324e3503..3da7e8572 100644 --- a/Replicator/Pusher.cc +++ b/Replicator/Pusher.cc @@ -412,7 +412,7 @@ namespace litecore::repl { bool Pusher::shouldRetryConflictWithNewerAncestor(RevToSend* rev, slice receivedRevID) { if ( !_proposeChanges ) return false; try { - Retained doc = _db->getDoc(getCollection(), rev->docID, kDocGetAll); + Retained doc = _db->getDoc(collectionSpec(), rev->docID, kDocGetAll); if ( doc && C4Document::equalRevIDs(doc->revID(), rev->revID) ) { if ( receivedRevID && receivedRevID != rev->remoteAncestorRevID ) { // Remote ancestor received in proposeChanges response, so try with @@ -470,20 +470,20 @@ namespace litecore::repl { // See if the doc is unchanged, by getting it by sequence: Retained rev = i->second; _conflictsIMightRetry.erase(i); - auto* collection = getCollection(); - Retained doc = _db->useCollection(collection)->getDocumentBySequence(rev->sequence); + auto coll = _db->useCollection(collectionSpec()); + Retained doc = coll->getDocumentBySequence(rev->sequence); if ( !doc || !C4Document::equalRevIDs(doc->revID(), rev->revID) ) { // Local document has changed, so stop working on this revision: logVerbose("Notified that remote rev of '%.*s' of '%.*s.%.*s' is now #%.*s, " "but local doc has changed", - SPLAT(docID), SPLAT(collection->getSpec().scope), SPLAT(collection->getSpec().name), + SPLAT(docID), SPLAT(collectionSpec().scope), SPLAT(collectionSpec().name), SPLAT(foreignAncestor)); } else if ( doc->selectRevision(foreignAncestor, false) && !(doc->selectedRev().flags & kRevIsConflict) ) { // The remote rev is an ancestor of my revision, so retry it: doc->selectCurrentRevision(); logInfo("Notified that remote rev of '%.*s' of '%.*s.%.*s' is now #%.*s; " "retrying push of #%.*s", - SPLAT(docID), SPLAT(collection->getSpec().scope), SPLAT(collection->getSpec().name), + SPLAT(docID), SPLAT(collectionSpec().scope), SPLAT(collectionSpec().name), SPLAT(foreignAncestor), SPLAT(doc->revID())); rev->remoteAncestorRevID = foreignAncestor; gotOutOfOrderChange(rev); diff --git a/Replicator/Replicator.cc b/Replicator/Replicator.cc index ccb6bb2d3..533cdc16c 100644 --- a/Replicator/Replicator.cc +++ b/Replicator/Replicator.cc @@ -65,11 +65,26 @@ namespace litecore::repl { {{WebSocketDomain, 503, 0}, false, "The server is over capacity"_sl}, {{LiteCoreDomain, kC4ErrorRemoteError, 0}, true, "Unexpected error from remote"_sl}}; - std::string Replicator::ProtocolName() { - stringstream result; - delimiter delim(","); - for ( auto& name : kCompatProtocols ) result << delim << name; - return result.str(); + string toString(ProtocolVersion version) { + return stringprintf("%s+CBMobile_%d", blip::Connection::kWSProtocolName, int(version)); + } + + vector Replicator::compatibleProtocols(C4DatabaseFlags flags, Options::Mode pushMode, + Options::Mode pullMode) { + bool v3, v4; + if ( flags & kC4DB_VersionVectors ) { + // Local db may have VVs. V4 is fine. V3 is OK if I don't push anything. + v4 = true; + v3 = (pushMode == kC4Disabled); + } else { + // Local db does not have VVs. V3 is fine. V4 is OK if I don't pull anything. + v3 = true; + v4 = (pullMode == kC4Disabled); + } + vector result; + if ( v3 ) result.emplace_back(toString(ProtocolVersion::v3)); + if ( v4 ) result.emplace_back(toString(ProtocolVersion::v4)); + return result; } Replicator::Replicator(C4Database* db, websocket::WebSocket* webSocket, Delegate& delegate, Options* options) @@ -100,7 +115,7 @@ namespace litecore::repl { _loggingID = string(db->useLocked()->getPath()) + " " + _loggingID; _importance = 2; - string dbLogName = db->useLocked([](const C4Database* db) { + string dbLogName = db->useLocked([](const C4Database* db) { DatabaseImpl* impl = asInternal(db); return impl->dataFile()->loggingName(); }); @@ -177,18 +192,20 @@ namespace litecore::repl { void Replicator::_findExistingConflicts() { // Active replicator - Stopwatch st; + Stopwatch st; + BorrowedDatabase db = _db->useLocked(); for ( CollectionIndex i = 0; i < _subRepls.size(); ++i ) { SubReplicator& sub = _subRepls[i]; try { - unique_ptr e = _db->unresolvedDocsEnumerator(sub.collection, false); + C4Collection* collection = db->getCollection(sub.collectionSpec); + unique_ptr e = _db->unresolvedDocsEnumerator(collection, false); cLogInfo(i, "Scanning for pre-existing conflicts..."); unsigned nConflicts = 0; while ( e->next() ) { C4DocumentInfo info = e->documentInfo(); auto rev = retained(new RevToInsert(nullptr, /* incoming rev */ info.docID, info.revID, nullslice, /* history buf */ - info.flags & kDocDeleted, false, sub.collection->getSpec(), + info.flags & kDocDeleted, false, sub.collectionSpec, _options->collectionCallbackContext(i))); rev->error = C4Error::make(LiteCoreDomain, kC4ErrorConflict); _docsEnded.push(rev); @@ -589,20 +606,6 @@ namespace litecore::repl { "Incompatible replication protocol " "(missing 'Sec-WebSocket-Protocol' response header)"_sl)); } - - const auto& compats = repl::kCompatProtocols; - - string acceptedProtocol; - stringstream s(headers["Sec-WebSocket-Protocol"].asString()); - string protocol; - while ( getline(s, protocol, ',') ) { - auto i = std::find(compats.begin(), compats.end(), protocol); - if ( i != compats.end() ) { - acceptedProtocol = protocol; - break; - } - } - if ( _delegate ) _delegate->replicatorGotHTTPResponse(this, status, headers); if ( slice x_corr = headers.get("X-Correlation-Id"_sl); x_corr ) { _correlationID = x_corr; @@ -689,7 +692,8 @@ namespace litecore::repl { bool Replicator::getLocalCheckpoint(bool reset, CollectionIndex coll) { SubReplicator& sub = _subRepls[coll]; try { - if ( sub.checkpointer->read(_db->useLocked(), reset) ) { + auto db = _db->useWriteable(); + if ( sub.checkpointer->read(db, reset) ) { auto remote = sub.checkpointer->remoteMinSequence(); cLogInfo(coll, "Read local checkpoint '%.*s': %.*s", SPLAT(sub.checkpointer->initialCheckpointID()), SPLAT(sub.checkpointer->checkpointJSON())); @@ -701,7 +705,7 @@ namespace litecore::repl { // If pulling into an empty db with no checkpoint, it's safe to skip deleted // revisions as an optimization. if ( _options->pull(coll) > kC4Passive && sub.puller - && _db->useCollection(sub.collection)->getLastSequence() == 0_seq ) + && _db->useCollection(sub.collectionSpec)->getLastSequence() == 0_seq ) sub.puller->setSkipDeleted(); } return true; @@ -959,10 +963,9 @@ namespace litecore::repl { SPLAT(sub.remoteCheckpointRevID)); try { - _db->useLocked([&](C4Database* db) { - _db->markRevsSyncedNow(); - sub.checkpointer->write(db, json); - }); + auto db = _db->useWriteable(); + _db->markRevsSyncedNow(db); + sub.checkpointer->write(db, json); cLogInfo(coll, "Saved local checkpoint '%.*s': %.*s", SPLAT(sub.remoteCheckpointDocID), SPLAT(json)); } catch ( ... ) { gotError(C4Error::fromCurrentException()); } @@ -977,17 +980,14 @@ namespace litecore::repl { if ( !db ) { return false; } try { - bool attempted = false; - db->useLocked([this, spec, callback, &attempted](const Retained& db) { - for ( auto& _subRepl : _subRepls ) { - if ( _subRepl.collection->getSpec() == spec ) { - _subRepl.checkpointer->pendingDocumentIDs(db, callback); - attempted = true; - break; - } + BorrowedDatabase c4db = db->useLocked(); + for ( auto& _subRepl : _subRepls ) { + if ( _subRepl.collectionSpec == spec ) { + _subRepl.checkpointer->pendingDocumentIDs(c4db, callback); + return true; } - }); - return attempted; + } + return false; } catch ( const error& err ) { if ( error{error::Domain::LiteCore, error::LiteCoreError::NotOpen} == err ) { return false; @@ -999,19 +999,16 @@ namespace litecore::repl { optional Replicator::isDocumentPending(slice docID, C4CollectionSpec spec) { // CBL-2448 - auto db = _db; - if ( !db ) { return nullopt; } + auto dbAccess = _db; + if ( !dbAccess ) { return nullopt; } try { - return db->useLocked([this, docID, spec](const Retained& db) { - for ( auto& _subRepl : _subRepls ) { - if ( _subRepl.collection->getSpec() == spec ) { - return _subRepl.checkpointer->isDocumentPending(db, docID); - } - } - throw error(error::LiteCore, error::NotFound, - stringprintf("collection '%*s' not found", SPLAT(Options::collectionSpecToPath(spec)))); - }); + auto db = dbAccess->useLocked(); + for ( auto& _subRepl : _subRepls ) { + if ( _subRepl.collectionSpec == spec ) { return _subRepl.checkpointer->isDocumentPending(db, docID); } + } + throw error(error::LiteCore, error::NotFound, + stringprintf("collection '%*s' not found", SPLAT(Options::collectionSpecToPath(spec)))); } catch ( const error& err ) { if ( error{error::Domain::LiteCore, error::LiteCoreError::NotOpen} == err ) { return nullopt; @@ -1052,7 +1049,8 @@ namespace litecore::repl { alloc_slice body, revID; int status = 0; try { - if ( !Checkpointer::getPeerCheckpoint(_db->useLocked(), checkpointID, body, revID) ) status = 404; + BorrowedDatabase db = _db->useWriteable(); + if ( !Checkpointer::getPeerCheckpoint(db, checkpointID, body, revID) ) status = 404; } catch ( ... ) { C4Error::warnCurrentException("Replicator::handleGetCheckpoint"); status = 502; @@ -1087,8 +1085,9 @@ namespace litecore::repl { bool ok; alloc_slice newRevID; try { - ok = Checkpointer::savePeerCheckpoint(_db->useLocked(), checkpointID, request->body(), - request->property("rev"_sl), newRevID); + BorrowedDatabase db = _db->useWriteable(); + ok = Checkpointer::savePeerCheckpoint(db, checkpointID, request->body(), request->property("rev"_sl), + newRevID); } catch ( ... ) { request->respondWithError(c4ToBLIPError(C4Error::fromCurrentException())); return; @@ -1187,7 +1186,8 @@ namespace litecore::repl { alloc_slice body, revID; int status = 0; try { - if ( !Checkpointer::getPeerCheckpoint(_db->useLocked(), checkpointID, body, revID) ) { + BorrowedDatabase db = _db->useWriteable(); + if ( !Checkpointer::getPeerCheckpoint(db, checkpointID, body, revID) ) { enc.writeValue(Dict::emptyDict()); continue; } @@ -1227,17 +1227,15 @@ namespace litecore::repl { // Retained C4Collection object may become invalid if the underlying collection // is deleted. By spec, all collections must exist when replication starts, // and it is an error if any collection is deleted while in progress. - // Note: retained C4Collection* may blow up if it is used after becoming invalid, - // and this is expected. - _db->useLocked([this](Retained& db) { + _db->useLocked([this](C4Database* db) { for ( CollectionIndex i = 0; i < _options->workingCollectionCount(); ++i ) { - C4Collection* c = db->getCollection(_options->collectionSpec(i)); - if ( c == nullptr ) { + C4CollectionSpec spec = _options->collectionSpec(i); + if ( db->getCollection(spec) == nullptr ) { _subRepls.clear(); error::_throw(error::NotFound, "collection %s is not found in the database.", _options->collectionPath(i).asString().c_str()); } - _subRepls[i].collection = c; + _subRepls[i].collectionSpec = spec; } }); @@ -1250,7 +1248,7 @@ namespace litecore::repl { for ( CollectionIndex i = 0; i < _options->workingCollectionCount(); ++i ) { SubReplicator& sub = _subRepls[i]; if ( _options->push(i) != kC4Disabled ) { - sub.checkpointer = std::make_unique(_options, _remoteURL, sub.collection); + sub.checkpointer = std::make_unique(_options, _remoteURL, sub.collectionSpec); sub.pusher = new Pusher(this, *sub.checkpointer, i); sub.pushStatus = Worker::Status(kC4Busy); isPushBusy = true; @@ -1261,7 +1259,7 @@ namespace litecore::repl { sub.puller = new Puller(this, i); sub.pullStatus = Worker::Status(kC4Busy); if ( sub.checkpointer == nullptr ) { - sub.checkpointer = std::make_unique(_options, _remoteURL, sub.collection); + sub.checkpointer = std::make_unique(_options, _remoteURL, sub.collectionSpec); } isPullBusy = true; } else { diff --git a/Replicator/Replicator.hh b/Replicator/Replicator.hh index 06ecc2e11..fde7aeb36 100644 --- a/Replicator/Replicator.hh +++ b/Replicator/Replicator.hh @@ -26,7 +26,12 @@ namespace litecore::repl { class Puller; class ReplicatedRev; - static const array kCompatProtocols = {{string(blip::Connection::kWSProtocolName) + "+CBMobile_4"}}; + enum class ProtocolVersion { + v3 = 3, + v4 = 4, + }; + + string toString(ProtocolVersion); /** The top-level replicator object, which runs the BLIP connection. Pull and push operations are run by subidiary Puller and Pusher objects. @@ -57,7 +62,8 @@ namespace litecore::repl { using DocumentsEnded = std::vector>; - static std::string ProtocolName(); + /// A list of WebSocket subprotocol names supported by a Replicator with the given Options. + static std::vector compatibleProtocols(C4DatabaseFlags, Options::Mode pushMode, Options::Mode pullMode); /** Replicator delegate; receives progress & error notifications. */ class Delegate { @@ -109,9 +115,9 @@ namespace litecore::repl { slice remoteURL() const { return _remoteURL; } - C4Collection* collection(CollectionIndex i) const { + C4CollectionSpec collectionSpec(CollectionIndex i) const { Assert(i < _subRepls.size()); - return _subRepls[i].collection; + return _subRepls[i].collectionSpec; } protected: @@ -214,7 +220,8 @@ namespace litecore::repl { alloc_slice checkpointJSONToSave; // JSON waiting to be saved to the checkpts alloc_slice remoteCheckpointDocID; // Checkpoint docID to use with peer alloc_slice remoteCheckpointRevID; // Latest revID of remote checkpoint - Retained collection; + C4CollectionSpec collectionSpec; // Collection being replicated + alloc_slice collectionName, collectionScope; }; using ReplicatedRevBatcher = actor::ActorBatcher; diff --git a/Replicator/RevFinder.cc b/Replicator/RevFinder.cc index aadb8a56d..799183f56 100644 --- a/Replicator/RevFinder.cc +++ b/Replicator/RevFinder.cc @@ -213,11 +213,11 @@ namespace litecore::repl { // In SG 2.x "deletion" is a boolean flag, 0=normal, 1=deleted. // SG 3.x adds 2=revoked, 3=revoked+deleted, 4=removal (from channel) // If the removal flag is accompanyied by the deleted flag, we don't purge, c.f. above remark. - auto mode = (deletion < 4) ? RevocationMode::kRevokedAccess : RevocationMode::kRemovedFromChannel; - auto collSpec = getCollection()->getSpec(); - logInfo("SG revoked access to rev \"%.*s.%.*s.%.*s/%.*s\" with deletion %lld", SPLAT(collSpec.scope), - SPLAT(collSpec.name), SPLAT(docID), SPLAT(revID), deletion); - revoked.emplace_back(new RevToInsert(docID, revID, mode, collSpec, + auto mode = (deletion < 4) ? RevocationMode::kRevokedAccess : RevocationMode::kRemovedFromChannel; + logInfo("SG revoked access to rev \"%.*s.%.*s.%.*s/%.*s\" with deletion %lld", + SPLAT(collectionSpec().scope), SPLAT(collectionSpec().name), SPLAT(docID), SPLAT(revID), + deletion); + revoked.emplace_back(new RevToInsert(docID, revID, mode, collectionSpec(), _options->collectionCallbackContext(collectionIndex()))); sequences.push_back({RemoteSequence(change[0]), 0}); } @@ -230,8 +230,7 @@ namespace litecore::repl { } // Ask the database to look up the ancestors: - auto collection = getCollection(); - vector ancestors = _db->useCollection(collection) + vector ancestors = _db->useCollection(collectionSpec()) ->findDocAncestors(docIDs, revIDs, kMaxPossibleAncestors, !_options->disableDeltaSupport(), // requireBodies _db->remoteDBID()); @@ -274,7 +273,7 @@ namespace litecore::repl { // remote server, so I better make it so: logDebug(" - Already have '%.*s' %.*s but need to mark it as remote ancestor", SPLAT(docID), SPLAT(revID)); - _db->setDocRemoteAncestor(getCollection(), docID, revID); + _db->setDocRemoteAncestor(collectionSpec(), docID, revID); if ( !passive() && !_db->usingVersionVectors() ) { auto repl = replicatorIfAny(); if ( repl ) { @@ -344,7 +343,7 @@ namespace litecore::repl { // Get the local doc's current revID/vector and flags: outCurrentRevID = nullslice; try { - if ( Retained doc = _db->getDoc(getCollection(), docID, kDocGetMetadata); doc ) { + if ( Retained doc = _db->getDoc(collectionSpec(), docID, kDocGetMetadata); doc ) { flags = doc->flags(); outCurrentRevID = doc->getSelectedRevIDGlobalForm(); } diff --git a/Replicator/Worker.cc b/Replicator/Worker.cc index ece17bf0a..ae589bdf6 100644 --- a/Replicator/Worker.cc +++ b/Replicator/Worker.cc @@ -95,6 +95,7 @@ namespace litecore::repl { , _loggingID(parent ? parent->replicator()->loggingName() : connection->name()) , _connection(connection) , _status{(connection->state() >= Connection::kConnected) ? kC4Idle : kC4Connecting} + , _collectionSpec(coll != kNotCollectionIndex ? replicator()->collectionSpec(coll) : C4CollectionSpec{}) , _collectionIndex(coll) { static std::once_flag f_once; std::call_once(f_once, [] { @@ -322,12 +323,6 @@ namespace litecore::repl { return std::make_pair(collIn, err); } - const C4Collection* Worker::getCollection() const { - Assert(collectionIndex() != kNotCollectionIndex); - auto* nonConstThis = const_cast(this); - return nonConstThis->replicator()->collection(collectionIndex()); - } - void Worker::addLoggingKeyValuePairs(std::stringstream& output) const { actor::Actor::addLoggingKeyValuePairs(output); if ( auto collIdx = collectionIndex(); collIdx != kNotCollectionIndex ) { diff --git a/Replicator/Worker.hh b/Replicator/Worker.hh index e03812f59..336f6a3be 100644 --- a/Replicator/Worker.hh +++ b/Replicator/Worker.hh @@ -84,6 +84,11 @@ namespace litecore::repl { C4ReplicatorProgressLevel progressNotificationLevel() const { return _options->progressLevel; } + C4CollectionSpec collectionSpec() const { + DebugAssert(_collectionIndex != kNotCollectionIndex); + return _collectionSpec; + } + CollectionIndex collectionIndex() const { return _collectionIndex; } /// My current status. @@ -237,10 +242,6 @@ namespace litecore::repl { // 'errorSlice' describes the nature of the violation. std::pair checkCollectionOfMsg(const blip::MessageIn& msg) const; - C4Collection* getCollection() { return const_cast(((const Worker*)this)->getCollection()); } - - const C4Collection* getCollection() const; - void addLoggingKeyValuePairs(std::stringstream& output) const override; RetainedConst _options; // The replicator options @@ -253,6 +254,7 @@ namespace litecore::repl { int _pendingResponseCount{0}; // # of responses I'm awaiting Status _status{kC4Idle}; // My status bool _statusChanged{false}; // Status changed during this event + C4CollectionSpec const _collectionSpec; const CollectionIndex _collectionIndex; static std::unordered_set _formatCache; // Store collection format strings for LogEncoders benefit static std::shared_mutex _formatMutex; // Ensure thread-safety for cache insert diff --git a/Replicator/c4IncomingReplicator.hh b/Replicator/c4IncomingReplicator.hh index 5925b5c7a..389d05192 100644 --- a/Replicator/c4IncomingReplicator.hh +++ b/Replicator/c4IncomingReplicator.hh @@ -22,8 +22,8 @@ namespace litecore { /** A passive replicator handling an incoming WebSocket connection, for P2P. */ class C4IncomingReplicator final : public C4ReplicatorImpl { public: - C4IncomingReplicator(C4Database* db NONNULL, const C4ReplicatorParameters& params, - WebSocket* openSocket NONNULL, slice logPrefix) + C4IncomingReplicator(DatabaseOrPool db, const C4ReplicatorParameters& params, WebSocket* openSocket NONNULL, + slice logPrefix = {}) : C4ReplicatorImpl(db, params), _openSocket(openSocket) { std::string logName = "C4IncomingRepl"; if ( !logPrefix.empty() ) logName = logPrefix.asString() + "/" + logName; @@ -34,10 +34,8 @@ namespace litecore { void createReplicator() override { Assert(_openSocket); - - auto dbOpenedAgain = _database->openAgain(); - _c4db_setDatabaseTag(dbOpenedAgain, DatabaseTag_C4IncomingReplicator); - _replicator = new Replicator(dbOpenedAgain.get(), _openSocket, *this, _options); + _replicator = new Replicator(makeDBAccess(_database, DatabaseTag_C4IncomingReplicator), _openSocket, *this, + _options); // Yes this line is disgusting, but the memory addresses that the logger logs // are not the _actual_ addresses of the object, but rather the pointer to diff --git a/Replicator/c4RemoteReplicator.hh b/Replicator/c4RemoteReplicator.hh index 583c41ad0..d1d405abb 100644 --- a/Replicator/c4RemoteReplicator.hh +++ b/Replicator/c4RemoteReplicator.hh @@ -17,6 +17,7 @@ #include "c4ReplicatorImpl.hh" #include "c4Socket+Internal.hh" #include "Address.hh" +#include "DBAccess.hh" #include "StringUtil.hh" #include "Timer.hh" #include @@ -38,9 +39,9 @@ namespace litecore { // This can be overridden by setting the option `kC4ReplicatorOptionMaxRetryInterval`. static constexpr unsigned kDefaultMaxRetryDelay = 5 * 60; - C4RemoteReplicator(C4Database* db NONNULL, const C4ReplicatorParameters& params, const C4Address& serverAddress, + C4RemoteReplicator(DatabaseOrPool db, const C4ReplicatorParameters& params, const C4Address& serverAddress, C4String remoteDatabaseName, slice logPrefix) - : C4ReplicatorImpl(db, params) + : C4ReplicatorImpl(std::move(db), params) , _url(effectiveURL(serverAddress, remoteDatabaseName)) , _retryTimer([this] { retry(false); }) { std::string logName = "C4RemoteRepl"; @@ -110,10 +111,15 @@ namespace litecore { alloc_slice URL() const noexcept override { return _url; } void createReplicator() override { - auto dbOpenedAgain = _database->openAgain(); - _c4db_setDatabaseTag(dbOpenedAgain, DatabaseTag_C4RemoteReplicator); - auto dbAccess = - make_shared(dbOpenedAgain, _options->properties["disable_blob_support"_sl].asBool()); + bool disableBlobs = _options->properties["disable_blob_support"_sl].asBool(); + std::shared_ptr dbAccess; + if ( _database.index() == 0 ) { + auto dbOpenedAgain = std::get<0>(_database)->openAgain(); + _c4db_setDatabaseTag(dbOpenedAgain, DatabaseTag_C4RemoteReplicator); + dbAccess = make_shared(dbOpenedAgain, disableBlobs); + } else { + dbAccess = std::make_shared(std::get<1>(_database), disableBlobs); + } auto webSocket = CreateWebSocket(_url, socketOptions(), dbAccess, _socketFactory); _replicator = new Replicator(dbAccess, webSocket, *this, _options); @@ -208,8 +214,19 @@ namespace litecore { // Options to pass to the C4Socket alloc_slice socketOptions() const { + // Get the database flags and the push/pull modes: + auto cfg = std::visit([](auto db) { return db->getConfiguration(); }, _database); + C4ReplicatorMode pushMode = kC4Disabled, pullMode = kC4Disabled; + for ( CollectionIndex i = 0; i < _options->collectionCount(); ++i ) { + pushMode = std::max(pushMode, _options->push(i)); + pullMode = std::max(pullMode, _options->pull(i)); + } + // From those, determine the compatible WS protocols: + auto protocols = Replicator::compatibleProtocols(cfg.flags, pushMode, pullMode); + + // Construct new Options including the protocols: Replicator::Options opts(kC4Disabled, kC4Disabled, _options->properties); - opts.setProperty(kC4SocketOptionWSProtocols, Replicator::ProtocolName().c_str()); + opts.setProperty(kC4SocketOptionWSProtocols, join(protocols, ",").c_str()); return opts.properties.data(); } diff --git a/Replicator/c4Replicator+Pool.hh b/Replicator/c4Replicator+Pool.hh new file mode 100644 index 000000000..678b6f75d --- /dev/null +++ b/Replicator/c4Replicator+Pool.hh @@ -0,0 +1,40 @@ +// +// c4Replicator+Pool.hh +// +// Copyright 2024-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + + +#pragma once +#include "c4Replicator.hh" + +C4_ASSUME_NONNULL_BEGIN + +namespace litecore { + class DatabasePool; + + namespace websocket { + class WebSocket; + } +} // namespace litecore + +// C4Replicator factory functions that take a DatabasePool instead of a C4Database. + +fleece::Retained NewReplicator(litecore::DatabasePool* dbPool, C4Address serverAddress, + fleece::slice remoteDatabaseName, const C4ReplicatorParameters& params, + fleece::slice logPrefix = {}); +fleece::Retained NewLocalReplicator(litecore::DatabasePool* dbPool, litecore::DatabasePool* otherLocalDB, + const C4ReplicatorParameters& params, fleece::slice logPrefix = {}); +fleece::Retained NewIncomingReplicator(litecore::DatabasePool* dbPool, + litecore::websocket::WebSocket* openSocket, + const C4ReplicatorParameters& params, + fleece::slice logPrefix = {}); + + +C4_ASSUME_NONNULL_END diff --git a/Replicator/c4Replicator.cc b/Replicator/c4Replicator.cc index f2b846ab9..76fb61dc8 100644 --- a/Replicator/c4Replicator.cc +++ b/Replicator/c4Replicator.cc @@ -11,6 +11,7 @@ // #include "c4Replicator.hh" +#include "c4Replicator+Pool.hh" #include "c4RemoteReplicator.hh" #include "c4IncomingReplicator.hh" #include "c4Database.hh" @@ -26,14 +27,15 @@ using namespace litecore::net; // work around missing 'using' in c4LocalReplica using namespace std; using namespace litecore; +using DatabaseOrPool = C4ReplicatorImpl::DatabaseOrPool; -#pragma mark - C++ API: +#pragma mark - C4DATABASE METHODS: -// All instances are subclasses of C4BaseReplicator. +// All instances are subclasses of C4ReplicatorImpl. static C4ReplicatorImpl* asInternal(const C4Replicator* repl) { return (C4ReplicatorImpl*)repl; } -Retained C4Database::newReplicator(C4Address serverAddress, slice remoteDatabaseName, - const C4ReplicatorParameters& params, slice logPrefix) { +static Retained newRemoteReplicator(DatabaseOrPool db, C4Address serverAddress, slice remoteDatabaseName, + const C4ReplicatorParameters& params, slice logPrefix) { if ( !params.socketFactory ) { C4Replicator::validateRemote(serverAddress, remoteDatabaseName); if ( serverAddress.port == 4985 && serverAddress.hostname != "localhost"_sl ) { @@ -42,23 +44,42 @@ Retained C4Database::newReplicator(C4Address serverAddress, slice "unreachable, but if opened, it would give anyone unlimited privileges."); } } - return new C4RemoteReplicator(this, params, serverAddress, remoteDatabaseName, logPrefix); + return new C4RemoteReplicator(std::move(db), params, serverAddress, remoteDatabaseName, logPrefix); +} + +Retained C4Database::newReplicator(C4Address serverAddress, slice remoteDatabaseName, + const C4ReplicatorParameters& params, slice logPrefix) { + return newRemoteReplicator(this, serverAddress, remoteDatabaseName, params, logPrefix); +} + +Retained NewReplicator(DatabasePool* dbPool, C4Address serverAddress, slice remoteDatabaseName, + const C4ReplicatorParameters& params, slice logPrefix) { + return newRemoteReplicator(dbPool, serverAddress, remoteDatabaseName, params, logPrefix); } #ifdef COUCHBASE_ENTERPRISE -Retained C4Database::newLocalReplicator(C4Database* otherLocalDB, const C4ReplicatorParameters& params, - slice logPrefix) { +static Retained _newLocalReplicator(DatabaseOrPool db, DatabaseOrPool otherDB, + const C4ReplicatorParameters& params, slice logPrefix) { std::for_each(params.collections, params.collections + params.collectionCount, [](const C4ReplicationCollection& coll) { AssertParam(coll.push != kC4Disabled || coll.pull != kC4Disabled, "Either push or pull must be enabled"); }); - AssertParam(otherLocalDB != this, "Can't replicate a database to itself"); - return new C4LocalReplicator(this, params, otherLocalDB, logPrefix); + AssertParam(db != otherDB, "Can't replicate a database to itself"); + return new C4LocalReplicator(db, params, otherDB, logPrefix); +} + +Retained C4Database::newLocalReplicator(C4Database* otherLocalDB, const C4ReplicatorParameters& params, + slice logPrefix) { + return _newLocalReplicator(this, otherLocalDB, params, logPrefix); } -#endif +Retained NewLocalReplicator(DatabasePool* dbPool, DatabasePool* otherLocalDB, + const C4ReplicatorParameters& params, slice logPrefix) { + return _newLocalReplicator(dbPool, otherLocalDB, params, logPrefix); +} +#endif Retained C4Database::newIncomingReplicator(WebSocket* openSocket, const C4ReplicatorParameters& params, slice logPrefix) { @@ -70,6 +91,13 @@ Retained C4Database::newIncomingReplicator(C4Socket* openSocket, c return newIncomingReplicator(WebSocketFrom(openSocket), params, logPrefix); } +Retained NewIncomingReplicator(DatabasePool* dbPool, WebSocket* openSocket, + const C4ReplicatorParameters& params, fleece::slice logPrefix) { + return new C4IncomingReplicator(dbPool, params, openSocket, logPrefix); +} + +#pragma mark - C4REPLICATOR METHODS: + bool C4Replicator::retry() const { return asInternal(this)->retry(true); } void C4Replicator::setOptions(slice optionsDictFleece) { @@ -89,8 +117,7 @@ C4Cert* C4Replicator::getPeerTLSCertificate() const { return asInternal(this)->g #endif -CBL_CORE_API const char* const kC4ReplicatorActivityLevelNames[6] = {"stopped", "offline", "connecting", - "idle", "busy", "stopping"}; +const char* const kC4ReplicatorActivityLevelNames[6] = {"stopped", "offline", "connecting", "idle", "busy", "stopping"}; static bool isValidScheme(slice scheme) { return scheme.size > 0 && isalpha(scheme[0]); } diff --git a/Replicator/c4ReplicatorImpl.cc b/Replicator/c4ReplicatorImpl.cc new file mode 100644 index 000000000..62b525fbd --- /dev/null +++ b/Replicator/c4ReplicatorImpl.cc @@ -0,0 +1,452 @@ +// +// C4ReplicatorImpl.cc +// +// Copyright 2017-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#include "c4ReplicatorImpl.hh" +#include "c4DocEnumeratorTypes.h" +#include "c4Certificate.hh" +#include "DatabasePool.hh" +#include "DBAccess.hh" +#include "Checkpointer.hh" +#include "Headers.hh" +#include "Error.hh" +#include +#include + +namespace litecore { + + C4ReplicatorImpl::C4ReplicatorImpl(DatabaseOrPool db, const C4ReplicatorParameters& params) + : Logging(SyncLog) + , _database(std::move(db)) + , _options(new Options(params)) + , _loggingName("C4Repl") + , _onStatusChanged(params.onStatusChanged) + , _onDocumentsEnded(params.onDocumentsEnded) + , _onBlobProgress(params.onBlobProgress) { + _status.flags |= kC4HostReachable; + _options->verify(); + } + + C4ReplicatorImpl::C4ReplicatorImpl(C4Database* db, const C4ReplicatorParameters& params) + : C4ReplicatorImpl(DatabaseOrPool(db), params) {} + + C4ReplicatorImpl::C4ReplicatorImpl(DatabasePool* pool, const C4ReplicatorParameters& params) + : C4ReplicatorImpl(DatabaseOrPool(pool), params) {} + + C4ReplicatorImpl::~C4ReplicatorImpl() { + logInfo("Freeing C4BaseReplicator"); + // Tear down the Replicator instance -- this is important in the case where it was + // never started, because otherwise there will be a bunch of ref cycles that cause many + // objects (including C4Databases) to be leaked. [CBL-524] + if ( _replicator ) _replicator->terminate(); + } + + void C4ReplicatorImpl::start(bool reset) noexcept { + LOCK(_mutex); + if ( _status.level == kC4Stopping ) { + logInfo("Rapid call to start() (stop() is not finished yet), scheduling a restart after stop() is " + "done..."); + _cancelStop = true; + return; + } + + if ( !_replicator ) { + if ( !_start(reset) ) { + UNLOCK(); + // error set as part of _start, + // but we cannot notify until outside of the lock + notifyStateChanged(); + } + } + } + + void C4ReplicatorImpl::setSuspended(bool suspended) noexcept { + LOCK(_mutex); + if ( _status.level == kC4Stopped ) { + // Suspending a stopped replicator? Get outta here... + logInfo("Ignoring a suspend call on a stopped replicator..."); + return; + } + + if ( _status.level == kC4Stopping && !statusFlag(kC4Suspended) ) { + // CBL-722: Stop was already called or Replicator is stopped, + // making suspending meaningless (stop() should override any + // suspending or unsuspending) + logInfo("Ignoring a suspend call on a stopping replicator..."); + return; + } + + if ( _status.level == kC4Stopping ) { + // CBL-729: At this point, the suspended state has changed from a previous + // call that caused a suspension to start. Register to restart later + // (or cancel the later restart) and move on + _cancelStop = !suspended; + if ( _cancelStop ) { + logInfo("Request to unsuspend, but Replicator is already suspending. Will restart after " + "suspending process is completed."); + } else { + logInfo("Replicator suspension process being spammed (request to suspend followed by at least one " + "request to unsuspend and then suspend again), attempting to cancel restart."); + } + return; + } + + if ( !setStatusFlag(kC4Suspended, suspended) ) { + logVerbose("Ignoring redundant suspend call..."); + // Duplicate call, ignore... + return; + } + + logInfo("%s", (suspended ? "Suspended" : "Un-suspended")); + if ( suspended ) { + _activeWhenSuspended = (_status.level >= kC4Connecting); + if ( _activeWhenSuspended ) _suspend(); + } else { + if ( _status.level == kC4Offline && _activeWhenSuspended ) { + if ( !_unsuspend() ) { + // error set as part of _unsuspend, + // but we cannot notify until outside of the lock + UNLOCK(); + notifyStateChanged(); + } + } + } + } + + alloc_slice C4ReplicatorImpl::getResponseHeaders() const noexcept { + LOCK(_mutex); + return _responseHeaders; + } + + C4ReplicatorStatus C4ReplicatorImpl::getStatus() const noexcept { + LOCK(_mutex); + switch ( _status.level ) { + // CBL-1513: Any new approved statuses must be added to this list, + // or they will be forced to busy in order to prevent internal statuses + // from leaking + case kC4Busy: + case kC4Connecting: + case kC4Idle: + case kC4Offline: + case kC4Stopped: + return _status; + + default: + return {kC4Busy, _status.progress, _status.error, _status.flags}; + } + } + + void C4ReplicatorImpl::stop() noexcept { + LOCK(_mutex); + _cancelStop = false; + setStatusFlag(kC4Suspended, false); + if ( _status.level == kC4Stopping ) { + // Already stopping, this call is spammy so ignore it + logVerbose("Duplicate call to stop()..."); + return; + } + + if ( _replicator ) { + _status.level = kC4Stopping; + _replicator->stop(); + } else if ( _status.level != kC4Stopped ) { + _status.level = kC4Stopped; + _status.progress = {}; + UNLOCK(); + notifyStateChanged(); + _selfRetain = nullptr; // balances retain in `_start` -- may destruct me! + } + } + + void C4ReplicatorImpl::setProperties(AllocedDict properties) { + LOCK(_mutex); + _options->properties = std::move(properties); + } + + void C4ReplicatorImpl::stopCallbacks() noexcept { + LOCK(_mutex); + _onStatusChanged = nullptr; + _onDocumentsEnded = nullptr; + _onBlobProgress = nullptr; + } + + void C4ReplicatorImpl::setProgressLevel(C4ReplicatorProgressLevel level) noexcept { + if ( _options->setProgressLevel(level) ) { logVerbose("Set progress notification level to %d", level); } + } + +#ifdef COUCHBASE_ENTERPRISE + C4Cert* C4ReplicatorImpl::getPeerTLSCertificate() const { + LOCK(_mutex); + if ( !_peerTLSCertificate && _peerTLSCertificateData ) { + _peerTLSCertificate = C4Cert::fromData(_peerTLSCertificateData); + _peerTLSCertificateData = nullptr; + } + return _peerTLSCertificate; + } +#endif + + BorrowedDatabase C4ReplicatorImpl::borrow(DatabaseOrPool const& dbp) { + return std::visit([](auto d) { return BorrowedDatabase(d); }, dbp); + } + + bool C4ReplicatorImpl::continuous(unsigned collectionIndex) const noexcept { + return _options->push(collectionIndex) == kC4Continuous || _options->pull(collectionIndex) == kC4Continuous; + } + + bool C4ReplicatorImpl::setStatusFlag(C4ReplicatorStatusFlags flag, bool on) noexcept { + auto flags = _status.flags; + if ( on ) flags |= flag; + else + flags &= ~flag; + if ( flags == _status.flags ) return false; + _status.flags = flags; + return true; + } + + void C4ReplicatorImpl::updateStatusFromReplicator(C4ReplicatorStatus status) noexcept { + if ( _status.level == kC4Stopping && status.level != kC4Stopped ) { + // From Stopping it can only go to Stopped + return; + } + // The Replicator doesn't use the flags, so don't copy them: + auto flags = _status.flags; + _status = status; + _status.flags = flags; + } + + unsigned C4ReplicatorImpl::getIntProperty(slice key, unsigned defaultValue) const noexcept { + if ( auto val = _options->properties[key]; val.type() == kFLNumber ) { + // CBL-3872: Large unsigned values (higher than max int64) will become + // negative, and thus get clamped to zero with the old logic, so add + // special handling for an unsigned fleece value + if ( val.isUnsigned() ) { return unsigned(std::min(val.asUnsigned(), uint64_t(UINT_MAX))); } + + return unsigned(std::max(int64_t(0), std::min(int64_t(UINT_MAX), val.asInt()))); + } + + return defaultValue; + } + + std::shared_ptr C4ReplicatorImpl::makeDBAccess(DatabaseOrPool const& dbp, C4DatabaseTag tag) const { + bool disableBlobs = _options->properties["disable_blob_support"_sl].asBool(); + if ( std::holds_alternative>(dbp) ) { + auto dbOpenedAgain = std::get>(dbp)->openAgain(); + _c4db_setDatabaseTag(dbOpenedAgain, tag); + return make_shared(dbOpenedAgain, disableBlobs); + } else { + return std::make_shared(std::get>(dbp), disableBlobs); + } + } + + bool C4ReplicatorImpl::_start(bool reset) noexcept { + if ( !_replicator ) { + try { + createReplicator(); + } catch ( exception& x ) { + _status.error = C4Error::fromException(x); + _replicator = nullptr; + return false; + } + } + + setStatusFlag(kC4Suspended, false); + logInfo("Starting Replicator %s with config: {%s} and endpoint: %.*s", _replicator->loggingName().c_str(), + std::string(*_options).c_str(), SPLAT(_replicator->remoteURL())); + _selfRetain = this; // keep myself alive till Replicator stops + updateStatusFromReplicator(_replicator->status()); + _responseHeaders = nullptr; + _replicator->start(reset); + return true; + } + + void C4ReplicatorImpl::_suspend() noexcept { + // called with _mutex locked + if ( _replicator ) { + _status.level = kC4Stopping; + _replicator->stop(); + } + } + + bool C4ReplicatorImpl::_unsuspend() noexcept { + // called with _mutex locked + return _start(false); + } + + void C4ReplicatorImpl::replicatorGotHTTPResponse(Replicator* repl, int status, const websocket::Headers& headers) { + LOCK(_mutex); + if ( repl == _replicator ) { + Assert(!_responseHeaders); + _responseHeaders = headers.encode(); + } + } + + void C4ReplicatorImpl::replicatorGotTLSCertificate(slice certData) { +#ifdef COUCHBASE_ENTERPRISE + LOCK(_mutex); + _peerTLSCertificateData = certData; + _peerTLSCertificate = nullptr; +#endif + } + + void C4ReplicatorImpl::replicatorStatusChanged(Replicator* repl, const Replicator::Status& newStatus) { + Retained selfRetain = this; // Keep myself alive till this method returns + + bool stopped, resume = false; + { + LOCK(_mutex); + if ( repl != _replicator ) return; + auto oldLevel = _status.level; + updateStatusFromReplicator((C4ReplicatorStatus)newStatus); + if ( _status.level > kC4Connecting && oldLevel <= kC4Connecting ) handleConnected(); + if ( _status.level == kC4Stopped ) { + _replicator->terminate(); + _replicator = nullptr; + if ( statusFlag(kC4Suspended) ) { + // If suspended, go to Offline state when Replicator stops + _status.level = kC4Offline; + } else if ( oldLevel != kC4Stopping ) { + // CBL-1054, only do this if a request to stop is not present, as it should + // override the offline handling + handleStopped(); // NOTE: handleStopped may change _status + } + + resume = _cancelStop; + _cancelStop = false; + } + stopped = (_status.level == kC4Stopped); + } + + notifyStateChanged(); + + if ( stopped ) _selfRetain = nullptr; // balances retain in `_start` + + if ( resume ) { start(); } + + // On return from this method, if I stopped I may be deleted (due to clearing _selfRetain) + } + + void C4ReplicatorImpl::replicatorDocumentsEnded(Replicator* repl, + const std::vector>& revs) { + if ( repl != _replicator ) return; + + auto nRevs = revs.size(); + std::vector docsEnded; + docsEnded.reserve(nRevs); + for ( int pushing = 0; pushing <= 1; ++pushing ) { + docsEnded.clear(); + for ( const auto& rev : revs ) { + if ( (rev->dir() == Dir::kPushing) == pushing ) docsEnded.push_back(rev->asDocumentEnded()); + } + if ( !docsEnded.empty() ) { + auto onDocsEnded = _onDocumentsEnded.load(); + if ( onDocsEnded ) + onDocsEnded(this, pushing, docsEnded.size(), docsEnded.data(), _options->callbackContext); + } + } + } + + void C4ReplicatorImpl::replicatorBlobProgress(Replicator* repl, const Replicator::BlobProgress& p) { + if ( repl != _replicator ) return; + auto onBlob = _onBlobProgress.load(); + if ( onBlob ) + onBlob(this, (p.dir == Dir::kPushing), p.collSpec, p.docID, p.docProperty, p.key, p.bytesCompleted, + p.bytesTotal, p.error, _options->callbackContext); + } + + void C4ReplicatorImpl::notifyStateChanged() noexcept { + C4ReplicatorStatus status = this->getStatus(); + + if ( willLog() ) { + double progress = 0.0; + if ( status.progress.unitsTotal > 0 ) + progress = 100.0 * double(status.progress.unitsCompleted) / double(status.progress.unitsTotal); + if ( status.error.code ) { + logError("State: %-s, progress=%.2f%%, error=%s", kC4ReplicatorActivityLevelNames[status.level], + progress, status.error.description().c_str()); + } else { + logInfo("State: %-s, progress=%.2f%%", kC4ReplicatorActivityLevelNames[status.level], progress); + } + } + + if ( !(status.error.code && status.level > kC4Offline) ) { + auto onStatusChanged = _onStatusChanged.load(); + if ( onStatusChanged && status.level != kC4Stopping /* Don't notify about internal state */ ) + onStatusChanged(this, status, _options->callbackContext); + } + } + + class C4ReplicatorImpl::PendingDocuments { + public: + PendingDocuments(const C4ReplicatorImpl* repl, C4CollectionSpec spec) : collectionSpec(spec) { + // Lock the replicator and copy the necessary state now, so I don't have to lock while + // calling pendingDocumentIDs (which might call into the app's validation function.) + LOCK(repl->_mutex); + replicator = repl->_replicator; + database = repl->_database; + + // CBL-2448: Also make my own checkpointer and database in case a call comes in + // after Replicator::terminate() is called. The fix includes the replicator + // pending document ID function now returning a boolean success, isDocumentPending returning + // an optional and if pendingDocumentIDs returns false or isDocumentPending + // returns nullopt, the checkpointer is fallen back on + // The collection must be included in the replicator's config options. + auto it = repl->_options->collectionSpecToIndex().find(collectionSpec); + if ( it == repl->_options->collectionSpecToIndex().end() + || it->second >= repl->_options->workingCollectionCount() ) { + error::_throw(error::NotOpen, "collection not in the Replicator's config"); + } + + checkpointer.emplace(repl->_options, repl->URL(), collectionSpec); + } + + alloc_slice pendingDocumentIDs() { + Encoder enc; + enc.beginArray(); + bool any = false; + auto callback = [&](const C4DocumentInfo& info) { + enc.writeString(info.docID); + any = true; + }; + + if ( !replicator || !replicator->pendingDocumentIDs(collectionSpec, callback) ) { + auto bdb = borrow(database); + checkpointer->pendingDocumentIDs(bdb, callback); + } + + if ( !any ) return {}; + enc.endArray(); + return enc.finish(); + } + + bool isDocumentPending(C4Slice docID) { + if ( replicator ) { + auto result = replicator->isDocumentPending(docID, collectionSpec); + if ( result.has_value() ) { return *result; } + } + auto bdb = borrow(database); + return checkpointer->isDocumentPending(bdb, docID); + } + + private: + Retained replicator; + optional checkpointer; // initialized in the constructor + DatabaseOrPool database; + C4CollectionSpec collectionSpec; + }; + + bool C4ReplicatorImpl::isDocumentPending(C4Slice docID, C4CollectionSpec spec) const { + return PendingDocuments(this, spec).isDocumentPending(docID); + } + + alloc_slice C4ReplicatorImpl::pendingDocumentIDs(C4CollectionSpec spec) const { + return PendingDocuments(this, spec).pendingDocumentIDs(); + } +} // namespace litecore diff --git a/Replicator/c4ReplicatorImpl.hh b/Replicator/c4ReplicatorImpl.hh index 151aa563c..5c499e0f8 100644 --- a/Replicator/c4ReplicatorImpl.hh +++ b/Replicator/c4ReplicatorImpl.hh @@ -14,23 +14,19 @@ #include "c4Replicator.hh" #include "c4Database.hh" -#include "c4DocEnumeratorTypes.h" -#include "c4Certificate.hh" #include "c4Internal.hh" +#include "c4Private.h" #include "Replicator.hh" -#include "Checkpointer.hh" -#include "Headers.hh" -#include "Error.hh" #include "Logging.hh" #include "fleece/Fleece.hh" #include "fleece/Expert.hh" // for AllocedDict -#include #include -#include -#include +#include #include namespace litecore { + class DatabasePool; + class BorrowedDatabase; using namespace fleece; using namespace litecore; @@ -45,231 +41,66 @@ namespace litecore { // Bump this when incompatible changes are made to API or implementation. // Subclass c4LocalReplicator is in the couchbase-lite-core-EE repo, which doesn not have a // submodule relationship to this one, so it's possible for it to get out of sync. - static constexpr int API_VERSION = 4; - - void start(bool reset = false) noexcept override { - LOCK(_mutex); - if ( _status.level == kC4Stopping ) { - logInfo("Rapid call to start() (stop() is not finished yet), scheduling a restart after stop() is " - "done..."); - _cancelStop = true; - return; - } - - if ( !_replicator ) { - if ( !_start(reset) ) { - UNLOCK(); - // error set as part of _start, - // but we cannot notify until outside of the lock - notifyStateChanged(); - } - } - } + static constexpr int API_VERSION = 5; + + using DatabaseOrPool = std::variant, Retained>; + + void start(bool reset = false) noexcept override; // Retry is not supported by default. C4RemoteReplicator overrides this. virtual bool retry(bool resetCount) { C4Error::raise(LiteCoreDomain, kC4ErrorUnsupported, "Can't retry this type of replication"); } - void setSuspended(bool suspended) noexcept override { - LOCK(_mutex); - if ( _status.level == kC4Stopped ) { - // Suspending a stopped replicator? Get outta here... - logInfo("Ignoring a suspend call on a stopped replicator..."); - return; - } - - if ( _status.level == kC4Stopping && !statusFlag(kC4Suspended) ) { - // CBL-722: Stop was already called or Replicator is stopped, - // making suspending meaningless (stop() should override any - // suspending or unsuspending) - logInfo("Ignoring a suspend call on a stopping replicator..."); - return; - } - - if ( _status.level == kC4Stopping ) { - // CBL-729: At this point, the suspended state has changed from a previous - // call that caused a suspension to start. Register to restart later - // (or cancel the later restart) and move on - _cancelStop = !suspended; - if ( _cancelStop ) { - logInfo("Request to unsuspend, but Replicator is already suspending. Will restart after " - "suspending process is completed."); - } else { - logInfo("Replicator suspension process being spammed (request to suspend followed by at least one " - "request to unsuspend and then suspend again), attempting to cancel restart."); - } - return; - } - - if ( !setStatusFlag(kC4Suspended, suspended) ) { - logVerbose("Ignoring redundant suspend call..."); - // Duplicate call, ignore... - return; - } - - logInfo("%s", (suspended ? "Suspended" : "Un-suspended")); - if ( suspended ) { - _activeWhenSuspended = (_status.level >= kC4Connecting); - if ( _activeWhenSuspended ) _suspend(); - } else { - if ( _status.level == kC4Offline && _activeWhenSuspended ) { - if ( !_unsuspend() ) { - // error set as part of _unsuspend, - // but we cannot notify until outside of the lock - UNLOCK(); - notifyStateChanged(); - } - } - } - } + void setSuspended(bool suspended) noexcept override; - alloc_slice getResponseHeaders() const noexcept override { - LOCK(_mutex); - return _responseHeaders; - } + alloc_slice getResponseHeaders() const noexcept override; - C4ReplicatorStatus getStatus() const noexcept override { - LOCK(_mutex); - switch ( _status.level ) { - // CBL-1513: Any new approved statuses must be added to this list, - // or they will be forced to busy in order to prevent internal statuses - // from leaking - case kC4Busy: - case kC4Connecting: - case kC4Idle: - case kC4Offline: - case kC4Stopped: - return _status; - - default: - return {kC4Busy, _status.progress, _status.error, _status.flags}; - } - } + C4ReplicatorStatus getStatus() const noexcept override; - void stop() noexcept override { - LOCK(_mutex); - _cancelStop = false; - setStatusFlag(kC4Suspended, false); - if ( _status.level == kC4Stopping ) { - // Already stopping, this call is spammy so ignore it - logVerbose("Duplicate call to stop()..."); - return; - } - - if ( _replicator ) { - _status.level = kC4Stopping; - _replicator->stop(); - } else if ( _status.level != kC4Stopped ) { - _status.level = kC4Stopped; - _status.progress = {}; - UNLOCK(); - notifyStateChanged(); - _selfRetain = nullptr; // balances retain in `_start` -- may destruct me! - } - } + void stop() noexcept override; - virtual void setProperties(AllocedDict properties) { - LOCK(_mutex); - _options->properties = std::move(properties); - } + virtual void setProperties(AllocedDict properties); // Prevents any future client callbacks (called by `c4repl_free`.) - void stopCallbacks() noexcept override { - LOCK(_mutex); - _onStatusChanged = nullptr; - _onDocumentsEnded = nullptr; - _onBlobProgress = nullptr; - } + void stopCallbacks() noexcept override; - bool isDocumentPending(C4Slice docID, C4CollectionSpec spec) const { - return PendingDocuments(this, spec).isDocumentPending(docID); - } + bool isDocumentPending(C4Slice docID, C4CollectionSpec spec) const; - alloc_slice pendingDocumentIDs(C4CollectionSpec spec) const { - return PendingDocuments(this, spec).pendingDocumentIDs(); - } + alloc_slice pendingDocumentIDs(C4CollectionSpec spec) const; - void setProgressLevel(C4ReplicatorProgressLevel level) noexcept override { - if ( _options->setProgressLevel(level) ) { logVerbose("Set progress notification level to %d", level); } - } + void setProgressLevel(C4ReplicatorProgressLevel level) noexcept override; #ifdef COUCHBASE_ENTERPRISE - - C4Cert* getPeerTLSCertificate() const override { - LOCK(_mutex); - if ( !_peerTLSCertificate && _peerTLSCertificateData ) { - _peerTLSCertificate = C4Cert::fromData(_peerTLSCertificateData); - _peerTLSCertificateData = nullptr; - } - return _peerTLSCertificate; - } - + C4Cert* getPeerTLSCertificate() const override; #endif protected: - // base constructor - C4ReplicatorImpl(C4Database* db NONNULL, const C4ReplicatorParameters& params) - : Logging(SyncLog) - , _database(db) - , _options(new Options(params)) - , _onStatusChanged(params.onStatusChanged) - , _onDocumentsEnded(params.onDocumentsEnded) - , _onBlobProgress(params.onBlobProgress) - , _loggingName("C4Repl") { - _status.flags |= kC4HostReachable; - _options->verify(); - } + static BorrowedDatabase borrow(DatabaseOrPool const& dbp); - ~C4ReplicatorImpl() override { - logInfo("Freeing C4BaseReplicator"); - // Tear down the Replicator instance -- this is important in the case where it was - // never started, because otherwise there will be a bunch of ref cycles that cause many - // objects (including C4Databases) to be leaked. [CBL-524] - if ( _replicator ) _replicator->terminate(); - } + /// Base constructor. For `db` you can pass either a `Retained` or a + /// `Retained`. + C4ReplicatorImpl(DatabaseOrPool db, const C4ReplicatorParameters& params); + C4ReplicatorImpl(C4Database*, const C4ReplicatorParameters& params); + C4ReplicatorImpl(DatabasePool*, const C4ReplicatorParameters& params); + + ~C4ReplicatorImpl() override; std::string loggingClassName() const override { return _loggingName; } - bool continuous(unsigned collectionIndex = 0) const noexcept { - return _options->push(collectionIndex) == kC4Continuous || _options->pull(collectionIndex) == kC4Continuous; - } + void setLoggingName(const string& loggingName) { _loggingName = loggingName; } - inline bool statusFlag(C4ReplicatorStatusFlags flag) const noexcept { return (_status.flags & flag) != 0; } + bool continuous(unsigned collectionIndex = 0) const noexcept; - bool setStatusFlag(C4ReplicatorStatusFlags flag, bool on) noexcept { - auto flags = _status.flags; - if ( on ) flags |= flag; - else - flags &= ~flag; - if ( flags == _status.flags ) return false; - _status.flags = flags; - return true; - } + inline bool statusFlag(C4ReplicatorStatusFlags flag) const noexcept { return (_status.flags & flag) != 0; } - void updateStatusFromReplicator(C4ReplicatorStatus status) noexcept { - if ( _status.level == kC4Stopping && status.level != kC4Stopped ) { - // From Stopping it can only go to Stopped - return; - } - // The Replicator doesn't use the flags, so don't copy them: - auto flags = _status.flags; - _status = status; - _status.flags = flags; - } + bool setStatusFlag(C4ReplicatorStatusFlags flag, bool on) noexcept; - unsigned getIntProperty(slice key, unsigned defaultValue) const noexcept { - if ( auto val = _options->properties[key]; val.type() == kFLNumber ) { - // CBL-3872: Large unsigned values (higher than max int64) will become - // negative, and thus get clamped to zero with the old logic, so add - // special handling for an unsigned fleece value - if ( val.isUnsigned() ) { return unsigned(std::min(val.asUnsigned(), uint64_t(UINT_MAX))); } + void updateStatusFromReplicator(C4ReplicatorStatus status) noexcept; - return unsigned(std::max(int64_t(0), std::min(int64_t(UINT_MAX), val.asInt()))); - } + unsigned getIntProperty(slice key, unsigned defaultValue) const noexcept; - return defaultValue; - } + std::shared_ptr makeDBAccess(DatabaseOrPool const& dbp, C4DatabaseTag tag) const; virtual void createReplicator() = 0; @@ -278,130 +109,24 @@ namespace litecore { // Base implementation of starting the replicator. // Subclass implementation of `start` must call this (with the mutex locked). // Rather than throw exceptions, it stores errors in _status.error. - virtual bool _start(bool reset) noexcept { - if ( !_replicator ) { - try { - createReplicator(); - } catch ( exception& x ) { - _status.error = C4Error::fromException(x); - _replicator = nullptr; - return false; - } - } - - setStatusFlag(kC4Suspended, false); - logInfo("Starting Replicator %s with config: {%s} and endpoint: %.*s", _replicator->loggingName().c_str(), - std::string(*_options).c_str(), SPLAT(_replicator->remoteURL())); - _selfRetain = this; // keep myself alive till Replicator stops - updateStatusFromReplicator(_replicator->status()); - _responseHeaders = nullptr; - _replicator->start(reset); - return true; - } - - virtual void _suspend() noexcept { - // called with _mutex locked - if ( _replicator ) { - _status.level = kC4Stopping; - _replicator->stop(); - } - } - - virtual bool _unsuspend() noexcept { - // called with _mutex locked - return _start(false); - } + virtual bool _start(bool reset) noexcept; + virtual void _suspend() noexcept; + virtual bool _unsuspend() noexcept; // ---- ReplicatorDelegate API: - // Replicator::Delegate method, notifying that the WebSocket has connected. - void replicatorGotHTTPResponse(Replicator* repl, int status, const websocket::Headers& headers) override { - LOCK(_mutex); - if ( repl == _replicator ) { - Assert(!_responseHeaders); - _responseHeaders = headers.encode(); - } - } - - void replicatorGotTLSCertificate(slice certData) override { -#ifdef COUCHBASE_ENTERPRISE - LOCK(_mutex); - _peerTLSCertificateData = certData; - _peerTLSCertificate = nullptr; -#endif - } - + void replicatorGotHTTPResponse(Replicator* repl, int status, const websocket::Headers& headers) override; + void replicatorGotTLSCertificate(slice certData) override; // Replicator::Delegate method, notifying that the status level or progress have changed. - void replicatorStatusChanged(Replicator* repl, const Replicator::Status& newStatus) override { - Retained selfRetain = this; // Keep myself alive till this method returns - - bool stopped, resume = false; - { - LOCK(_mutex); - if ( repl != _replicator ) return; - auto oldLevel = _status.level; - updateStatusFromReplicator((C4ReplicatorStatus)newStatus); - if ( _status.level > kC4Connecting && oldLevel <= kC4Connecting ) handleConnected(); - if ( _status.level == kC4Stopped ) { - _replicator->terminate(); - _replicator = nullptr; - if ( statusFlag(kC4Suspended) ) { - // If suspended, go to Offline state when Replicator stops - _status.level = kC4Offline; - } else if ( oldLevel != kC4Stopping ) { - // CBL-1054, only do this if a request to stop is not present, as it should - // override the offline handling - handleStopped(); // NOTE: handleStopped may change _status - } - - resume = _cancelStop; - _cancelStop = false; - } - stopped = (_status.level == kC4Stopped); - } - - notifyStateChanged(); - - if ( stopped ) _selfRetain = nullptr; // balances retain in `_start` - - if ( resume ) { start(); } - - // On return from this method, if I stopped I may be deleted (due to clearing _selfRetain) - } - + void replicatorStatusChanged(Replicator* repl, const Replicator::Status& newStatus) override; // Replicator::Delegate method, notifying that document(s) have finished. - void replicatorDocumentsEnded(Replicator* repl, const std::vector>& revs) override { - if ( repl != _replicator ) return; - - auto nRevs = revs.size(); - std::vector docsEnded; - docsEnded.reserve(nRevs); - for ( int pushing = 0; pushing <= 1; ++pushing ) { - docsEnded.clear(); - for ( const auto& rev : revs ) { - if ( (rev->dir() == Dir::kPushing) == pushing ) docsEnded.push_back(rev->asDocumentEnded()); - } - if ( !docsEnded.empty() ) { - auto onDocsEnded = _onDocumentsEnded.load(); - if ( onDocsEnded ) - onDocsEnded(this, pushing, docsEnded.size(), docsEnded.data(), _options->callbackContext); - } - } - } - + void replicatorDocumentsEnded(Replicator* repl, const std::vector>& revs) override; // Replicator::Delegate method, notifying of blob up/download progress. - void replicatorBlobProgress(Replicator* repl, const Replicator::BlobProgress& p) override { - if ( repl != _replicator ) return; - auto onBlob = _onBlobProgress.load(); - if ( onBlob ) - onBlob(this, (p.dir == Dir::kPushing), p.collSpec, p.docID, p.docProperty, p.key, p.bytesCompleted, - p.bytesTotal, p.error, _options->callbackContext); - } + void replicatorBlobProgress(Replicator* repl, const Replicator::BlobProgress& p) override; // ---- Responding to state changes - // Called when the replicator's status changes to connected. virtual void handleConnected() {} @@ -412,104 +137,19 @@ namespace litecore { // Posts a notification to the client. // The mutex MUST NOT be locked, else if the `onStatusChanged` function calls back into me // I will deadlock! - void notifyStateChanged() noexcept { - C4ReplicatorStatus status = this->getStatus(); - - if ( willLog() ) { - double progress = 0.0; - if ( status.progress.unitsTotal > 0 ) - progress = 100.0 * double(status.progress.unitsCompleted) / double(status.progress.unitsTotal); - if ( status.error.code ) { - logError("State: %-s, progress=%.2f%%, error=%s", kC4ReplicatorActivityLevelNames[status.level], - progress, status.error.description().c_str()); - } else { - logInfo("State: %-s, progress=%.2f%%", kC4ReplicatorActivityLevelNames[status.level], progress); - } - } - - if ( !(status.error.code && status.level > kC4Offline) ) { - auto onStatusChanged = _onStatusChanged.load(); - if ( onStatusChanged && status.level != kC4Stopping /* Don't notify about internal state */ ) - onStatusChanged(this, status, _options->callbackContext); - } - } - - class PendingDocuments { - public: - PendingDocuments(const C4ReplicatorImpl* repl, C4CollectionSpec spec) : collectionSpec(spec) { - // Lock the replicator and copy the necessary state now, so I don't have to lock while - // calling pendingDocumentIDs (which might call into the app's validation function.) - LOCK(repl->_mutex); - replicator = repl->_replicator; - - // CBL-2448: Also make my own checkpointer and database in case a call comes in - // after Replicator::terminate() is called. The fix includes the replicator - // pending document ID function now returning a boolean success, isDocumentPending returning - // an optional and if pendingDocumentIDs returns false or isDocumentPending - // returns nullopt, the checkpointer is fallen back on - C4Collection* collection = nullptr; - // The collection must be included in the replicator's config options. - auto it = repl->_options->collectionSpecToIndex().find(collectionSpec); - if ( it != repl->_options->collectionSpecToIndex().end() ) { - if ( it->second < repl->_options->workingCollectionCount() ) { - collection = repl->_database->getCollection(collectionSpec); - } - } - - if ( collection == nullptr ) { - error::_throw(error::NotOpen, "collection not in the Replicator's config"); - } - - checkpointer.emplace(repl->_options, repl->URL(), collection); - database = repl->_database; - } - - alloc_slice pendingDocumentIDs() { - Encoder enc; - enc.beginArray(); - bool any = false; - auto callback = [&](const C4DocumentInfo& info) { - enc.writeString(info.docID); - any = true; - }; - - if ( !replicator || !replicator->pendingDocumentIDs(collectionSpec, callback) ) { - checkpointer->pendingDocumentIDs(database, callback); - } - - if ( !any ) return {}; - enc.endArray(); - return enc.finish(); - } - - bool isDocumentPending(C4Slice docID) { - if ( replicator ) { - auto result = replicator->isDocumentPending(docID, collectionSpec); - if ( result.has_value() ) { return *result; } - } - return checkpointer->isDocumentPending(database, docID); - } - - private: - Retained replicator; - optional checkpointer; // initialized in the constructor - Retained database; - C4CollectionSpec collectionSpec; - }; + void notifyStateChanged() noexcept; mutable std::mutex _mutex; - Retained const _database; + DatabaseOrPool const _database; Retained _options; - - Retained _replicator; - C4ReplicatorStatus _status{kC4Stopped}; - bool _activeWhenSuspended{false}; - bool _cancelStop{false}; - - protected: - void setLoggingName(const string& loggingName) { _loggingName = loggingName; } + Retained _replicator; + C4ReplicatorStatus _status{kC4Stopped}; + bool _activeWhenSuspended{false}; + bool _cancelStop{false}; private: + class PendingDocuments; + std::string _loggingName; alloc_slice _responseHeaders; #ifdef COUCHBASE_ENTERPRISE diff --git a/Replicator/c4Socket+Internal.hh b/Replicator/c4Socket+Internal.hh index 8f386cdf8..5fe68d81a 100644 --- a/Replicator/c4Socket+Internal.hh +++ b/Replicator/c4Socket+Internal.hh @@ -13,11 +13,11 @@ #pragma once #include "c4Socket.hh" #include "WebSocketImpl.hh" -#include "DBAccess.hh" struct c4Database; namespace litecore::repl { + class DBAccess; // Main factory function to create a WebSocket. fleece::Retained CreateWebSocket(const websocket::URL&, const fleece::alloc_slice& options, diff --git a/Replicator/tests/DBAccessTestWrapper.cc b/Replicator/tests/DBAccessTestWrapper.cc index 795c48e40..9761738bd 100644 --- a/Replicator/tests/DBAccessTestWrapper.cc +++ b/Replicator/tests/DBAccessTestWrapper.cc @@ -28,8 +28,7 @@ C4DocEnumerator* DBAccessTestWrapper::unresolvedDocsEnumerator(C4Database* db) { } C4DocEnumerator* DBAccessTestWrapper::unresolvedDocsEnumerator(C4Collection* coll) { - std::shared_ptr acc = make_shared(coll->getDatabase(), false); - return acc->unresolvedDocsEnumerator(coll, true).release(); + return DBAccess::unresolvedDocsEnumerator(coll, true).release(); } unsigned DBAccessTestWrapper::numDeltasApplied() { return DBAccess::gNumDeltasApplied; } diff --git a/Replicator/tests/ReplicatorAPITest.cc b/Replicator/tests/ReplicatorAPITest.cc index 572748b36..57de801bb 100644 --- a/Replicator/tests/ReplicatorAPITest.cc +++ b/Replicator/tests/ReplicatorAPITest.cc @@ -15,13 +15,12 @@ #include "ReplicatorAPITest.hh" #include "c4Collection.h" #include "c4ReplicatorHelpers.hh" +#include "c4ReplicatorImpl.hh" #include "StringUtil.hh" #include "c4Socket.h" -//#include "c4Socket+Internal.hh" #include "c4Socket.hh" #include "c4Internal.hh" #include "fleece/Fleece.hh" -#include "c4ReplicatorImpl.hh" #include "SequenceSet.hh" using namespace fleece; @@ -986,18 +985,6 @@ TEST_CASE_METHOD(ReplicatorAPITest, "Progress Level vs Options", "[Pull][C]") { // NOLINTEND(cppcoreguidelines-slicing) -# include "c4ReplicatorImpl.hh" - -struct C4TestReplicator : public litecore::C4ReplicatorImpl { - C4TestReplicator(C4Database* db, C4ReplicatorParameters params) : C4ReplicatorImpl(db, params) {} - - alloc_slice propertiesMemory() const { return _options->properties.data(); } - - void createReplicator() override {} - - alloc_slice URL() const override { return nullslice; } -}; - #endif TEST_CASE_METHOD(ReplicatorAPITest, "Connection Timeout stop properly", "[C][Push][Pull][.Slow]") { diff --git a/Replicator/tests/ReplicatorAPITest.hh b/Replicator/tests/ReplicatorAPITest.hh index 7c19afa16..00cbd50db 100644 --- a/Replicator/tests/ReplicatorAPITest.hh +++ b/Replicator/tests/ReplicatorAPITest.hh @@ -251,10 +251,11 @@ class ReplicatorAPITest : public C4Test { } if ( s.level == kC4Idle ) { - C4Log("*** Replicator idle; stopping..."); if ( _stopWhenIdle.load() ) { + C4Log("*** Replicator idle; stopping..."); c4repl_stop(r); } else if ( _callbackWhenIdle ) { + C4Log("*** Replicator idle"); _callbackWhenIdle(); } } diff --git a/Replicator/tests/ReplicatorLoopbackTest.cc b/Replicator/tests/ReplicatorLoopbackTest.cc index 8c76099ee..e5fe2a03a 100644 --- a/Replicator/tests/ReplicatorLoopbackTest.cc +++ b/Replicator/tests/ReplicatorLoopbackTest.cc @@ -1421,7 +1421,7 @@ N_WAY_TEST_CASE_METHOD(ReplicatorLoopbackTest, "UnresolvedDocs", "[Push][Pull][C runReplicators(Replicator::Options::pulling(kC4OneShot, _collSpec), Replicator::Options::passive(_collSpec)); validateCheckpoints(db, db2, "{\"remote\":7}"); - auto e = DBAccessTestWrapper::unresolvedDocsEnumerator(_collDB1); + c4::ref e = DBAccessTestWrapper::unresolvedDocsEnumerator(_collDB1); REQUIRE(e); // verify only returns the conflicted documents, including the deleted ones. @@ -1443,7 +1443,6 @@ N_WAY_TEST_CASE_METHOD(ReplicatorLoopbackTest, "UnresolvedDocs", "[Push][Pull][C CHECK(deleted == deleteds[count]); } CHECK(!c4enum_next(e, WITH_ERROR(&err))); - c4enum_free(e); } #pragma mark - DELTA: diff --git a/Xcode/LiteCore.xcodeproj/project.pbxproj b/Xcode/LiteCore.xcodeproj/project.pbxproj index 634c92005..3f9b521dc 100644 --- a/Xcode/LiteCore.xcodeproj/project.pbxproj +++ b/Xcode/LiteCore.xcodeproj/project.pbxproj @@ -61,7 +61,6 @@ 27098ABC217525B7002751DA /* SQLiteKeyStore+FTSIndexes.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27098ABB217525B7002751DA /* SQLiteKeyStore+FTSIndexes.cc */; }; 27098AC02175279F002751DA /* SQLiteKeyStore+ArrayIndexes.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27098ABF2175279F002751DA /* SQLiteKeyStore+ArrayIndexes.cc */; }; 27098AC421752A29002751DA /* SQLiteKeyStore+PredictiveIndexes.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27098AC321752A29002751DA /* SQLiteKeyStore+PredictiveIndexes.cc */; }; - 270C6B691EB7DDAD00E73415 /* RESTListener+Replicate.cc in Sources */ = {isa = PBXBuildFile; fileRef = 270C6B681EB7DDAD00E73415 /* RESTListener+Replicate.cc */; }; 270C6B8C1EBA2CD600E73415 /* LogEncoder.cc in Sources */ = {isa = PBXBuildFile; fileRef = 270C6B891EBA2CD600E73415 /* LogEncoder.cc */; }; 270C6B981EBA3AD200E73415 /* LogEncoderTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 270C6B901EBA2D5600E73415 /* LogEncoderTest.cc */; }; 270C7D522022916D00FF86D3 /* CoreFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 270515581D907F6200D62D05 /* CoreFoundation.framework */; }; @@ -72,7 +71,7 @@ 27139B3118F8E9750021A9A3 /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 275072AB18E4A68E00A80C5A /* XCTest.framework */; }; 2716F91F248578D000BE21D9 /* mbedSnippets.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2716F91E248578D000BE21D9 /* mbedSnippets.cc */; }; 2716F9BF249AD3CB00BE21D9 /* c4CertificateTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2716F9BE249AD3CB00BE21D9 /* c4CertificateTest.cc */; }; - 27175B12261B91170045F3AC /* REST_CAPI.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27175B11261B91170045F3AC /* REST_CAPI.cc */; }; + 27175B12261B91170045F3AC /* c4Listener_CAPI.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27175B11261B91170045F3AC /* c4Listener_CAPI.cc */; }; 2718B38A2CAF1545006C09CB /* libCatch2.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2718B3892CAF1545006C09CB /* libCatch2.a */; }; 2718B38B2CAF154C006C09CB /* libCatch2.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2718B3892CAF1545006C09CB /* libCatch2.a */; }; 271925132396FE160053DDA6 /* c4BaseTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 275FF6D11E4947E1005F90DD /* c4BaseTest.cc */; }; @@ -94,7 +93,6 @@ 2719252A23970BDF0053DDA6 /* c4PredictiveQueryTest+CoreML.mm in Sources */ = {isa = PBXBuildFile; fileRef = 27C77301216FCF5400D5FB44 /* c4PredictiveQueryTest+CoreML.mm */; }; 2719252B23970BE40053DDA6 /* CoreMLPredictiveModel.mm in Sources */ = {isa = PBXBuildFile; fileRef = 2700BB5A217005A900797537 /* CoreMLPredictiveModel.mm */; }; 2719252C23970BFD0053DDA6 /* RESTClientTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E19D652316EDEA00E031F8 /* RESTClientTest.cc */; }; - 2719252D23970BFD0053DDA6 /* RESTListenerTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 276E02101EA9717200FEFE8A /* RESTListenerTest.cc */; }; 2719252E23970BFD0053DDA6 /* SyncListenerTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27B6491F2065AD2B00FC12F7 /* SyncListenerTest.cc */; }; 2719252F23970C050053DDA6 /* CertificateTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2762A01422EB7CC800F9AB18 /* CertificateTest.cc */; }; 2719253023970C480053DDA6 /* libLiteCoreWebSocket.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2771A098228624C000B18E0A /* libLiteCoreWebSocket.a */; }; @@ -122,7 +120,6 @@ 272850EE1E9D4D23009CA22F /* libz.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 2759DC251E70908900F3C4B2 /* libz.tbd */; }; 272850F11E9D4F94009CA22F /* ReplicatorAPITest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2745DE4B1E735B9000F02CA0 /* ReplicatorAPITest.cc */; }; 272850F21E9D4FB1009CA22F /* libfleeceStatic.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 27FA09BC1D70ADE8005888AA /* libfleeceStatic.a */; }; - 272851241EA4537A009CA22F /* RESTListener.hh in Headers */ = {isa = PBXBuildFile; fileRef = 272851211EA4537A009CA22F /* RESTListener.hh */; }; 2728512B1EA46421009CA22F /* Request.hh in Headers */ = {isa = PBXBuildFile; fileRef = 272851281EA46421009CA22F /* Request.hh */; }; 272851301EA46475009CA22F /* Server.hh in Headers */ = {isa = PBXBuildFile; fileRef = 2728512D1EA46475009CA22F /* Server.hh */; }; 272B1BE11FB13B7400F56620 /* stopwordset.cc in Sources */ = {isa = PBXBuildFile; fileRef = 272B1BDF1FB13B7400F56620 /* stopwordset.cc */; }; @@ -179,7 +176,6 @@ 2746C8E62639E88700A3B2CC /* ThreadUtil.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2746C8E52639E88700A3B2CC /* ThreadUtil.cc */; }; 2747A1D0279B37E100F286AF /* SQLUtil.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2747A1CF279B37E100F286AF /* SQLUtil.cc */; }; 27480E37253A5D9C0091CF37 /* VectorRecordTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27480E36253A5D9C0091CF37 /* VectorRecordTest.cc */; }; - 2749B9871EB298360068DBF9 /* RESTListener+Handlers.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2749B9861EB298360068DBF9 /* RESTListener+Handlers.cc */; }; 274B36D225B271F7001FC28D /* Version.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274B36D125B271F7001FC28D /* Version.cc */; }; 274D040F1BA75E5000FF7C35 /* c4DatabaseTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274D04001BA75C0400FF7C35 /* c4DatabaseTest.cc */; }; 274D04201BA892B100FF7C35 /* libLiteCore.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 720EA3F51BA7EAD9002B8416 /* libLiteCore.dylib */; }; @@ -198,8 +194,6 @@ 2753AFC91EC0F4E900C12E98 /* CoreFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 270515581D907F6200D62D05 /* CoreFoundation.framework */; }; 2753B00D1EC39E5D00C12E98 /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 27139B1F18F8E9750021A9A3 /* Foundation.framework */; }; 2754B0C71E5F5C2900A05FD0 /* StringUtil.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2754B0C01E5F49AA00A05FD0 /* StringUtil.cc */; }; - 275A74D11ED3A4E1008CB57B /* Listener.cc in Sources */ = {isa = PBXBuildFile; fileRef = 275A74CF1ED3A4E1008CB57B /* Listener.cc */; }; - 275A74D31ED3A4E1008CB57B /* Listener.hh in Headers */ = {isa = PBXBuildFile; fileRef = 275A74D01ED3A4E1008CB57B /* Listener.hh */; }; 275A74D61ED3AA11008CB57B /* c4Listener.cc in Sources */ = {isa = PBXBuildFile; fileRef = 275A74D51ED3AA11008CB57B /* c4Listener.cc */; }; 275B2DF42A4A0A1A00B7215E /* HybridClock.cc in Sources */ = {isa = PBXBuildFile; fileRef = 275B2DF32A4A0A1A00B7215E /* HybridClock.cc */; }; 275B35A5234E753800FE9CF0 /* Housekeeper.cc in Sources */ = {isa = PBXBuildFile; fileRef = 275B35A4234E753800FE9CF0 /* Housekeeper.cc */; }; @@ -238,9 +232,13 @@ 277911BE2C66DD610044E660 /* Arena.cc in Sources */ = {isa = PBXBuildFile; fileRef = 277911BD2C66DD610044E660 /* Arena.cc */; }; 277911C12C6A96FA0044E660 /* Node.cc in Sources */ = {isa = PBXBuildFile; fileRef = 277911C02C6A96FA0044E660 /* Node.cc */; }; 277D40062C2F3DE700DBDC97 /* c4IndexUpdaterTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = EADC3D4F2C1346DA00933269 /* c4IndexUpdaterTest.cc */; }; + 277DA44A2CEBEFFD001A15D0 /* c4ReplicatorImpl.cc in Sources */ = {isa = PBXBuildFile; fileRef = 277DA4492CEBEFFD001A15D0 /* c4ReplicatorImpl.cc */; }; + 277DA44D2CEBF110001A15D0 /* DatabasePool.cc in Sources */ = {isa = PBXBuildFile; fileRef = 277DA44C2CEBF110001A15D0 /* DatabasePool.cc */; }; 2780548C2B3214BB009630D8 /* PredictiveVectorQueryTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27F602FD2A968503006FA1D0 /* PredictiveVectorQueryTest.cc */; }; 278054952B3214BF009630D8 /* VectorQueryTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27BEEE782A783A17005AD4BF /* VectorQueryTest.cc */; }; 2783DF991D27436700F84E6E /* c4ThreadingTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2783DF981D27436700F84E6E /* c4ThreadingTest.cc */; }; + 2785C26B2D012B5600F22817 /* DatabaseRegistry.hh in Headers */ = {isa = PBXBuildFile; fileRef = 2785C2692D012B5600F22817 /* DatabaseRegistry.hh */; }; + 2785C26C2D012B5600F22817 /* DatabaseRegistry.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2785C26A2D012B5600F22817 /* DatabaseRegistry.cc */; }; 2787EB271F4C91B000DB97B0 /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 27766E151982DA8E00CAA464 /* Security.framework */; }; 2787EB291F4C929C00DB97B0 /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 27766E151982DA8E00CAA464 /* Security.framework */; }; 278963621D7A376900493096 /* EncryptedStream.cc in Sources */ = {isa = PBXBuildFile; fileRef = 278963601D7A376900493096 /* EncryptedStream.cc */; }; @@ -273,7 +271,6 @@ 278F478A24C914CD00E1CA7A /* SystemConfiguration.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 278F478924C914CC00E1CA7A /* SystemConfiguration.framework */; }; 278F478B24C914DE00E1CA7A /* libLiteCoreWebSocket.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 2771A098228624C000B18E0A /* libLiteCoreWebSocket.a */; }; 2791EA1420326F7100BD813C /* SQLiteChooser.c in Sources */ = {isa = PBXBuildFile; fileRef = 2791EA1320326F7100BD813C /* SQLiteChooser.c */; }; - 279691981ED4C3950086565D /* c4Listener+RESTFactory.cc in Sources */ = {isa = PBXBuildFile; fileRef = 279691971ED4C3950086565D /* c4Listener+RESTFactory.cc */; }; 2796A28423072F7000774850 /* Server.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2728512C1EA46475009CA22F /* Server.cc */; }; 2797BCB21C10F71700E5C991 /* c4AllDocsPerformanceTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2797BCAE1C10F69E00E5C991 /* c4AllDocsPerformanceTest.cc */; }; 2797BCB41C10F76100E5C991 /* libLiteCore-static.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 27EF81121917EEC600A327B9 /* libLiteCore-static.a */; }; @@ -342,7 +339,6 @@ 27B699E11F27B85900782145 /* SQLiteFleeceUtil.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27B699E01F27B85900782145 /* SQLiteFleeceUtil.cc */; }; 27B953DD239872C700C8AA90 /* CoreML.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 2700BB4D216FF2DA00797537 /* CoreML.framework */; settings = {ATTRIBUTES = (Required, ); }; }; 27B953DE239872D900C8AA90 /* Vision.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 27098AB721714AB0002751DA /* Vision.framework */; }; - 27B9669723284F2900B2897F /* RESTListenerTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 276E02101EA9717200FEFE8A /* RESTListenerTest.cc */; }; 27BEEE712A72FCEA005AD4BF /* SQLiteKeyStore+VectorIndex.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27BEEE702A72FCEA005AD4BF /* SQLiteKeyStore+VectorIndex.cc */; }; 27BEEE792A783A17005AD4BF /* VectorQueryTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27BEEE782A783A17005AD4BF /* VectorQueryTest.cc */; }; 27C319EE1A143F5D00A89EDC /* KeyStore.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27C319EC1A143F5D00A89EDC /* KeyStore.cc */; }; @@ -398,6 +394,9 @@ 27E48713192171EA007D8940 /* DataFile.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E48711192171EA007D8940 /* DataFile.cc */; }; 27E487231922A64F007D8940 /* RevTree.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E487211922A64F007D8940 /* RevTree.cc */; }; 27E4872B1923F24D007D8940 /* RevTreeRecord.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E487291923F24D007D8940 /* RevTreeRecord.cc */; }; + 27E55D262CE55963000DE11A /* SyncListener.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27E55D232CE55963000DE11A /* SyncListener.hh */; }; + 27E55D272CE55963000DE11A /* HTTPListener.hh in Headers */ = {isa = PBXBuildFile; fileRef = 27E55D202CE55963000DE11A /* HTTPListener.hh */; }; + 27E55D292CE55963000DE11A /* HTTPListener.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E55D212CE55963000DE11A /* HTTPListener.cc */; }; 27E609A21951E4C000202B72 /* RecordEnumerator.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E609A11951E4C000202B72 /* RecordEnumerator.cc */; }; 27E6737D1EC78144008F50C4 /* QueryTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E6737C1EC78144008F50C4 /* QueryTest.cc */; }; 27E6739F1EC8DC97008F50C4 /* c4QueryTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27416E291E0494DF00F10F65 /* c4QueryTest.cc */; }; @@ -494,7 +493,6 @@ 27FA568924AD0F8E00B2F1F8 /* Pusher+Revs.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27FA568824AD0F8E00B2F1F8 /* Pusher+Revs.cc */; }; 27FA99E01917EF9600912F96 /* libTokenizer.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 27EF807419142C2500A327B9 /* libTokenizer.a */; }; 27FB0C3D205B18A500987D9C /* Instrumentation.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27FB0C3C205B18A500987D9C /* Instrumentation.cc */; }; - 27FC81F61EAAB7C60028E38E /* RESTListener.cc in Sources */ = {isa = PBXBuildFile; fileRef = 272851201EA4537A009CA22F /* RESTListener.cc */; }; 27FC81F91EAAB7C60028E38E /* Request.cc in Sources */ = {isa = PBXBuildFile; fileRef = 272851271EA46421009CA22F /* Request.cc */; }; 27FC8DB622135BCE0083B033 /* ChangesFeed.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27FC8DB522135BCE0083B033 /* ChangesFeed.cc */; }; 27FC8DBD22135BDA0083B033 /* RevFinder.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27FC8DBC22135BDA0083B033 /* RevFinder.cc */; }; @@ -519,7 +517,6 @@ 27FE0CFD24BE817A00A36EC2 /* ReplicatorSGTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 277FEE5721ED10FA00B60E3C /* ReplicatorSGTest.cc */; }; 27FE0CFE24BE817A00A36EC2 /* CookieStoreTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2761F3F61EEA00C3006D4BB8 /* CookieStoreTest.cc */; }; 27FE0CFF24BE817B00A36EC2 /* RESTClientTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27E19D652316EDEA00E031F8 /* RESTClientTest.cc */; }; - 27FE0D0024BE817B00A36EC2 /* RESTListenerTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 276E02101EA9717200FEFE8A /* RESTListenerTest.cc */; }; 27FE0D0124BE817B00A36EC2 /* SyncListenerTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 27B6491F2065AD2B00FC12F7 /* SyncListenerTest.cc */; }; 27FE0D0224BE817B00A36EC2 /* CertificateTest.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2762A01422EB7CC800F9AB18 /* CertificateTest.cc */; }; 42030A3E2498442F00283CE8 /* SecureRandomize.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274D5BA31DF8D90100BDAF9D /* SecureRandomize.cc */; }; @@ -909,7 +906,6 @@ 2709D3A52363651B00462AF7 /* CertHelper.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = CertHelper.hh; sourceTree = ""; }; 270BEE1D20647E8A005E8BE8 /* RESTSyncListener_stub.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RESTSyncListener_stub.cc; sourceTree = ""; }; 270BEE29206483C0005E8BE8 /* Listener */ = {isa = PBXFileReference; lastKnownFileType = folder; name = Listener; path = "../../../couchbase-lite-core-EE/Listener"; sourceTree = ""; }; - 270C6B681EB7DDAD00E73415 /* RESTListener+Replicate.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = "RESTListener+Replicate.cc"; sourceTree = ""; }; 270C6B871EBA2CD600E73415 /* LogDecoder.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LogDecoder.cc; sourceTree = ""; }; 270C6B881EBA2CD600E73415 /* LogDecoder.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = LogDecoder.hh; sourceTree = ""; }; 270C6B891EBA2CD600E73415 /* LogEncoder.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LogEncoder.cc; sourceTree = ""; }; @@ -929,7 +925,7 @@ 2716F91E248578D000BE21D9 /* mbedSnippets.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = mbedSnippets.cc; sourceTree = ""; }; 2716F9BE249AD3CB00BE21D9 /* c4CertificateTest.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = c4CertificateTest.cc; sourceTree = ""; }; 27175AF1261B80E70045F3AC /* c4BlobStoreTypes.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = c4BlobStoreTypes.h; sourceTree = ""; }; - 27175B11261B91170045F3AC /* REST_CAPI.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = REST_CAPI.cc; sourceTree = ""; }; + 27175B11261B91170045F3AC /* c4Listener_CAPI.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = c4Listener_CAPI.cc; sourceTree = ""; }; 27175B73261BC3030045F3AC /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; wrapsLines = 1; }; 2719253323970C7F0053DDA6 /* data */ = {isa = PBXFileReference; lastKnownFileType = folder; path = data; sourceTree = ""; }; 2719253B23970F4E0053DDA6 /* replacedb */ = {isa = PBXFileReference; lastKnownFileType = folder; name = replacedb; path = data/replacedb; sourceTree = ""; }; @@ -965,8 +961,6 @@ 272850AA1E9AF53B009CA22F /* Upgrader.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Upgrader.hh; sourceTree = ""; }; 272850B41E9BE361009CA22F /* UpgraderTest.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = UpgraderTest.cc; sourceTree = ""; }; 272850EC1E9D4B7D009CA22F /* CppTests.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = CppTests.xcconfig; sourceTree = ""; }; - 272851201EA4537A009CA22F /* RESTListener.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RESTListener.cc; sourceTree = ""; }; - 272851211EA4537A009CA22F /* RESTListener.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = RESTListener.hh; sourceTree = ""; }; 272851271EA46421009CA22F /* Request.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Request.cc; sourceTree = ""; }; 272851281EA46421009CA22F /* Request.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Request.hh; sourceTree = ""; }; 2728512C1EA46475009CA22F /* Server.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Server.cc; sourceTree = ""; }; @@ -1114,7 +1108,6 @@ 27491C9A1E7B1001001DC54B /* c4Socket.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = c4Socket.h; sourceTree = ""; }; 27491C9E1E7B2532001DC54B /* c4Socket.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = c4Socket.cc; sourceTree = ""; }; 27491CA21E7B6A79001DC54B /* c4Socket+Internal.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = "c4Socket+Internal.hh"; sourceTree = ""; }; - 2749B9861EB298360068DBF9 /* RESTListener+Handlers.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = "RESTListener+Handlers.cc"; sourceTree = ""; }; 274A116A1D7F484000E97A62 /* SecureSymmetricCrypto.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = SecureSymmetricCrypto.hh; sourceTree = ""; }; 274A69871BED288D00D16D37 /* c4Document.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = c4Document.cc; sourceTree = ""; }; 274A69881BED288D00D16D37 /* c4Document.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = c4Document.h; sourceTree = ""; }; @@ -1169,8 +1162,6 @@ 2759DC251E70908900F3C4B2 /* libz.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libz.tbd; path = usr/lib/libz.tbd; sourceTree = SDKROOT; }; 275A73BD1ED255AF008CB57B /* c4ReplicatorImpl.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = c4ReplicatorImpl.hh; sourceTree = ""; }; 275A74461ED37992008CB57B /* Base.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Base.hh; sourceTree = ""; }; - 275A74CF1ED3A4E1008CB57B /* Listener.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Listener.cc; sourceTree = ""; }; - 275A74D01ED3A4E1008CB57B /* Listener.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Listener.hh; sourceTree = ""; }; 275A74D51ED3AA11008CB57B /* c4Listener.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = c4Listener.cc; sourceTree = ""; }; 275A74DF1ED4A05C008CB57B /* c4ListenerInternal.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = c4ListenerInternal.hh; sourceTree = ""; }; 275AA5D72A5F603800ABC172 /* SourceID.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = SourceID.hh; sourceTree = ""; }; @@ -1240,7 +1231,6 @@ 276D153E1DFF53F500543B1B /* SQLiteEnumerator.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SQLiteEnumerator.cc; sourceTree = ""; }; 276D15401DFF541000543B1B /* SQLiteQuery.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SQLiteQuery.cc; sourceTree = ""; }; 276D4AD42787709200F61A89 /* c4EnumUtil.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = c4EnumUtil.hh; sourceTree = ""; }; - 276E02101EA9717200FEFE8A /* RESTListenerTest.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RESTListenerTest.cc; sourceTree = ""; }; 276E02191EA983EE00FEFE8A /* Response.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Response.cc; sourceTree = ""; }; 276E021A1EA983EE00FEFE8A /* Response.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Response.hh; sourceTree = ""; }; 27700DFD1FB642B80005D48E /* Increment.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = Increment.hh; sourceTree = ""; }; @@ -1272,10 +1262,16 @@ 2779CC6E1E85E4FC00F0D251 /* ReplicatorTypes.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = ReplicatorTypes.hh; sourceTree = ""; }; 277CB6251D0DED5E00702E56 /* Fleece.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = Fleece.xcodeproj; path = fleece/Fleece.xcodeproj; sourceTree = ""; }; 277D19C9194E295B008E91EB /* Error.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = Error.hh; sourceTree = ""; }; + 277DA4492CEBEFFD001A15D0 /* c4ReplicatorImpl.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = c4ReplicatorImpl.cc; sourceTree = ""; }; + 277DA44B2CEBF110001A15D0 /* DatabasePool.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = DatabasePool.hh; sourceTree = ""; }; + 277DA44C2CEBF110001A15D0 /* DatabasePool.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = DatabasePool.cc; sourceTree = ""; }; 277FEE5721ED10FA00B60E3C /* ReplicatorSGTest.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = ReplicatorSGTest.cc; sourceTree = ""; }; 278054D72B321BDB009630D8 /* LiteCore-iOS shell.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = "LiteCore-iOS shell.xcconfig"; sourceTree = ""; }; 278054D82B321D54009630D8 /* LiteCore-IOS Tests.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = "LiteCore-IOS Tests.xcconfig"; sourceTree = ""; }; 2783DF981D27436700F84E6E /* c4ThreadingTest.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = c4ThreadingTest.cc; sourceTree = ""; }; + 2785C2682D012AA100F22817 /* c4Replicator+Pool.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = "c4Replicator+Pool.hh"; sourceTree = ""; }; + 2785C2692D012B5600F22817 /* DatabaseRegistry.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = DatabaseRegistry.hh; sourceTree = ""; }; + 2785C26A2D012B5600F22817 /* DatabaseRegistry.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = DatabaseRegistry.cc; sourceTree = ""; }; 278963601D7A376900493096 /* EncryptedStream.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = EncryptedStream.cc; path = ../Support/EncryptedStream.cc; sourceTree = ""; }; 278963611D7A376900493096 /* EncryptedStream.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = EncryptedStream.hh; path = ../Support/EncryptedStream.hh; sourceTree = ""; }; 278963651D7B3E0E00493096 /* Stream.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; name = Stream.hh; path = ../Support/Stream.hh; sourceTree = ""; }; @@ -1295,7 +1291,6 @@ 2791EA1320326F7100BD813C /* SQLiteChooser.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = SQLiteChooser.c; sourceTree = ""; }; 2791EA192032732500BD813C /* Project_Debug_EE.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Project_Debug_EE.xcconfig; sourceTree = ""; }; 2791EA1A203273BF00BD813C /* Project_Release_EE.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Project_Release_EE.xcconfig; sourceTree = ""; }; - 279691971ED4C3950086565D /* c4Listener+RESTFactory.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = "c4Listener+RESTFactory.cc"; sourceTree = ""; }; 2797BCAE1C10F69E00E5C991 /* c4AllDocsPerformanceTest.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = c4AllDocsPerformanceTest.cc; sourceTree = ""; }; 27984E422249AEDD000FE777 /* dylib_Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = dylib_Release.xcconfig; sourceTree = ""; }; 279976311E94AAD000B27639 /* IncomingRev+Blobs.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = "IncomingRev+Blobs.cc"; sourceTree = ""; }; @@ -1519,6 +1514,9 @@ 27E487221922A64F007D8940 /* RevTree.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = RevTree.hh; sourceTree = ""; }; 27E487291923F24D007D8940 /* RevTreeRecord.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RevTreeRecord.cc; sourceTree = ""; }; 27E4872A1923F24D007D8940 /* RevTreeRecord.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = RevTreeRecord.hh; sourceTree = ""; }; + 27E55D202CE55963000DE11A /* HTTPListener.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = HTTPListener.hh; sourceTree = ""; }; + 27E55D212CE55963000DE11A /* HTTPListener.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = HTTPListener.cc; sourceTree = ""; }; + 27E55D232CE55963000DE11A /* SyncListener.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = SyncListener.hh; sourceTree = ""; }; 27E609A11951E4C000202B72 /* RecordEnumerator.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RecordEnumerator.cc; sourceTree = ""; }; 27E609A41951E53F00202B72 /* RecordEnumerator.hh */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = RecordEnumerator.hh; sourceTree = ""; }; 27E6737C1EC78144008F50C4 /* QueryTest.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = QueryTest.cc; sourceTree = ""; }; @@ -2013,29 +2011,27 @@ 272851111EA44902009CA22F /* REST */ = { isa = PBXGroup; children = ( - 275A74CF1ED3A4E1008CB57B /* Listener.cc */, - 275A74D01ED3A4E1008CB57B /* Listener.hh */, - 272851201EA4537A009CA22F /* RESTListener.cc */, - 2749B9861EB298360068DBF9 /* RESTListener+Handlers.cc */, - 270C6B681EB7DDAD00E73415 /* RESTListener+Replicate.cc */, - 272851211EA4537A009CA22F /* RESTListener.hh */, - 2728512C1EA46475009CA22F /* Server.cc */, - 2728512D1EA46475009CA22F /* Server.hh */, - 276E02191EA983EE00FEFE8A /* Response.cc */, - 276E021A1EA983EE00FEFE8A /* Response.hh */, - 272851271EA46421009CA22F /* Request.cc */, - 272851281EA46421009CA22F /* Request.hh */, 275A74D51ED3AA11008CB57B /* c4Listener.cc */, - 279691971ED4C3950086565D /* c4Listener+RESTFactory.cc */, 275A74DF1ED4A05C008CB57B /* c4ListenerInternal.hh */, + 27175B11261B91170045F3AC /* c4Listener_CAPI.cc */, 27DDC549236B56D000580B2B /* CertRequest.hh */, 27DDC54A236B56D000580B2B /* CertRequest.cc */, - 279D40F51EA533D900D8DD9D /* netUtils.cc */, + 2785C2692D012B5600F22817 /* DatabaseRegistry.hh */, + 2785C26A2D012B5600F22817 /* DatabaseRegistry.cc */, + 27E55D202CE55963000DE11A /* HTTPListener.hh */, + 27E55D212CE55963000DE11A /* HTTPListener.cc */, 279D40F61EA533D900D8DD9D /* netUtils.hh */, - 27175B11261B91170045F3AC /* REST_CAPI.cc */, + 279D40F51EA533D900D8DD9D /* netUtils.cc */, + 272851281EA46421009CA22F /* Request.hh */, + 272851271EA46421009CA22F /* Request.cc */, + 276E021A1EA983EE00FEFE8A /* Response.hh */, + 276E02191EA983EE00FEFE8A /* Response.cc */, + 2728512D1EA46475009CA22F /* Server.hh */, + 2728512C1EA46475009CA22F /* Server.cc */, + 27E55D232CE55963000DE11A /* SyncListener.hh */, + 2750735B1F4B5EDB003D2CCE /* CMakeLists.txt */, 270BEE1C20647E8A005E8BE8 /* EE */, 276E020F1EA9712700FEFE8A /* tests */, - 2750735B1F4B5EDB003D2CCE /* CMakeLists.txt */, ); name = REST; path = ../REST; @@ -2396,6 +2392,8 @@ 277911C32C6AD3F90044E660 /* checked_ptr.hh */, 2744B334241854F2005A194D /* Codec.cc */, 2744B33E241854F2005A194D /* Codec.hh */, + 277DA44B2CEBF110001A15D0 /* DatabasePool.hh */, + 277DA44C2CEBF110001A15D0 /* DatabasePool.cc */, 276CF337254C893200C493B5 /* DeDuplicateEncoder.hh */, 2762A01F22EF641900F9AB18 /* Defer.hh */, 27393A861C8A353A00829C9B /* Error.cc */, @@ -2711,7 +2709,6 @@ 2709D3A52363651B00462AF7 /* CertHelper.hh */, 27047CE3233AE8DE009D1CE9 /* ListenerHarness.hh */, 27E19D652316EDEA00E031F8 /* RESTClientTest.cc */, - 276E02101EA9717200FEFE8A /* RESTListenerTest.cc */, 27B6491F2065AD2B00FC12F7 /* SyncListenerTest.cc */, ); path = tests; @@ -2784,7 +2781,10 @@ 27491C9E1E7B2532001DC54B /* c4Socket.cc */, 27491CA21E7B6A79001DC54B /* c4Socket+Internal.hh */, 275CE0E11E57B7E70084E014 /* c4Replicator.cc */, + 2785C2682D012AA100F22817 /* c4Replicator+Pool.hh */, + D64D17BB2894777A008B68FD /* c4ReplicatorHelpers.hh */, 2743E35525F85F26006F696D /* c4Replicator_CAPI.cc */, + 277DA4492CEBEFFD001A15D0 /* c4ReplicatorImpl.cc */, 275A73BD1ED255AF008CB57B /* c4ReplicatorImpl.hh */, 27D965252330394C00F4A51C /* c4RemoteReplicator.hh */, 27D9652A23303A2B00F4A51C /* c4LocalReplicator.hh */, @@ -2993,7 +2993,6 @@ 27CCC7D71E52613C00CE1989 /* Replicator.hh */, 275E98FF238360B200EA516B /* Checkpointer.cc */, 275E9904238360B200EA516B /* Checkpointer.hh */, - D64D17BB2894777A008B68FD /* c4ReplicatorHelpers.hh */, 275E4CD22241C701006C5B71 /* Pull */, 275E4CD32241C726006C5B71 /* Push */, 277B9567224597E1005B7E79 /* Support */, @@ -3510,7 +3509,9 @@ buildActionMask = 2147483647; files = ( 728EC54D1EC14611002C9A73 /* c4Listener.h in Headers */, - 275A74D31ED3A4E1008CB57B /* Listener.hh in Headers */, + 27E55D262CE55963000DE11A /* SyncListener.hh in Headers */, + 27E55D272CE55963000DE11A /* HTTPListener.hh in Headers */, + 2785C26B2D012B5600F22817 /* DatabaseRegistry.hh in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -3534,7 +3535,6 @@ 279D40F91EA533D900D8DD9D /* netUtils.hh in Headers */, 272851301EA46475009CA22F /* Server.hh in Headers */, 274EDDEE1DA2F488003AD158 /* SQLiteKeyStore.hh in Headers */, - 272851241EA4537A009CA22F /* RESTListener.hh in Headers */, 278963641D7A376900493096 /* EncryptedStream.hh in Headers */, 27E89BA81D679542002C32B3 /* FilePath.hh in Headers */, 72A3AF8D1F425134001E16D4 /* PrebuiltCopier.hh in Headers */, @@ -4203,7 +4203,6 @@ FCC064D7287E31D6000C5BD7 /* ReplicatorCollectionTest.cc in Sources */, 272850EA1E9D4860009CA22F /* ReplicatorLoopbackTest.cc in Sources */, 27E19D662316EDEA00E031F8 /* RESTClientTest.cc in Sources */, - 27B9669723284F2900B2897F /* RESTListenerTest.cc in Sources */, D6F99A0428E4EFB200D2DC63 /* ReplicatorCollectionSGTest.cc in Sources */, 27628CD22AC21F0D004C3740 /* ReplicatorVVUpgradeTest.cc in Sources */, 27AFF3BA2303758E00B4D6C4 /* ReplicatorAPITest.cc in Sources */, @@ -4246,7 +4245,6 @@ 2719252F23970C050053DDA6 /* CertificateTest.cc in Sources */, 2719252A23970BDF0053DDA6 /* c4PredictiveQueryTest+CoreML.mm in Sources */, 2719252C23970BFD0053DDA6 /* RESTClientTest.cc in Sources */, - 2719252D23970BFD0053DDA6 /* RESTListenerTest.cc in Sources */, 2719252E23970BFD0053DDA6 /* SyncListenerTest.cc in Sources */, 2719253523970E070053DDA6 /* c4Listener.cc in Sources */, 271925172396FE2C0053DDA6 /* PredictiveQueryTest.cc in Sources */, @@ -4387,7 +4385,6 @@ 27FE0CFF24BE817B00A36EC2 /* RESTClientTest.cc in Sources */, EA4740D32CC80C6D00401B68 /* c4ArrayIndexTest.cc in Sources */, 2740A7522B321120003387E9 /* SGTestUser.cc in Sources */, - 27FE0D0024BE817B00A36EC2 /* RESTListenerTest.cc in Sources */, 27FE0D0124BE817B00A36EC2 /* SyncListenerTest.cc in Sources */, 273E55661F79B535000182F1 /* Logging_Stub.cc in Sources */, 27A924C91D9B374500086206 /* Catch_Tests.mm in Sources */, @@ -4578,6 +4575,7 @@ 273D25F62564666A008643D2 /* VectorDocument.cc in Sources */, 27CCD4AF2315DB11003DEB99 /* Address.cc in Sources */, 279976331E94AAD000B27639 /* IncomingRev+Blobs.cc in Sources */, + 277DA44D2CEBF110001A15D0 /* DatabasePool.cc in Sources */, 27DD1513193CD005009A367D /* RevID.cc in Sources */, 2734F61A206ABEB000C982FF /* ReplicatorTypes.cc in Sources */, 2753AF721EBD190600C12E98 /* LogDecoder.cc in Sources */, @@ -4591,6 +4589,7 @@ 27FA568924AD0F8E00B2F1F8 /* Pusher+Revs.cc in Sources */, 278963621D7A376900493096 /* EncryptedStream.cc in Sources */, 275B2DF42A4A0A1A00B7215E /* HybridClock.cc in Sources */, + 277DA44A2CEBEFFD001A15D0 /* c4ReplicatorImpl.cc in Sources */, 72A3AF891F424EC0001E16D4 /* PrebuiltCopier.cc in Sources */, 93CD010B1E933BE100AFB3FA /* Worker.cc in Sources */, 27098AC02175279F002751DA /* SQLiteKeyStore+ArrayIndexes.cc in Sources */, @@ -4604,15 +4603,12 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 2785C26C2D012B5600F22817 /* DatabaseRegistry.cc in Sources */, 275A74D61ED3AA11008CB57B /* c4Listener.cc in Sources */, - 275A74D11ED3A4E1008CB57B /* Listener.cc in Sources */, + 27E55D292CE55963000DE11A /* HTTPListener.cc in Sources */, 275313392065844800463E74 /* RESTSyncListener_stub.cc in Sources */, - 27FC81F61EAAB7C60028E38E /* RESTListener.cc in Sources */, - 279691981ED4C3950086565D /* c4Listener+RESTFactory.cc in Sources */, - 270C6B691EB7DDAD00E73415 /* RESTListener+Replicate.cc in Sources */, - 2749B9871EB298360068DBF9 /* RESTListener+Handlers.cc in Sources */, 27FC81F91EAAB7C60028E38E /* Request.cc in Sources */, - 27175B12261B91170045F3AC /* REST_CAPI.cc in Sources */, + 27175B12261B91170045F3AC /* c4Listener_CAPI.cc in Sources */, 2796A28423072F7000774850 /* Server.cc in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/cmake/platform_android.cmake b/cmake/platform_android.cmake index 4133a3a7a..7bacfc078 100644 --- a/cmake/platform_android.cmake +++ b/cmake/platform_android.cmake @@ -70,11 +70,13 @@ function(setup_litecore_build) -DSQLITE_UNLINK_AFTER_CLOSE ) - target_link_libraries( - LiteCore PUBLIC - log - atomic - ) + if(LITECORE_BUILD_SHARED) + target_link_libraries( + LiteCore PUBLIC + log + atomic + ) + endif() target_include_directories( LiteCoreWebSocket @@ -83,15 +85,6 @@ function(setup_litecore_build) ) endfunction() -function(setup_support_build) - setup_support_build_linux() - - target_include_directories( - Support PRIVATE - LiteCore/Android - ) -endfunction() - function(setup_rest_build) # No-op endfunction() diff --git a/cmake/platform_base.cmake b/cmake/platform_base.cmake index 869aba0b8..ba7390a9c 100644 --- a/cmake/platform_base.cmake +++ b/cmake/platform_base.cmake @@ -100,6 +100,7 @@ function(set_litecore_source_base) vendor/vector_search/VectorIndexSpec.cc Replicator/c4Replicator.cc Replicator/c4Replicator_CAPI.cc + Replicator/c4ReplicatorImpl.cc Replicator/c4Socket.cc Replicator/ChangesFeed.cc Replicator/Checkpoint.cc @@ -121,6 +122,7 @@ function(set_litecore_source_base) Replicator/Worker.cc LiteCore/Support/Arena.cc LiteCore/Support/Logging.cc + LiteCore/Support/DatabasePool.cc LiteCore/Support/DefaultLogger.cc LiteCore/Support/Error.cc LiteCore/Support/EncryptedStream.cc diff --git a/cmake/platform_linux.cmake b/cmake/platform_linux.cmake index 025645ac9..c75c3c348 100644 --- a/cmake/platform_linux.cmake +++ b/cmake/platform_linux.cmake @@ -71,11 +71,6 @@ function(setup_litecore_build_linux) ${liteCoreVariant} PRIVATE -DLITECORE_USES_ICU=1 ) - - target_link_libraries( - ${liteCoreVariant} INTERFACE - ${ICU_LIBS} - ) endforeach() endif() diff --git a/cmake/platform_linux_desktop.cmake b/cmake/platform_linux_desktop.cmake index 0129e61d3..67d841d08 100644 --- a/cmake/platform_linux_desktop.cmake +++ b/cmake/platform_linux_desktop.cmake @@ -1,39 +1,55 @@ include("${CMAKE_CURRENT_LIST_DIR}/platform_linux.cmake") +MACRO (_install_gcc_file GCCFILENAME) + IF (UNIX AND NOT APPLE) + EXECUTE_PROCESS( + COMMAND "${CMAKE_CXX_COMPILER}" ${CMAKE_CXX_FLAGS} -print-file-name=${GCCFILENAME} + OUTPUT_VARIABLE _gccfile OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_VARIABLE _errormsg + RESULT_VARIABLE _failure) + IF (_failure) + MESSAGE (FATAL_ERROR "Error (${_failure}) determining path to ${GCCFILENAME}: ${_errormsg}") + ENDIF () + # We actually need to copy any files with longer filenames - this can be eg. + # libstdc++.so.6, or libgcc_s.so.1. + # Note: RPM demands that .so files be executable or else it won't + # extract debug info from them. + FILE (GLOB _gccfiles "${_gccfile}*") + FOREACH (_gccfile ${_gccfiles}) + # Weird extraneous file not desired + IF (_gccfile MATCHES ".py$") + CONTINUE () + ENDIF () + INSTALL (FILES "${_gccfile}" DESTINATION lib + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + ENDFOREACH () + ENDIF () +ENDMACRO (_install_gcc_file) + +_install_gcc_file(libstdc++.so.6) +_install_gcc_file(libgcc_s.so.1) + +IF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + SET(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install" CACHE STRING + "The install location" FORCE) + LIST(APPEND CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}") +ENDIF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + +INCLUDE (${CMAKE_CURRENT_LIST_DIR}/../jenkins/CBDeps.cmake) + +if(NOT LITECORE_DISABLE_ICU AND NOT LITECORE_DYNAMIC_ICU) + # Install cbdeps packages using cbdep tool + CBDEP_INSTALL (PACKAGE icu4c VERSION 76.1-1) + FILE (COPY "${CBDEP_icu4c_DIR}/lib" DESTINATION "${CMAKE_INSTALL_PREFIX}") +endif() + function(setup_globals) setup_globals_linux() # Enable relative RPATHs for installed bits set (CMAKE_INSTALL_RPATH "\$ORIGIN" PARENT_SCOPE) - # NOTE: We used to do a whole dog and pony show here to get clang to use libc++. Now we don't care - # and if the person using this project wants to do so then they will have to set the options - # accordingly - - if(NOT LITECORE_DISABLE_ICU AND NOT LITECORE_DYNAMIC_ICU) - set (_icu_libs) - foreach (_lib icuuc icui18n icudata) - unset (_iculib CACHE) - find_library(_iculib ${_lib} HINTS "${CBDEP_icu4c_DIR}/lib") - if (NOT _iculib) - message(FATAL_ERROR "${_lib} not found") - endif() - list(APPEND _icu_libs ${_iculib}) - endforeach() - set (ICU_LIBS ${_icu_libs} CACHE STRING "ICU libraries" FORCE) - message("Found ICU libs at ${ICU_LIBS}") - - find_path(LIBICU_INCLUDE unicode/ucol.h - HINTS "${CBDEP_icu4c_DIR}" - PATH_SUFFIXES include) - if (NOT LIBICU_INCLUDE) - message(FATAL_ERROR "libicu header files not found") - endif() - message("Using libicu header files in ${LIBICU_INCLUDE}") - include_directories("${LIBICU_INCLUDE}") - mark_as_advanced(ICU_LIBS LIBICU_INCLUDE) - endif() - find_library(ZLIB_LIB z) if (NOT ZLIB_LIB) message(FATAL_ERROR "libz not found") @@ -47,7 +63,7 @@ function(setup_globals) message("Using libz header files in ${ZLIB_INCLUDE}") mark_as_advanced( - ICU_LIBS LIBICU_INCLUDE ZLIB_LIB ZLIB_INCLUDE + ZLIB_LIB ZLIB_INCLUDE ) endfunction() @@ -76,15 +92,6 @@ function(setup_litecore_build) ${target} PRIVATE "$<$:-Wno-psabi;-Wno-odr>" ) - - # Enough is enough, we keep bumping the compiler to get newer C++ stuff - # and then suffering the consequences of that stuff not being available - # everywhere that we need. Just build it into the product directly - target_link_options( - ${target} PRIVATE - "-static-libstdc++" - "-static-libgcc" - ) endforeach() foreach(liteCoreVariant LiteCoreObjects LiteCoreUnitTesting) @@ -93,6 +100,15 @@ function(setup_litecore_build) Threads::Threads ) endforeach() + + if(NOT LITECORE_DISABLE_ICU AND NOT LITECORE_DYNAMIC_ICU) + foreach(liteCoreVariant LiteCoreObjects LiteCoreUnitTesting) + target_link_libraries( + ${liteCoreVariant} PUBLIC + icu::icu4c + ) + endforeach() + endif() endfunction() function(setup_rest_build) diff --git a/cmake/platform_unix.cmake b/cmake/platform_unix.cmake index 74ce4e551..ac78d62e0 100644 --- a/cmake/platform_unix.cmake +++ b/cmake/platform_unix.cmake @@ -39,10 +39,12 @@ function(setup_litecore_build_unix) set_property(TARGET FleeceStatic PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) endif() - set_property(TARGET LiteCore PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) + if(LITECORE_BUILD_SHARED) + set_property(TARGET LiteCore PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) + endif() endif() - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[67]") + if(LITECORE_BUILD_SHARED AND CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[67]") # C/C++ atomic operations on ARM6/7 emit calls to functions in libatomic target_link_libraries( LiteCore PRIVATE @@ -65,12 +67,13 @@ function(setup_litecore_build_unix) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LITECORE_SAN_FLAGS}" CACHE INTERNAL "") # The linker also needs to be told, so it can link the appropriate sanitizer runtime libs: - foreach(target LiteCore CppTests C4Tests) - target_link_options(${target} PRIVATE - -fsanitize=address - -fsanitize=undefined - ) - endforeach () + if (LITECORE_BUILD_SHARED) + target_link_options(LiteCore PRIVATE -fsanitize=address -fsanitize=undefined) + endif() + if (LITECORE_BUILD_TESTS) + target_link_options(CppTests PRIVATE -fsanitize=address -fsanitize=undefined) + target_link_options(C4Tests PRIVATE -fsanitize=address -fsanitize=undefined) + endif() else() set(LITECORE_COMPILE_OPTIONS -fstack-protector diff --git a/jenkins/CBDeps.cmake b/jenkins/CBDeps.cmake index 8f665f7d6..4dcbcee3e 100644 --- a/jenkins/CBDeps.cmake +++ b/jenkins/CBDeps.cmake @@ -57,7 +57,7 @@ IF (NOT CBDeps_INCLUDED) EXECUTE_PROCESS ( COMMAND "${CBDEP_EXE}" -p linux - install -d "${cbdep_INSTALL_DIR}" + install -C -d "${cbdep_INSTALL_DIR}" ${cbdep_PACKAGE} ${cbdep_VERSION} RESULT_VARIABLE _cbdep_result OUTPUT_VARIABLE _cbdep_out @@ -72,6 +72,10 @@ IF (NOT CBDeps_INCLUDED) CACHE STRING "Version of cbdep package '${cbdep_PACKAGE}'" FORCE) SET (CBDEP_${cbdep_PACKAGE}_DIR "${cbdep_TARGET_DIR}" CACHE STRING "Install location of cbdep package '${cbdep_PACKAGE}'" FORCE) + IF (IS_DIRECTORY "${cbdep_TARGET_DIR}/cmake") + SET ("${cbdep_PACKAGE}_ROOT" "${cbdep_TARGET_DIR}") + FIND_PACKAGE (${cbdep_PACKAGE} REQUIRED) + ENDIF () MESSAGE (STATUS "Using cbdeps package ${cbdep_PACKAGE} ${cbdep_VERSION}") ENDFUNCTION (CBDEP_INSTALL) diff --git a/jenkins/CMakeLists.txt b/jenkins/CMakeLists.txt deleted file mode 100644 index 4af50da44..000000000 --- a/jenkins/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# This is the top level CMake project for the Couchbase build server -# which uses internal tooling in order to download binary deps - -CMAKE_MINIMUM_REQUIRED (VERSION 3.19) -CMAKE_POLICY (VERSION 3.19) - -# Tell CMake to use headers / frameworks from SDK inside XCode instead of -# the ones found on the system (for weak linking). Ignored on non-Apple -SET(CMAKE_OSX_SYSROOT macosx) - -# Top-level CMakeLists for Couchbase Lite Core -PROJECT (couchbase-lite-core-build) - -# Provide reasonable default for CMAKE_INSTALL_PREFIX -IF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - SET(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install" CACHE STRING - "The install location" FORCE) - LIST(APPEND CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}") -ENDIF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - -# Download additional cbdeps packages -if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - INCLUDE (couchbase-lite-core/jenkins/CBDeps.cmake) - - # Install cbdeps packages using cbdep tool - CBDEP_INSTALL (PACKAGE icu4c VERSION 71.1-2) - FILE (COPY "${CBDEP_icu4c_DIR}/lib" DESTINATION "${CMAKE_INSTALL_PREFIX}") -endif() - -if("${EDITION}" STREQUAL "enterprise") - message(STATUS "Building Enterprise Edition...") - set(BUILD_ENTERPRISE ON CACHE BOOL "Set whether or not to build enterprise edition") -else() - message(STATUS "Building Community Edition because EDITION was set to ${EDITION}...") -endif() - -ADD_SUBDIRECTORY (couchbase-lite-core) \ No newline at end of file diff --git a/jenkins/build_server_android.sh b/jenkins/build_server_android.sh index d89e954c3..d3c212b24 100755 --- a/jenkins/build_server_android.sh +++ b/jenkins/build_server_android.sh @@ -58,6 +58,14 @@ if [ -z "$EDITION" ]; then usage fi +if [[ "$EDITION" == "enterprise" ]]; then + echo "Building enterprise edition (EDITION = enterprise)" + build_enterprise="ON" +else + echo "Building community edition (EDITION = $EDITION)" + build_enterprise="OFF" +fi + SHA_VERSION="$7" if [ -z "$SHA_VERSION" ]; then usage @@ -92,10 +100,7 @@ if [ ! -f ${NINJA} ]; then .tools/cbdep install -d .tools ninja ${NINJA_VER} fi -ARCH_VERSION="19" -if [[ "${ANDROID_ARCH}" == "x86_64" ]] || [[ "${ANDROID_ARCH}" == "arm64-v8a" ]]; then - ARCH_VERSION="21" -fi +ARCH_VERSION="22" #create artifacts dir for publishing to latestbuild ARTIFACTS_SHA_DIR=${WORKSPACE}/artifacts/couchbase-lite-core/sha/${SHA_VERSION:0:2}/${SHA_VERSION} @@ -111,7 +116,7 @@ ${CMAKE} \ -DCMAKE_MAKE_PROGRAM="${NINJA}" \ -DANDROID_PLATFORM=${ARCH_VERSION} \ -DANDROID_ABI=${ANDROID_ARCH} \ - -DEDITION=${EDITION} \ + -DBUILD_ENTERPRISE=${build_enterprise} \ -DCMAKE_INSTALL_PREFIX=`pwd`/install \ -DCMAKE_BUILD_TYPE=MinSizeRel \ .. @@ -126,7 +131,7 @@ ${CMAKE} \ -DCMAKE_MAKE_PROGRAM="${NINJA}" \ -DANDROID_PLATFORM=${ARCH_VERSION} \ -DANDROID_ABI=${ANDROID_ARCH} \ - -DEDITION=${EDITION} \ + -DBUILD_ENTERPRISE=${build_enterprise} \ -DCMAKE_INSTALL_PREFIX=`pwd`/install \ -DCMAKE_BUILD_TYPE=Debug \ .. diff --git a/jenkins/build_server_unix.sh b/jenkins/build_server_unix.sh index 923393093..5421eff8c 100755 --- a/jenkins/build_server_unix.sh +++ b/jenkins/build_server_unix.sh @@ -40,25 +40,19 @@ case "${OSTYPE}" in PKG_CMD='tar czf' PKG_TYPE='tar.gz' PROP_FILE=${WORKSPACE}/publish.prop - OS_NAME=`lsb_release -is` - if [[ "$OS_NAME" != "CentOS" ]]; then - echo "Error: Unsupported Linux distro $OS_NAME" - exit 2 - fi - - OS_VERSION=`lsb_release -rs` - if [[ $OS_VERSION =~ ^6.* ]]; then - OS="centos6" - elif [[ ! $OS_VERSION =~ ^7.* ]]; then - echo "Error: Unsupported CentOS version $OS_VERSION" - exit 3 - fi;; + ;; *) echo "unknown: $OSTYPE" exit 1;; esac -project_dir=couchbase-lite-core -strip_dir=${project_dir} +if [[ "$EDITION" == "enterprise" ]]; then + echo "Building enterprise edition (EDITION = enterprise)" + build_enterprise="ON" +else + echo "Building community edition (EDITION = $EDITION)" + build_enterprise="OFF" +fi + ios_xcode_proj="couchbase-lite-core/Xcode/LiteCore.xcodeproj" macosx_lib="libLiteCore.dylib" @@ -121,34 +115,22 @@ build_binaries () { CMAKE_BUILD_TYPE_NAME="cmake_build_type_${FLAVOR}" mkdir -p ${WORKSPACE}/build_${FLAVOR} pushd ${WORKSPACE}/build_${FLAVOR} - cmake -DEDITION=${EDITION} -DCMAKE_INSTALL_PREFIX=`pwd`/install -DCMAKE_BUILD_TYPE=${!CMAKE_BUILD_TYPE_NAME} -DLITECORE_MACOS_FAT_DEBUG=ON .. + cmake -DBUILD_ENTERPRISE=$build_enterprise -DCMAKE_INSTALL_PREFIX=`pwd`/install -DCMAKE_BUILD_TYPE=${!CMAKE_BUILD_TYPE_NAME} -DLITECORE_MACOS_FAT_DEBUG=ON ../couchbase-lite-core make -j8 - if [[ ${OS} == 'linux' ]] || [[ ${OS} == 'centos6' ]]; then - ${WORKSPACE}/couchbase-lite-core/build_cmake/scripts/strip.sh ${strip_dir} + if [[ ${OS} == 'linux' ]]; then + ${WORKSPACE}/couchbase-lite-core/build_cmake/scripts/strip.sh $PWD else - pushd ${project_dir} dsymutil ${macosx_lib} -o libLiteCore.dylib.dSYM strip -x ${macosx_lib} - popd fi make install if [[ ${OS} == 'macosx' ]]; then # package up the strip symbols - cp -rp ${project_dir}/libLiteCore.dylib.dSYM ./install/lib - else - # copy C++ stdlib, etc to output - libstdcpp=`g++ --print-file-name=libstdc++.so` - libstdcppname=`basename "$libstdcpp"` - libgcc_s=`gcc --print-file-name=libgcc_s.so` - libgcc_sname=`basename "$libgcc_s"` - - cp -p "$libstdcpp" "./install/lib/$libstdcppname" - ln -s "$libstdcppname" "./install/lib/${libstdcppname}.6" - cp -p "${libgcc_s}" "./install/lib" + cp -rp libLiteCore.dylib.dSYM ./install/lib fi if [[ -z ${SKIP_TESTS} ]] && [[ ${EDITION} == 'enterprise' ]]; then chmod 777 ${WORKSPACE}/couchbase-lite-core/build_cmake/scripts/test_unix.sh - cd ${WORKSPACE}/build_${FLAVOR}/${project_dir} && ${WORKSPACE}/couchbase-lite-core/build_cmake/scripts/test_unix.sh + cd ${WORKSPACE}/build_${FLAVOR}/ && ${WORKSPACE}/couchbase-lite-core/build_cmake/scripts/test_unix.sh fi popd } @@ -175,7 +157,7 @@ create_pkgs () { ${PKG_CMD} ${WORKSPACE}/${SYMBOLS_PKG_NAME} lib/libLiteCore.dylib.dSYM else # linux ${PKG_CMD} ${WORKSPACE}/${PACKAGE_NAME} * - cd ${WORKSPACE}/build_${FLAVOR}/${strip_dir} + cd ${WORKSPACE}/build_${FLAVOR}/ ${PKG_CMD} ${WORKSPACE}/${SYMBOLS_PKG_NAME} libLiteCore*.sym fi popd diff --git a/jenkins/build_server_win.ps1 b/jenkins/build_server_win.ps1 index 80fb79fe4..24e5e9688 100644 --- a/jenkins/build_server_win.ps1 +++ b/jenkins/build_server_win.ps1 @@ -78,9 +78,15 @@ function Build() { $MsArch = "x64" } + if($Edition -eq "enterprise") { + $build_enterprise = "ON" + } else { + $build_enterprise = "OFF" + } + & "C:\Program Files\CMake\bin\cmake.exe" ` -A $MsArch ` - -DEDITION="$Edition" ` + -DBUILD_ENTERPRISE=$build_enterprise ` -DCMAKE_INSTALL_PREFIX="$(Get-Location)\install" ` .. diff --git a/vendor/fleece b/vendor/fleece index ebe000ea5..6601272e3 160000 --- a/vendor/fleece +++ b/vendor/fleece @@ -1 +1 @@ -Subproject commit ebe000ea5a6a084c43170b743a31b5d35c2f93f9 +Subproject commit 6601272e3f5632873d6b186386381f2acd54d4e6 diff --git a/vendor/mbedtls b/vendor/mbedtls index 18cd3603d..c211590fa 160000 --- a/vendor/mbedtls +++ b/vendor/mbedtls @@ -1 +1 @@ -Subproject commit 18cd3603d8a0879315b1bd56a2517a34b2673570 +Subproject commit c211590fa64f66ba582bb3f9a87ff0d09286578c