From 27d3530bfc3e9dabd43735f7489059d8002b07d3 Mon Sep 17 00:00:00 2001 From: callumbirks <73551885+callumbirks@users.noreply.github.com> Date: Wed, 18 Dec 2024 18:30:30 +0000 Subject: [PATCH] CBL-6289: Re-enable Actions Replication tests (#2190) --- C/include/c4Document.h | 1 + Replicator/Replicator.cc | 14 + Replicator/tests/ReplParams.cc | 3 +- Replicator/tests/ReplParams.hh | 5 +- Replicator/tests/ReplicatorAPITest.hh | 4 +- .../tests/ReplicatorCollectionSGTest.cc | 754 ++---------------- .../tests/ReplicatorCollectionSGTest.hh | 48 +- Replicator/tests/ReplicatorSG30Test.cc | 7 +- Replicator/tests/SG.cc | 23 +- Replicator/tests/SG.hh | 6 + Replicator/tests/data/docker/sg/Dockerfile | 18 +- .../sg/config/collections/bootstrap.json | 2 +- 12 files changed, 173 insertions(+), 712 deletions(-) diff --git a/C/include/c4Document.h b/C/include/c4Document.h index 09408dfd4..0c88b9d7c 100644 --- a/C/include/c4Document.h +++ b/C/include/c4Document.h @@ -138,6 +138,7 @@ CBL_CORE_API C4SliceResult c4doc_getRevisionHistory(C4Document* doc, unsigned ma CBL_CORE_API C4SliceResult c4doc_getSelectedRevIDGlobalForm(C4Document* doc) C4API; /** Selects the parent of the selected revision, if it's known, else returns false. + * Throws if the document is a version-vectors document. \note The caller must use a lock for Document when this function is called. */ NODISCARD CBL_CORE_API bool c4doc_selectParentRevision(C4Document* doc) C4API; diff --git a/Replicator/Replicator.cc b/Replicator/Replicator.cc index 25f94d83a..ccb6bb2d3 100644 --- a/Replicator/Replicator.cc +++ b/Replicator/Replicator.cc @@ -589,6 +589,20 @@ namespace litecore::repl { "Incompatible replication protocol " "(missing 'Sec-WebSocket-Protocol' response header)"_sl)); } + + const auto& compats = repl::kCompatProtocols; + + string acceptedProtocol; + stringstream s(headers["Sec-WebSocket-Protocol"].asString()); + string protocol; + while ( getline(s, protocol, ',') ) { + auto i = std::find(compats.begin(), compats.end(), protocol); + if ( i != compats.end() ) { + acceptedProtocol = protocol; + break; + } + } + if ( _delegate ) _delegate->replicatorGotHTTPResponse(this, status, headers); if ( slice x_corr = headers.get("X-Correlation-Id"_sl); x_corr ) { _correlationID = x_corr; diff --git a/Replicator/tests/ReplParams.cc b/Replicator/tests/ReplParams.cc index adf1dd6c0..194d9f6bd 100644 --- a/Replicator/tests/ReplParams.cc +++ b/Replicator/tests/ReplParams.cc @@ -3,6 +3,7 @@ // #include "ReplParams.hh" +#include ReplParams::ReplParams(const std::vector& collections_) : C4ReplicatorParameters() { _collectionVector = {collections_}; @@ -70,7 +71,7 @@ ReplParams& ReplParams::setCollectionOptions(C4CollectionSpec collectionSpec, co return *this; } -ReplParams& ReplParams::setDocIDs(const std::vector>& docIDs) { +ReplParams& ReplParams::setDocIDs(const std::vector>& docIDs) { for ( size_t i = 0; i < docIDs.size(); ++i ) { fleece::Encoder enc; enc.beginArray(); diff --git a/Replicator/tests/ReplParams.hh b/Replicator/tests/ReplParams.hh index 2ab7795e8..de435b06c 100644 --- a/Replicator/tests/ReplParams.hh +++ b/Replicator/tests/ReplParams.hh @@ -12,6 +12,7 @@ #include "fleece/Expert.hh" #include "c4ReplicatorTypes.h" #include "ReplicatorOptions.hh" +#include #include #include @@ -57,11 +58,11 @@ class ReplParams : public C4ReplicatorParameters { // Set an option for all collections ReplParams& setCollectionOptions(const AllocedDict& options); // Set docIDs in options of each collection - ReplParams& setDocIDs(const std::vector>& docIDs); + ReplParams& setDocIDs(const std::vector>& docIDs); // Same as above, with array parameter template - ReplParams& setDocIDs(const std::array, N>& docIDs) { + ReplParams& setDocIDs(const std::array, N>& docIDs) { return setDocIDs({docIDs.begin(), docIDs.end()}); } diff --git a/Replicator/tests/ReplicatorAPITest.hh b/Replicator/tests/ReplicatorAPITest.hh index e5c999897..7c19afa16 100644 --- a/Replicator/tests/ReplicatorAPITest.hh +++ b/Replicator/tests/ReplicatorAPITest.hh @@ -58,7 +58,9 @@ class ReplicatorAPITest : public C4Test { static std::once_flag once; - ReplicatorAPITest() : C4Test(0), _sg({kDefaultAddress, kScratchDBName}) { + ReplicatorAPITest() : ReplicatorAPITest(0) {} + + explicit ReplicatorAPITest(int option) : C4Test(option), _sg({kDefaultAddress, kScratchDBName}) { std::call_once(once, [&]() { // Register the BuiltInWebSocket class as the C4Replicator's WebSocketImpl. C4RegisterBuiltInWebSocket(); diff --git a/Replicator/tests/ReplicatorCollectionSGTest.cc b/Replicator/tests/ReplicatorCollectionSGTest.cc index c98a71687..2e55d749d 100644 --- a/Replicator/tests/ReplicatorCollectionSGTest.cc +++ b/Replicator/tests/ReplicatorCollectionSGTest.cc @@ -10,6 +10,7 @@ // the file licenses/APL2.txt. // +#include #include #include "ReplicatorCollectionSGTest.hh" @@ -17,6 +18,7 @@ #include "Base64.hh" #include "ReplicatorTypes.hh" #include "c4ReplicatorTypes.h" +#include "catch.hpp" #include // Tests in this file, tagged by [.SyncServerCollection], are not done automatically in the @@ -112,6 +114,7 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "API Push 5000 Changes Collections for ( int i = 2; i <= revisionCount; ++i ) { *revID = createNewRev(coll, slice(docID), slice(*revID), kFleeceBody); REQUIRE(!revID->empty()); + C4Log("Created rev %s", revID->c_str()); } ++revID; } @@ -316,8 +319,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Multiple Collections Push & Pull S initTest({Lavenders, Roses, Tulips}); - std::vector> docIDs{_collectionCount}; - std::vector> localDocIDs{_collectionCount}; + std::vector> docIDs{_collectionCount}; + std::vector> localDocIDs{_collectionCount}; for ( size_t i = 0; i < _collectionCount; ++i ) { addDocs(_collections[i], 20, idPrefix + "remote-"); @@ -618,8 +621,10 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull deltas from Collection SG", " while ( c4enum_next(e, ERROR_INFO(error)) ) { C4DocumentInfo info; c4enum_getDocumentInfo(e, &info); - CHECK(slice(info.docID).hasPrefix(slice(docIDPref))); - CHECK(slice(info.revID).hasPrefix("2-"_sl)); + alloc_slice docID = info.docID; + CHECK(docID.hasPrefix(slice(docIDPref))); + alloc_slice revID = getLegacyRevID(_collectionSpecs[i], info); + CHECK(c4rev_getGeneration(revID) == 2); ++n; } CHECK(error.code == 0); @@ -677,18 +682,18 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Push & Pull Deletion SG", "[.SyncS createRev(coll, slice(docID), kRev2ID, kEmptyFleeceBody, kRevDeleted); } - std::vector> docIDs{_collectionCount}; + std::vector> docIDs{_collectionCount}; for ( size_t i = 0; i < _collectionCount; ++i ) { - docIDs[i] = unordered_map{{alloc_slice(docID), 0}}; + docIDs[i] = unordered_map{{alloc_slice(docID), 0}}; } ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; replParams.setDocIDs(docIDs); replicate(replParams); - C4Log("-------- Deleting and re-creating database --------"); - deleteAndRecreateDBAndCollections(); + C4Log("-------- Purging & Recreating doc --------"); + for ( auto& coll : _collections ) { REQUIRE(c4coll_purgeDoc(coll, slice(docID), ERROR_INFO())); } for ( size_t i = 0; i < _collectionCount; ++i ) { createRev(_collections[i], slice(docID), kRevID, kFleeceBody); } @@ -701,141 +706,7 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Push & Pull Deletion SG", "[.SyncS CHECK(remoteDoc->revID == kRev2ID); CHECK((remoteDoc->flags & kDocDeleted) != 0); CHECK((remoteDoc->selectedRev.flags & kRevDeleted) != 0); - REQUIRE(c4doc_selectParentRevision(remoteDoc)); - CHECK(remoteDoc->selectedRev.revID == kRevID); - } -} - -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Resolve Conflict SG", "[.SyncServerCollection]") { - const string idPrefix = timePrefix(); - - initTest({Roses, Tulips, Lavenders}); - - std::vector collNames{_collectionCount}; - - for ( size_t i = 0; i < _collectionCount; ++i ) { - collNames[i] = idPrefix + Options::collectionSpecToPath(_collectionSpecs[i]).asString(); - createFleeceRev(_collections[i], slice(collNames[i]), kRev1ID, "{}"_sl); - createFleeceRev(_collections[i], slice(collNames[i]), revOrVersID("2-12121212", "1@cafe"), - R"({"db":"remote"})"_sl); - } - - updateDocIDs(); - - // Send the docs to remote - ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; - replParams.setDocIDs(_docIDs); - replicate(replParams); - - verifyDocs(_docIDs, true); - - deleteAndRecreateDBAndCollections(); - - for ( size_t i = 0; i < _collectionCount; ++i ) { - createFleeceRev(_collections[i], slice(collNames[i]), kRev1ID, "{}"_sl); - createFleeceRev(_collections[i], slice(collNames[i]), revOrVersID("2-13131313", "1@babe"), - R"({"db":"local"})"_sl); - } - - updateDocIDs(); // Might need to delete this - replParams.setDocIDs(_docIDs); - - _conflictHandler = [&](const C4DocumentEnded* docEndedWithConflict) { - C4Error error; - int i = -1; - for ( int k = 0; k < _collectionCount; ++k ) { - if ( docEndedWithConflict->collectionSpec == _collectionSpecs[k] ) { i = k; } - } - Assert(i >= 0, "Internal logical error"); - - TransactionHelper t(db); - - slice docID = docEndedWithConflict->docID; - // Get the local doc. It is the current revision - c4::ref localDoc = c4coll_getDoc(_collections[i], docID, true, kDocGetAll, WITH_ERROR(error)); - CHECK(error.code == 0); - - // Get the remote doc. It is the next leaf revision of the current revision. - c4::ref remoteDoc = c4coll_getDoc(_collections[i], docID, true, kDocGetAll, &error); - bool succ = c4doc_selectNextLeafRevision(remoteDoc, true, false, &error); - Assert(remoteDoc->selectedRev.revID == docEndedWithConflict->revID); - CHECK(error.code == 0); - CHECK(succ); - - C4Document* resolvedDoc = remoteDoc; - - FLDict mergedBody = c4doc_getProperties(resolvedDoc); - C4RevisionFlags mergedFlags = resolvedDoc->selectedRev.flags; - alloc_slice winRevID = resolvedDoc->selectedRev.revID; - alloc_slice lostRevID = (resolvedDoc == remoteDoc) ? localDoc->selectedRev.revID : remoteDoc->selectedRev.revID; - bool result = c4doc_resolveConflict2(localDoc, winRevID, lostRevID, mergedBody, mergedFlags, &error); - Assert(result, "conflictHandler: c4doc_resolveConflict2 failed for '%.*s' in '%.*s.%.*s'", SPLAT(docID), - SPLAT(_collectionSpecs[i].scope), SPLAT(_collectionSpecs[i].name)); - Assert((localDoc->flags & kDocConflicted) == 0); - - if ( !c4doc_save(localDoc, 0, &error) ) { - Assert(false, "conflictHandler: c4doc_save failed for '%.*s' in '%.*s.%.*s'", SPLAT(docID), - SPLAT(_collectionSpecs[i].scope), SPLAT(_collectionSpecs[i].name)); - } - }; - - replParams.setPushPull(kC4Disabled, kC4OneShot); - replicate(replParams); - - for ( size_t i = 0; i < _collectionCount; ++i ) { - c4::ref doc = c4coll_getDoc(_collections[i], slice(collNames[i]), true, kDocGetAll, nullptr); - REQUIRE(doc); - CHECK(fleece2json(c4doc_getRevisionBody(doc)) == "{db:\"remote\"}"); // Remote Wins - REQUIRE(!c4doc_selectNextLeafRevision(doc, true, false, nullptr)); - } -} - -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Update Once-Conflicted Doc - SGColl", "[.SyncServerCollection]") { - const string idPrefix = timePrefix(); - const string docID = idPrefix + "uocd-doc"; - const string channelID = idPrefix + "uocd"; - - initTest({Roses, Tulips, Lavenders}, {channelID}); - - std::array bodies{R"({"_rev":"1-aaaa","foo":1})", - R"({"_revisions":{"start":2,"ids":["bbbb","aaaa"]},"foo":2.1})", - R"({"_revisions":{"start":2,"ids":["cccc","aaaa"]},"foo":2.2})", - R"({"_revisions":{"start":3,"ids":["dddd","cccc"]},"_deleted":true})"}; - - // Create a conflicted doc on SG, and resolve the conflict - for ( auto& spec : _collectionSpecs ) { - for ( const auto& body : bodies ) { _sg.upsertDoc(spec, docID + "?new_edits=false", body, {channelID}); } - } - - // Pull doc into CBL: - C4Log("-------- Pulling"); - ReplParams replParams{_collectionSpecs, kC4Disabled, kC4OneShot}; - replicate(replParams); - - // Verify doc: - for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(docID), true, kDocGetAll, nullptr); - REQUIRE(doc); - C4Slice revID = C4STR("2-bbbb"); - CHECK(doc->revID == revID); - CHECK((doc->flags & kDocDeleted) == 0); - REQUIRE(c4doc_selectParentRevision(doc)); - CHECK(doc->selectedRev.revID == "1-aaaa"_sl); } - - // Update doc: - alloc_slice body = SG::addChannelToJSON(R"({"ans*wer":42})"_sl, "channels"_sl, {channelID}); - const std::string bodyStr = body.asString(); - for ( auto& coll : _collections ) { createRev(coll, slice(docID), "3-ffff"_sl, json2fleece(bodyStr.c_str())); } - - // Push change back to SG: - C4Log("-------- Pushing"); - replParams.setPushPull(kC4OneShot, kC4Disabled); - replicate(replParams); - - updateDocIDs(); - - verifyDocs(_docIDs, true); } TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled - Filter Revoked Revision - SGColl", @@ -1047,13 +918,10 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled - Revoke Access for ( int i = 0; i < _collectionCount; ++i ) { // Verify - c4::ref doc1 = c4coll_getDoc(_collections[i], slice(docIDstr), true, kDocGetAll, nullptr); - REQUIRE(doc1); - CHECK(slice(doc1->revID).hasPrefix("1-"_sl)); - + alloc_slice revID = getLegacyRevID(_collections[i], slice(docIDstr)); + CHECK(c4rev_getGeneration(revID) == 1); // Update doc to only channel 'b' - auto oRevID = slice(doc1->revID).asString(); - REQUIRE(_sg.upsertDoc(_collectionSpecs[i], docIDstr, oRevID, "{}", {channelIDb})); + REQUIRE(_sg.upsertDoc(_collectionSpecs[i], docIDstr, revID.asString(), "{}", {channelIDb})); } C4Log("-------- Pull update"); @@ -1061,9 +929,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled - Revoke Access // Verify the update: for ( auto& coll : _collections ) { - c4::ref doc1 = c4coll_getDoc(coll, slice(docIDstr), true, kDocGetAll, nullptr); - REQUIRE(doc1); - CHECK(slice(doc1->revID).hasPrefix("2-"_sl)); + alloc_slice revID = getLegacyRevID(coll, slice(docIDstr)); + CHECK(revID.hasPrefix("2-"_sl)); } CHECK(_docsEnded == 0); CHECK(_counter == 0); @@ -1707,13 +1574,11 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Remove Doc From Channel SG", "[.Sy for ( int i = 0; i < _collectionCount; ++i ) { // Verify doc - c4::ref doc1 = c4coll_getDoc(_collections[i], slice(doc1ID), true, kDocGetCurrentRev, nullptr); - REQUIRE(doc1); - CHECK(c4rev_getGeneration(doc1->revID) == 1); + alloc_slice revID = getLegacyRevID(_collections[i], slice(doc1ID)); + CHECK(c4rev_getGeneration(revID) == 1); // Once verified, remove it from channel 'a' in that collection - auto oRevID = slice(doc1->revID).asString(); - _sg.upsertDoc(_collectionSpecs[i], doc1ID, R"({"_rev":")" + oRevID + "\"}", {chIDs[1]}); + _sg.upsertDoc(_collectionSpecs[i], doc1ID, revID.asString(), "{}", {chIDs[1]}); } C4Log("-------- Pull update"); @@ -1727,13 +1592,11 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Remove Doc From Channel SG", "[.Sy for ( int i = 0; i < _collectionCount; ++i ) { // Verify the update: - c4::ref doc1 = c4coll_getDoc(_collections[i], slice(doc1ID), true, kDocGetCurrentRev, nullptr); - REQUIRE(doc1); - CHECK(c4rev_getGeneration(doc1->revID) == 2); + alloc_slice revID = getLegacyRevID(_collections[i], slice(doc1ID)); + CHECK(c4rev_getGeneration(revID) == 2); // Remove doc from all channels: - auto oRevID = slice(doc1->revID).asString(); - _sg.upsertDoc(_collectionSpecs[i], doc1ID, R"({"_rev":")" + oRevID + "\"}", {}); + _sg.upsertDoc(_collectionSpecs[i], doc1ID, revID.asString(), "{}", {}); } C4Log("-------- Pull the removed"); @@ -1828,7 +1691,7 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled - Filter Remove // Remove doc from all channels auto oRevID = slice(doc1->revID).asString(); - _sg.upsertDoc(_collectionSpecs[i], doc1ID, R"({"_rev":")" + oRevID + "\"}", {}); + _sg.upsertDoc(_collectionSpecs[i], doc1ID, oRevID, "{}", {}); } C4Log("-------- Pull the removed"); @@ -1892,6 +1755,7 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled(default) - Dele replicate(replParams); bool deleteThenCreate = true; + SECTION("Delete then Create Doc") { // Create a new doc with the same id that was deleted: { @@ -1928,79 +1792,6 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Auto Purge Enabled(default) - Dele } } -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "API Push Conflict SG", "[.SyncServerCollection]") { - const string originalRevID = "1-3cb9cfb09f3f0b5142e618553966ab73539b8888"; - const string idPrefix = timePrefix(); - const string doc13ID = idPrefix + "0000013"; - - initTest({Roses, Tulips, Lavenders}); - - for ( auto coll : _collections ) { importJSONLines(sFixturesDir + "names_100.json", coll, 0, false, 0, idPrefix); } - - updateDocIDs(); - - // Push to remote - ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; - replicate(replParams); - - // Update doc 13 on the remote - string body = R"({"_rev":")" + originalRevID + R"(","serverSideUpdate":true})"; - for ( auto& spec : _collectionSpecs ) { REQUIRE(_sg.upsertDoc(spec, doc13ID, slice(body), {})); } - - for ( auto& coll : _collections ) { - // Create a conflict doc13 at local - createRev(coll, slice(doc13ID), "2-f000"_sl, kFleeceBody); - // Verify doc - c4::ref doc = c4coll_getDoc(coll, slice(doc13ID), true, kDocGetAll, nullptr); - REQUIRE(doc); - C4Slice revID = C4STR("2-f000"); - CHECK(doc->selectedRev.revID == revID); - CHECK(c4doc_getProperties(doc) != nullptr); - REQUIRE(c4doc_selectParentRevision(doc)); - revID = slice(originalRevID); - CHECK(doc->selectedRev.revID == revID); - CHECK(c4doc_getProperties(doc) != nullptr); - CHECK((doc->selectedRev.flags & kRevKeepBody) != 0); - } - - - C4Log("-------- Pushing Again (conflict) --------"); - _expectedDocPushErrors = {doc13ID}; - replicate(replParams); - - C4Log("-------- Pulling --------"); - replParams.setPushPull(kC4Disabled, kC4OneShot); - replParams.setDocIDs(_docIDs); - - _expectedDocPushErrors = {}; - _expectedDocPullErrors = {doc13ID}; - replicate(replParams); - - C4Log("-------- Checking Conflict --------"); - for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(doc13ID), true, kDocGetAll, nullptr); - REQUIRE(doc); - CHECK((doc->flags & kDocConflicted) != 0); - C4Slice revID = C4STR("2-f000"); - CHECK(doc->selectedRev.revID == revID); - CHECK(c4doc_getProperties(doc) != nullptr); - REQUIRE(c4doc_selectParentRevision(doc)); - revID = slice(originalRevID); - CHECK(doc->selectedRev.revID == revID); - CHECK(c4doc_getProperties(doc) != nullptr); - CHECK((doc->selectedRev.flags & kRevKeepBody) != 0); - REQUIRE(c4doc_selectCurrentRevision(doc)); - REQUIRE(c4doc_selectNextRevision(doc)); - revID = C4STR("2-883a2dacc15171a466f76b9d2c39669b"); - CHECK(doc->selectedRev.revID == revID); - CHECK((doc->selectedRev.flags & kRevIsConflict) != 0); - CHECK(c4doc_getProperties(doc) != nullptr); - REQUIRE(c4doc_selectParentRevision(doc)); - revID = slice(originalRevID); - CHECK(doc->selectedRev.revID == revID); - } -} - TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull multiply-updated SG", "[.SyncServerCollection]") { // From : // 1. Setup CB cluster & Configure SG @@ -2024,10 +1815,10 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull multiply-updated SG", "[.Sync _sg.upsertDoc(spec, docID + "?new_edits=false", R"({"count":1, "_rev":"1-1111"})"); } - std::vector> docIDs{_collectionCount}; + std::vector> docIDs{_collectionCount}; for ( size_t i = 0; i < _collectionCount; ++i ) { - docIDs[i] = unordered_map{{alloc_slice(docID), 0}}; + docIDs[i] = unordered_map{{alloc_slice(docID), 0}}; } ReplParams replParams{_collectionSpecs, kC4Disabled, kC4OneShot}; @@ -2036,9 +1827,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull multiply-updated SG", "[.Sync CHECK(_callbackStatus.progress.documentCount == _collectionCount); for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(docID), true, kDocGetCurrentRev, nullptr); - REQUIRE(doc); - CHECK(doc->revID == "1-1111"_sl); + alloc_slice revID = getLegacyRevID(coll, slice(docID)); + CHECK(c4rev_getGeneration(revID) == 1); } const std::array bodies{R"({"count":2, "_rev":"1-1111"})", @@ -2051,9 +1841,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull multiply-updated SG", "[.Sync replicate(replParams); for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(docID), true, kDocGetCurrentRev, nullptr); - REQUIRE(doc); - CHECK(doc->revID == "4-ffa3011c5ade4ec3a3ec5fe2296605ce"_sl); + alloc_slice revID = getLegacyRevID(coll, slice(docID)); + CHECK(c4rev_getGeneration(revID) == 4); } } @@ -2146,8 +1935,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull iTunes deltas from Collection while ( c4enum_next(e, ERROR_INFO(error)) ) { C4DocumentInfo info; c4enum_getDocumentInfo(e, &info); - auto revID = slice(info.revID); - CHECK(revID.hasPrefix("2-"_sl)); + alloc_slice revID = getLegacyRevID(coll->getSpec(), info); + CHECK(c4rev_getGeneration(revID) == 2); ++n; } CHECK(error.code == 0); @@ -2159,299 +1948,8 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull iTunes deltas from Collection timeWithoutDelta / timeWithDelta); } -// Test replication and conflict resolution behaviour when we send SG a rev history with a gap (number of new -// revs between syncs > repl::tuning::kDefaultMaxHistory). -// This test attempts to emulate two separate clients with the same doc, where one client's rev history lands -// in the gap of the other client's rev history. -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Give SG a rev history with a gap", "[.xSyncServerCollection]") { - constexpr size_t maxHistory = tuning::kDefaultMaxHistory; - constexpr size_t numInitialRevs = 2; - constexpr const char* saveDBName = "revsgap"; - const string docID = timePrefix() + "doc1"; - - size_t numRevs1 = 0; - size_t numRevs2 = 0; - - SECTION("No gap in history") { - numRevs1 = maxHistory - 5; - numRevs2 = numRevs1 - 10; - } - - SECTION("Gap in history") { - numRevs1 = maxHistory + 50; - // num revs of the second 'client' land within the gap of the first 'client' history - numRevs2 = numRevs1 - (maxHistory + 10); - } - - initTest({Roses, Tulips, Lavenders}); - - // Initial doc - for ( auto& coll : _collections ) { - for ( size_t i = 0; i < numInitialRevs; ++i ) { - createFleeceRev(coll, slice(docID), nullslice, slice(R"({"a":)" + to_string(i) + "}")); - } - } - - // Replication parameters and docID filter - updateDocIDs(); - ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; - replParams.setDocIDs(_docIDs); - // Push doc to remote - replicate(replParams); - - // Save current db state for later - alloc_slice path(c4db_getPath(db)); - REQUIRE(c4db_copyNamed(path, slice(saveDBName), &dbConfig(), ERROR_INFO())); - - // Create 'client 1' mutations - for ( auto& coll : _collections ) { - for ( size_t i = numInitialRevs; i < numRevs1 + numInitialRevs; ++i ) { - createFleeceRev(coll, slice(docID), nullslice, slice(R"({"a":)" + to_string(i) + "}")); - } - } - - // Sync with remote - replParams.setPushPull(kC4OneShot, kC4OneShot); - replicate(replParams); - - // Reload db from save (back to initial doc with `numInitialRevs` revs) - c4db_release(db); - db = c4db_openNamed(slice(saveDBName), &dbConfig(), ERROR_INFO()); - REQUIRE(db); - _collections = collectionPreamble(_collectionSpecs); - - // 'client 2' mutations - for ( auto& coll : _collections ) { - for ( int i = numInitialRevs; i < numRevs2 + numInitialRevs; ++i ) { - createFleeceRev(coll, slice(docID), nullslice, slice(R"({"b":)" + to_string(i) + "}")); - } - } - - // There will be a conflict in the push because different revIDs from separate 'clients' - // We need to make sure the test suite knows we expect this, otherwise an assertion fails - _expectedDocPullErrors = {docID}; - - // Sync with remote - replicate(replParams); - - // Verify 'client 2' still has their own latest rev - for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(docID), true, kDocGetAll, nullptr); - REQUIRE(doc); - auto revID = slice(doc->revID); - REQUIRE(revID.hasPrefix(to_string(numRevs2 + numInitialRevs) + "-")); - } - - // Clear local db - deleteAndRecreateDBAndCollections(); - - // No pull errors as local db is clear - _expectedDocPullErrors = {}; - - // One shot pull - replParams.setPushPull(kC4Disabled, kC4OneShot); - replicate(replParams); - - // Verify latest rev is from 'client 1' (they have the highest generation) - for ( auto& coll : _collections ) { - c4::ref doc = c4coll_getDoc(coll, slice(docID), true, kDocGetAll, nullptr); - REQUIRE(doc); - auto revID = slice(doc->revID); - REQUIRE(revID.hasPrefix(to_string(numRevs1 + numInitialRevs) + "-")); - } -} - -// This test requires the sync function of the collection it uses contains the following statement, -// "if(doc.isRejected == \"true\")throw({\"forbidden\":\"read_only\"})" -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Use isRevRejected to Resolve Conflict", "[.SyncServerCollection]") { - string idPrefix = timePrefix(); - const string channelID = idPrefix + "ch"; - initTest({Tulips}, {channelID}, "user1"); - SG::TestUser user2(_sg, "user2", {channelID}, {Tulips}, "password"); - - auto bodyOfNum = [&](bool good, int n) { - char buf[80]; - snprintf(buf, 80, R"({"isRejected": "%s", "num": %d, "channels": ["%s"]})", good ? "false" : "true", n, - channelID.c_str()); - return alloc_slice(buf); - }; - - string docID = idPrefix + "doc01"; - string rev1 = createFleeceRev(_collections[0], slice(docID), nullslice, bodyOfNum(true, 1)); - - ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; - // Push a good revision of gen 1 to remote - replicate(replParams); - - // Put a bad revision of gen 2 in the local db - string rev2_bad; - { - TransactionHelper t(db); - SharedEncoder enc(c4db_getSharedFleeceEncoder(db)); - enc.convertJSON(bodyOfNum(false, 2)); - fleece::alloc_slice fleeceBody = enc.finish(); - rev2_bad = createNewRev(_collections[0], slice(docID), slice(rev1), fleeceBody); - } - - auto getAllRevs = [](C4Document* doc) { - std::vector ret; - do { - C4SliceResult j = c4doc_bodyAsJSON(doc, true, nullptr); - if ( j.buf ) { - ret.push_back(string(doc->selectedRev.revID) + "/" + string(j)); - } else { - ret.push_back(string(doc->selectedRev.revID) + "/missing"); - } - } while ( c4doc_selectNextRevision(doc) ); - c4doc_selectCurrentRevision(doc); - return ret; - }; - - auto conflictResolver = [&](const C4DocumentEnded* docEndedWithConflict) { - C4Error error; - int i = 0; - for ( ; i < _collectionCount; ++i ) { - if ( docEndedWithConflict->collectionSpec == _collectionSpecs[i] ) { break; } - } - Assert(i < _collectionCount, "Internal logical error"); - - TransactionHelper t(db); - - slice docID = docEndedWithConflict->docID; - // Get the local doc. It is the current revision - c4::ref localDoc = c4coll_getDoc(_collections[i], docID, true, kDocGetAll, WITH_ERROR(error)); - CHECK(error.code == 0); - - bool wasRejected = c4doc_isRevRejected(localDoc); - CHECK(wasRejected); - // The current 2-gen is pushed but rejected. We are going to pick the remote rev as the winner - - // Get the remote doc. It is the next leaf revision of the current revision. - c4::ref remoteDoc = c4coll_getDoc(_collections[i], docID, true, kDocGetAll, &error); - bool succ = c4doc_selectNextLeafRevision(remoteDoc, true, false, &error); - Assert(remoteDoc->selectedRev.revID == docEndedWithConflict->revID); - CHECK(error.code == 0); - CHECK(succ); - - C4Document* resolvedDoc = remoteDoc; - - FLDict mergedBody = c4doc_getProperties(resolvedDoc); - C4RevisionFlags mergedFlags = resolvedDoc->selectedRev.flags; - alloc_slice winRevID = resolvedDoc->selectedRev.revID; - alloc_slice lostRevID = (resolvedDoc == remoteDoc) ? localDoc->selectedRev.revID : remoteDoc->selectedRev.revID; - bool result = c4doc_resolveConflict2(localDoc, winRevID, lostRevID, mergedBody, mergedFlags, &error); - Assert(result, "conflictHandler: c4doc_resolveConflict2 failed for '%.*s' in '%.*s.%.*s'", SPLAT(docID), - SPLAT(_collectionSpecs[i].scope), SPLAT(_collectionSpecs[i].name)); - Assert((localDoc->flags & kDocConflicted) == 0); - - if ( !c4doc_save(localDoc, 0, &error) ) { - Assert(false, "conflictHandler: c4doc_save failed for '%.*s' in '%.*s.%.*s'", SPLAT(docID), - SPLAT(_collectionSpecs[i].scope), SPLAT(_collectionSpecs[i].name)); - } - }; - - c4::ref doc = c4coll_getDoc(_collections[0], slice(docID), true, kDocGetAll, nullptr); - REQUIRE(doc); - auto revsLocal = getAllRevs(doc); - // revsLocal has rev1 with good content, sync'ed to the remote, - // rev2 with bad content, not synced. The order is the same as in the rev tree. - // revsLocal = { 2-badContent, 1-goodContent } - CHECK(revsLocal.size() == 2); - CHECK(revsLocal[0].substr(0, 2) == "2-"); - auto pos = revsLocal[0].find('/'); - CHECK(revsLocal[0].substr(pos).find("\"isRejected\":\"true\"") != string::npos); - CHECK(revsLocal[1].substr(0, 2) == "1-"); - pos = revsLocal[1].find('/'); - CHECK(revsLocal[1].substr(pos).find("\"isRejected\":\"false\"") != string::npos); - - SECTION("Simultaneous push and pull") { - // Push the bad rev of 2-gen and pull a good rev of 2-gen - _callbackWhenIdle = [this]() { - auto seq = c4coll_getLastSequence(_collections[0]); - // seq 1 is the the good 1-gen rev - // seq 2 is the bad 2-gen rev - // The above are currently in the local db. - // seq 3 is the good 2-gen pulled from the remote - // seq 4 is new rev from the conflict resolution. It is 3-gen. - if ( seq == 4 ) { c4repl_stop(_repl); } - }; - _conflictHandler = conflictResolver; - - replParams.setPushPull(kC4OneShot, kC4Continuous); - _expectedDocPushErrors = {docID}; // rejected by the remote with error code 403 - auto replAsync = std::async(std::launch::async, [&]() { replicate(replParams); }); - - bool waitForThePush = WaitUntil(2s, [&]() { - std::scoped_lock lock(_mutex); - return !_docPushErrors.empty(); - }); - REQUIRE(waitForThePush); - - // user2 sends a good revision of 2-gen to the remote - _sg.authHeader = user2.authHeader(); - bool succ = _sg.upsertDoc(_collectionSpecs[0], string(docID), rev1, bodyOfNum(true, 2), {channelID}); - REQUIRE(succ); - - // wait for pull getting the good 2-gen rev and conflict resolved. - replAsync.wait(); - - // Check it out - doc = c4coll_getDoc(_collections[0], slice(docID), true, kDocGetAll, nullptr); - REQUIRE(doc); - - // revisions after pulling from the remote - auto revsLocal2 = getAllRevs(doc); - CHECK(revsLocal2.size() == 3); - // revsLocal2[] = { 3-goodMerged, 2-goodRemote, 1-goodContent } - CHECK(revsLocal2[0].substr(0, 2) == "3-"); - auto pos2 = revsLocal2[0].find('/'); - CHECK(revsLocal2[0].substr(pos2).find("\"isRejected\":\"false\"") != string::npos); - CHECK(revsLocal2[1].substr(0, 2) == "2-"); // This one is pulled from remote. - CHECK(revsLocal2[2].substr(0, 2) == "1-"); // This original local one. - pos2 = revsLocal2[2].find('/'); - // The 1-gen rev is not changed after push&pull - CHECK(revsLocal2[2].substr(0, pos2) == revsLocal[1].substr(0, pos)); - } - - SECTION("Separate Push and Pull without Conflict Resolver") { - replParams.setPushPull(kC4OneShot, kC4Disabled); - _expectedDocPushErrors = {docID}; // rejected by the remote with error code 403 - replicate(replParams, false); - _expectedDocPushErrors = {}; - - // user2 sends a good revision of 2-gen to the remote - _sg.authHeader = user2.authHeader(); - bool succ = _sg.upsertDoc(_collectionSpecs[0], string(docID), rev1, bodyOfNum(true, 2), {channelID}); - REQUIRE(succ); - - _stopWhenIdle.store(true); - replParams.setPushPull(kC4Disabled, kC4Continuous); - _expectedDocPullErrors = {docID}; - replicate(replParams, false); - } - - SECTION("Separate Push and Pull with Conflict Resolver") { - replParams.setPushPull(kC4OneShot, kC4Disabled); - _expectedDocPushErrors = {docID}; // rejected by the remote with error code 403 - replicate(replParams, false); - _expectedDocPushErrors = {}; - - // user2 sends a good revision of 2-gen to the remote - _sg.authHeader = user2.authHeader(); - bool succ = _sg.upsertDoc(_collectionSpecs[0], string(docID), rev1, bodyOfNum(true, 2), {channelID}); - REQUIRE(succ); - - _callbackWhenIdle = [this]() { - auto seq = c4coll_getLastSequence(_collections[0]); - if ( seq == 4 ) { c4repl_stop(_repl); } - }; - _conflictHandler = conflictResolver; - - replParams.setPushPull(kC4Disabled, kC4Continuous); - replicate(replParams); - } -} - +// Disabled, to be re-enabled with CBL-5621 +#if 0 // cbl-4499 TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull invalid deltas with filter from SG", "[.SyncServerCollection][Delta]") { @@ -2465,13 +1963,17 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull invalid deltas with filter fr const string docPrefix = idPrefix + cblTicket + "_"; + vector docIDs(kNumDocs); + + for (int docNo = 0; docNo < kNumDocs; ++docNo) { + docIDs[docNo] = stringprintf("%sdoc-%03d", docPrefix.c_str(), docNo); + } + // -------- Populating local db -------- auto populateDB = [&]() { TransactionHelper t(db); std::srand(123456); // start random() sequence at a known place - for ( int docNo = 0; docNo < kNumDocs; ++docNo ) { - char docID[kDocBufSize]; - snprintf(docID, kDocBufSize, "%sdoc-%03d", docPrefix.c_str(), docNo); + for (const string& docID : docIDs) { Encoder enc(c4db_createFleeceEncoder(db)); enc.beginDict(); for ( int p = 0; p < kNumProps; ++p ) { @@ -2495,54 +1997,36 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull invalid deltas with filter fr replicate(replParams); // -------- Updating docs on SG -------- - { - JSONEncoder enc; - enc.beginDict(); - enc.writeKey("docs"_sl); - enc.beginArray(); - for ( int docNo = 0; docNo < kNumDocs; ++docNo ) { - char docID[kDocBufSize]; - snprintf(docID, kDocBufSize, "%sdoc-%03d", docPrefix.c_str(), docNo); - C4Error error; - c4::ref doc = c4coll_getDoc(_collections[0], slice(docID), true, kDocGetAll, ERROR_INFO(error)); - REQUIRE(doc); - Dict props = c4doc_getProperties(doc); + for (const string& docID : docIDs) { + C4Error error; + c4::ref doc = c4coll_getDoc(_collections[0], slice(docID), true, kDocGetAll, ERROR_INFO(error)); + REQUIRE(doc); + Dict props = c4doc_getProperties(doc); - enc.beginDict(); - enc.writeKey("_id"_sl); - enc.writeString(docID); - enc.writeKey("_rev"_sl); - enc.writeString(doc->revID); - for ( Dict::iterator i(props); i; ++i ) { - enc.writeKey(i.keyString()); - auto value = i.value().asInt(); - if ( RandomNumber() % 2 == 0 ) value = RandomNumber(); - enc.writeInt(value); - } - enc.writeKey("channels"_sl); - enc.beginArray(); - enc.writeString(channelID); - enc.endArray(); - enc.endDict(); + JSONEncoder enc{}; + enc.beginDict(); + for ( Dict::iterator i(props); i; ++i ) { + enc.writeKey(i.keyString()); + auto value = i.value().asInt(); + if ( RandomNumber() % 2 == 0 ) value = RandomNumber(); + enc.writeInt(value); } - enc.endArray(); enc.endDict(); - (void)_sg.sendRemoteRequest("POST", _collectionSpecs[0], "_bulk_docs", enc.finish(), false, - HTTPStatus::Created); - } + alloc_slice body = enc.finish(); - // -------- Repopulating local db -------- - deleteAndRecreateDBAndCollections(); - populateDB(); + alloc_slice revID = getLegacyRevID(_collectionSpecs[0], doc); + + _sg.upsertDoc(_collectionSpecs[0], docID, revID.asString(), body, {channelID}); + } // Setup pull filter: _pullFilter = [](C4CollectionSpec collectionName, C4String docID, C4String revID, C4RevisionFlags flags, FLDict flbody, void* context) { return true; }; // -------- Pulling changes from SG -------- -#ifdef LITECORE_CPPTEST +# ifdef LITECORE_CPPTEST _expectedDocPullErrors = {docPrefix + "doc-001"}; -#endif +# endif replParams.setPushPull(kC4Disabled, kC4OneShot); replParams.setPullFilter(_pullFilter).setCallbackContext(this); { @@ -2559,12 +2043,15 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Pull invalid deltas with filter fr CHECK(error.code == 0); C4DocumentInfo info; c4enum_getDocumentInfo(e, &info); - CHECK(slice(info.docID).hasPrefix(slice(docPrefix + "doc-"))); - CHECK(slice(info.revID).hasPrefix("2-"_sl)); + alloc_slice docID = info.docID; + CHECK(docID.hasPrefix(slice(docPrefix + "doc-"))); + alloc_slice revID = getLegacyRevID(_collectionSpecs[0], info); + CHECK(revID.hasPrefix("2-"_sl)); ++n; } CHECK(n == kNumDocs); } +#endif // cbl-4499 TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Push invalid deltas to SG", "[.SyncServerCollection][Delta]") { @@ -2752,113 +2239,6 @@ TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Revoked docs queue behind revs", " _repl = nullptr; } -TEST_CASE_METHOD(ReplicatorCollectionSGTest, "Push&Pull Replication with the Copy of Fully Synced Database SG", - "[.SyncServerCollection]") { - string idPrefix = timePrefix(); - const string channelID = idPrefix; - initTest({Lavenders}, {channelID}); - - enum { iPushPull }; - - constexpr slice body = "{\"ans*wer\":42}"_sl; - alloc_slice bodyWithChannel = SG::addChannelToJSON(body, "channels", {channelID}); - alloc_slice localPrefix{idPrefix + "local-"}; - alloc_slice remotePrefix{idPrefix + "remote-"}; - unsigned docCount = 20; - - // push documents prefixed with "remote-" to the push/pull collection - for ( size_t i = 0; i < _collectionCount; ++i ) { - for ( int d = 1; d <= docCount; ++d ) { - constexpr size_t bufSize = 80; - char docID[bufSize]; - snprintf(docID, bufSize, "%.*s%d", SPLAT(remotePrefix), d); - createFleeceRev(_collections[i], slice(docID), nullslice, bodyWithChannel); - } - } - - // And send them to remote - { - ReplParams replParams{_collectionSpecs}; - replParams.setPushPull(kC4OneShot, kC4Disabled); - replicate(replParams); - } - - deleteAndRecreateDBAndCollections(); - - // add 20(docCount) "local-" docs to the local db - // The 20(docCount) "remote-" docs are already in the remote db - for ( size_t i = 0; i < _collectionCount; ++i ) { - for ( int d = 1; d <= docCount; ++d ) { - constexpr size_t bufSize = 80; - char docID[bufSize]; - snprintf(docID, bufSize, "%.*s%d", SPLAT(localPrefix), d); - createFleeceRev(_collections[i], slice(docID), nullslice, bodyWithChannel); - } - } - - // Perform a Push&Pull replication on db - // It will push 20 "local-" docs to the remote, and pull "remote-" docs to local db - { - ReplParams replParams{_collectionSpecs}; - replParams.collections[iPushPull].push = kC4OneShot; - replParams.collections[iPushPull].pull = kC4OneShot; - replicate(replParams); - } - - auto require = [&]() { - for ( size_t i = 0; i < _collectionCount; ++i ) { - c4::ref e = c4coll_enumerateAllDocs(_collections[i], nullptr, ERROR_INFO()); - unsigned total = 0; - unsigned local = 0; - unsigned remote = 0; - while ( c4enum_next(e, ERROR_INFO()) ) { - C4DocumentInfo info; - c4enum_getDocumentInfo(e, &info); - slice docID_sl{info.docID}; - total++; - if ( docID_sl.hasPrefix(localPrefix) ) { local++; } - if ( docID_sl.hasPrefix(remotePrefix) ) { remote++; } - } - switch ( i ) { - case iPushPull: - REQUIRE(total == 2 * docCount); - REQUIRE(local == docCount); - REQUIRE(remote == docCount); - break; - default: - break; - } - } - }; - // Make sure in the local db, there are docCount docs prefixed with "local-", which are - // in the local db prior to Push/Pull sync, and docCount docs prefixed "remote-", which are - // pulled from the remote db. - require(); - - // Use c4db_copyNamed to copy the db to a new file (with new UUIDs): - C4Error error; - alloc_slice path(c4db_getPath(db)); - string scratchDBName = stringprintf("scratch%" PRIms, chrono::milliseconds(time(nullptr)).count()); - REQUIRE(c4db_copyNamed(path, slice(scratchDBName), &dbConfig(), WITH_ERROR(&error))); - - // release the old db and Open the copied db: - c4db_release(db); - db = c4db_openNamed(slice(scratchDBName), &dbConfig(), ERROR_INFO(error)); - REQUIRE(db); - _collections = collectionPreamble(_collectionSpecs); - - { - ReplParams replParams{_collectionSpecs}; - replParams.setPushPull(kC4OneShot, kC4OneShot); - replicate(replParams); - } - require(); - - // The Pull/Push replicator finishes without pushing or pulling any documents because the - // checkpoints of the copied (prebuilt) db are inheritted from the original db. - CHECK((_callbackStatus.progress == C4Progress{0, 0, 0})); -} - static C4Database* copy_and_open(C4Database* db, const string& idPrefix) { const auto dbPath = db->getPath(); const string db2Name = idPrefix + "db2"; diff --git a/Replicator/tests/ReplicatorCollectionSGTest.hh b/Replicator/tests/ReplicatorCollectionSGTest.hh index e5aff61d6..6b7b0ce66 100644 --- a/Replicator/tests/ReplicatorCollectionSGTest.hh +++ b/Replicator/tests/ReplicatorCollectionSGTest.hh @@ -15,6 +15,8 @@ #include "SGTestUser.hh" #include "ReplParams.hh" #include +#include +#include #include #include @@ -37,7 +39,7 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { public: explicit ReplicatorCollectionSGTest(const std::string& minSGVer = "3.1", const std::string& maxSGVer = "", uint16_t port = 4984, const slice remoteDBName = kScratchDBName) - : ReplicatorAPITest(), _expectedSGVersion{minSGVer, maxSGVer} { + : ReplicatorAPITest(VersionVectorOption), _expectedSGVersion{minSGVer, maxSGVer} { _sg.pinnedCert = C4Test::readFile("Replicator/tests/data/cert/cert.pem"); if ( getenv("NOTLS") ) { _sg.address = {kC4Replicator2Scheme, C4STR("localhost"), port}; @@ -112,7 +114,7 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { } // propertyEncryption: 0, no encryption; 1, encryption only; 2, encryption and decryption - void verifyDocs(const std::vector>& docIDs, bool checkRev = false, + void verifyDocs(const std::vector>& docIDs, bool checkRev = false, int propertyEncryption = 0) { REQUIRE((!_collectionSpecs.empty() && !_collections.empty())); resetVerifyDb(); @@ -158,7 +160,7 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { c4enum_getDocumentInfo(e, &info); auto it = docIDs[i].find(info.docID); CHECK(it != docIDs[i].end()); - CHECK(it->second == c4rev_getGeneration(info.revID)); + CHECK(it->second == c4rev_getTimestamp(info.revID)); } CHECK(count == docIDs[i].size()); } else { @@ -169,14 +171,14 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { } // map: docID -> rev generation - static std::unordered_map getDocIDs(C4Collection* collection) { - std::unordered_map ret; + static std::unordered_map getDocIDs(C4Collection* collection) { + std::unordered_map ret; c4::ref e = c4coll_enumerateAllDocs(collection, nullptr, ERROR_INFO()); { while ( c4enum_next(e, ERROR_INFO()) ) { C4DocumentInfo info; c4enum_getDocumentInfo(e, &info); - ret.emplace(info.docID, c4rev_getGeneration(info.revID)); + ret.emplace(info.docID, c4rev_getTimestamp(info.revID)); } } return ret; @@ -224,8 +226,9 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { _collectionCount = _collectionSpecs.size(); // Avoid copy constructor new (&_testUser) SG::TestUser{_sg, username, channelIDs, collSpecs}; - _options = createOptionsAuth(_testUser._username, _testUser._password); - _sg.authHeader = _testUser.authHeader(); + _options = createOptionsAuth(_testUser._username, _testUser._password); + _sg.authHeader = _testUser.authHeader(); + _sg.useRevTrees = isRevTrees(); _docIDs.resize(_collectionCount); REQUIRE(sgVersionCheck()); } @@ -243,6 +246,33 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { for ( int i = 0; i < _collectionCount; ++i ) { _docIDs[i] = getDocIDs(_collections[i]); } } + alloc_slice getLegacyRevID(C4Collection* coll, slice docID) { + c4::ref doc = c4coll_getDoc(coll, docID, true, kDocGetCurrentRev, ERROR_INFO()); + return getLegacyRevID(coll->getSpec(), doc); + } + + alloc_slice getLegacyRevID(C4CollectionSpec spec, C4Document* doc) { + alloc_slice revID; + if ( isRevTrees() ) { + revID = doc->revID; + } else { + alloc_slice docID = doc->docID; + revID = _sg.getRevID(docID.asString(), spec); + } + return revID; + } + + alloc_slice getLegacyRevID(C4CollectionSpec spec, const C4DocumentInfo& info) { + alloc_slice revID; + if ( isRevTrees() ) { + revID = info.revID; + } else { + alloc_slice docID = info.docID; + revID = _sg.getRevID(docID.asString(), spec); + } + return revID; + } + struct CipherContext { C4Collection* collection; slice docID; @@ -261,7 +291,7 @@ class ReplicatorCollectionSGTest : public ReplicatorAPITest { std::vector _collections{}; SG::TestUser _testUser{}; size_t _collectionCount = 0; - std::vector> _docIDs{}; + std::vector> _docIDs{}; // Pair of strings representing min and max SG version // min is inclusive, max is exclusive. Empty string for max will ignore it // i.e.: {"3.0", "3.1"} represents anything 3.0 and above, and excludes 3.1 and above diff --git a/Replicator/tests/ReplicatorSG30Test.cc b/Replicator/tests/ReplicatorSG30Test.cc index 385644b5a..f93c2f013 100644 --- a/Replicator/tests/ReplicatorSG30Test.cc +++ b/Replicator/tests/ReplicatorSG30Test.cc @@ -305,9 +305,9 @@ TEST_CASE_METHOD(ReplicatorSG30Test, "Push & Pull Deletion SG3.0", "[.SyncServer createRev(_collections[0], slice(docID), kRevID, kFleeceBody); createRev(_collections[0], slice(docID), kRev2ID, kEmptyFleeceBody, kRevDeleted); - std::vector> docIDs{_collectionCount}; + std::vector> docIDs{_collectionCount}; - docIDs[0] = unordered_map{{alloc_slice(docID), 0}}; + docIDs[0] = unordered_map{{alloc_slice(docID), 0}}; ReplParams replParams{_collectionSpecs, kC4OneShot, kC4Disabled}; replParams.setDocIDs(docIDs); @@ -419,8 +419,7 @@ TEST_CASE_METHOD(ReplicatorSG30Test, "Update Once-Conflicted Doc - SG3.0", "[.Sy // Create a conflicted doc on SG, and resolve the conflict for ( const auto& body : bodies ) { _sg.upsertDoc(_collectionSpecs[0], docID + "?new_edits=false", body); } - std::vector> docIDs = { - std::unordered_map{{alloc_slice(docID), 0}}}; + std::vector docIDs = {std::unordered_map{{alloc_slice(docID), 0}}}; // Pull doc into CBL: C4Log("-------- Pulling"); diff --git a/Replicator/tests/SG.cc b/Replicator/tests/SG.cc index 61870de58..97e8e7445 100644 --- a/Replicator/tests/SG.cc +++ b/Replicator/tests/SG.cc @@ -31,8 +31,10 @@ std::unique_ptr SG::createRequest(const std::string& method, C4C path = std::string("/") + kspace + path; } } - if ( logRequests ) + if ( logRequests ) { C4Log("*** Server command: %s %.*s:%d%s", method.c_str(), SPLAT(address.hostname), port, path.c_str()); + C4Log("Body: %.*s", SPLAT(body)); + } Encoder enc; enc.beginDict(); @@ -202,7 +204,15 @@ bool SG::upsertDoc(C4CollectionSpec collectionSpec, const std::string& docID, sl bool SG::upsertDoc(C4CollectionSpec collectionSpec, const std::string& docID, const std::string& revID, slice body, const std::vector& channelIDs, C4Error* err) const { - return upsertDoc(collectionSpec, docID, addRevToJSON(body, revID), channelIDs, err); + if ( useRevTrees ) { + return upsertDoc(collectionSpec, docID, addRevToJSON(body, revID), channelIDs, err); + } else { + // For Version Vectors, SG does not allow us to provide a version vector in the '_rev' property, it must be a + // traditional revID. So we fetch the current revID from sync gateway, then we will use that. + const auto sgRevID = getRevID(docID, collectionSpec); + Assert(!sgRevID.empty()); + return upsertDoc(collectionSpec, docID, addRevToJSON(body, sgRevID.asString()), channelIDs, err); + } } bool SG::upsertDocWithEmptyChannels(C4CollectionSpec collectionSpec, const std::string& docID, slice body, @@ -241,6 +251,15 @@ alloc_slice SG::getDoc(const std::string& docID, C4CollectionSpec collectionSpec return result; } +alloc_slice SG::getRevID(const std::string& docID, C4CollectionSpec collectionSpec) const { + const alloc_slice bodyJSON = getDoc(docID, collectionSpec); + Encoder e{}; + e.convertJSON(bodyJSON); + const auto bodyFleece = e.finishDoc(); + const auto bodyDict = bodyFleece.asDict(); + return bodyDict.get("_rev").toString(); +} + alloc_slice SG::sendRemoteRequest(const std::string& method, C4CollectionSpec collectionSpec, std::string path, HTTPStatus* outStatus, C4Error* outError, slice body, bool admin, bool logRequests) { if ( method != "GET" ) Assert(slice(remoteDBName).hasPrefix("scratch"_sl)); diff --git a/Replicator/tests/SG.hh b/Replicator/tests/SG.hh index 9bdb39634..33c542db1 100644 --- a/Replicator/tests/SG.hh +++ b/Replicator/tests/SG.hh @@ -69,6 +69,8 @@ class SG { C4Error* err = nullptr) const; [[nodiscard]] alloc_slice getDoc(const std::string& docID, C4CollectionSpec collectionSpec = kC4DefaultCollectionSpec) const; + [[nodiscard]] alloc_slice getRevID(const std::string& docID, + C4CollectionSpec collectionSpec = kC4DefaultCollectionSpec) const; void setAdminCredentials(const std::string& username, const std::string& password) { adminUsername = username; @@ -113,6 +115,10 @@ class SG { c4::ref identityKey{nullptr}; #endif + // Whether to use rev-trees logic for SG requests. + // If false, uses specialized logic for Version Vectors. + bool useRevTrees{true}; + private: [[nodiscard]] std::unique_ptr createRequest(const std::string& method, C4CollectionSpec collectionSpec, std::string path, diff --git a/Replicator/tests/data/docker/sg/Dockerfile b/Replicator/tests/data/docker/sg/Dockerfile index 27d17d208..8933360f2 100644 --- a/Replicator/tests/data/docker/sg/Dockerfile +++ b/Replicator/tests/data/docker/sg/Dockerfile @@ -1,3 +1,8 @@ +FROM golang:1.23 AS build +RUN apt update +RUN git clone -b release/anemone https://github.com/couchbase/sync_gateway.git +RUN SG_EDITION=CE ./sync_gateway/build.sh + FROM ubuntu:22.04 ARG SG_DEB_ARM64 @@ -8,18 +13,18 @@ ARG LEGACY_MODE=false RUN [ -z "$SG_DEB_ARM64" ] && echo "SG_DEB_ARM64 is required" && exit 1 || true RUN [ -z "$SG_DEB_AMD64" ] && echo "SG_DEB_AMD64 is required" && exit 1 || true -RUN apt -yqq update +RUN apt -yqq update RUN apt -yqq install curl RUN mkdir -p /opt/sg && \ ARCHITECTURE="$(dpkg --print-architecture)" && \ if [ "$ARCHITECTURE" = "amd64" ]; then \ - curl -o /opt/sg/couchbase-sync-gateway.deb $SG_DEB_AMD64; \ + curl -o /opt/sg/couchbase-sync-gateway.deb $SG_DEB_AMD64; \ elif [ "$ARCHITECTURE" = "arm64" ]; then \ - curl -o /opt/sg/couchbase-sync-gateway.deb $SG_DEB_ARM64; \ + curl -o /opt/sg/couchbase-sync-gateway.deb $SG_DEB_ARM64; \ else \ - echo "Unsupported architecture"; \ - exit 1; \ + echo "Unsupported architecture"; \ + exit 1; \ fi COPY cert /opt/sg/cert @@ -28,9 +33,12 @@ COPY scripts /opt/sg/scripts WORKDIR /opt/sg +RUN apt update RUN apt -yqq install systemctl RUN dpkg -i ./couchbase-sync-gateway.deb +COPY --from=build /go/sync_gateway/bin/sync_gateway_ce /opt/couchbase-sync-gateway/bin/sync_gateway + EXPOSE 4984 EXPOSE 4985 EXPOSE 4884 diff --git a/Replicator/tests/data/docker/sg/config/collections/bootstrap.json b/Replicator/tests/data/docker/sg/config/collections/bootstrap.json index c99c357ca..6a8d98c10 100644 --- a/Replicator/tests/data/docker/sg/config/collections/bootstrap.json +++ b/Replicator/tests/data/docker/sg/config/collections/bootstrap.json @@ -16,7 +16,7 @@ "logging": { "console": { "enabled": true, - "log_level": "info", + "log_level": "debug", "log_keys": ["*"] } }