From 0a7f4c8d78a20b8576f457106c7a43e3c2bf2a76 Mon Sep 17 00:00:00 2001 From: Sejong Kim <46182768+sejongk@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:09:42 +0900 Subject: [PATCH] Introduce MongoDB sharding rules to Project-wide and Document-wide collections (#776) This commit introduces MongoDB sharding rules to Project-wide and Document-wide collections. There are three types of collections: 1. Cluster-wide: `users`, `projects`(less than 10K) 4. Project-wide: `documents`, `clients`(more than 1M) 7. Document-wide: `changes`, `snapshots`, `syncedseqs`(more than 100M) We determine whether a collection is required to be sharded based on the expected data count of its types. 1. Cluster-wide: not sharded 2. Project-wide, Document-wide: sharded Project-wide collections contain range queries using a `project_id` filter, and document-wide collections usually contain them using a `doc_id` filter. We choose the shard key based on the query pattern of each collection: 1. Project-wide: `project_id` 2. Document-wide: `doc_id` This involves changes in reference keys of collections: 1. `documents`: `_id` -> `(project_id, _id)` 2. `clients`: `_id` -> `(project_id, _id)` 4. `changes`: `_id` -> `(project_id, doc_id, server_seq)` 5. `snapshots`: `_id` -> `(project_id, doc_id, server_seq)` 6. `syncedseqs`: `_id` -> `(project_id, doc_id, client_id)` --------- Co-authored-by: Youngteac Hong --- .github/workflows/ci.yml | 40 + api/types/project.go | 4 +- api/types/resource_ref_key.go | 54 + build/charts/yorkie-mongodb/values.yaml | 5 + build/docker/sharding/README.md | 29 + build/docker/sharding/docker-compose.yml | 121 +++ build/docker/sharding/scripts/deploy.sh | 19 + build/docker/sharding/scripts/init-config1.js | 9 + build/docker/sharding/scripts/init-mongos1.js | 39 + .../docker/sharding/scripts/init-shard1-1.js | 8 + .../docker/sharding/scripts/init-shard2-1.js | 8 + client/client.go | 11 - server/backend/database/change_info.go | 1 + server/backend/database/client_info.go | 54 +- server/backend/database/client_info_test.go | 7 +- server/backend/database/database.go | 62 +- server/backend/database/doc_info.go | 8 + server/backend/database/memory/database.go | 196 ++-- .../backend/database/memory/database_test.go | 4 + server/backend/database/mongo/client.go | 527 ++++------ server/backend/database/mongo/client_test.go | 4 + server/backend/database/mongo/indexes.go | 62 +- server/backend/database/mongo/registry.go | 44 +- .../backend/database/mongo/registry_test.go | 48 +- server/backend/database/snapshot_info.go | 15 + server/backend/database/synced_seq_info.go | 5 +- .../backend/database/testcases/testcases.go | 308 +++--- server/backend/housekeeping/housekeeping.go | 3 +- server/backend/sync/coordinator.go | 4 +- server/backend/sync/memory/coordinator.go | 10 +- .../backend/sync/memory/coordinator_test.go | 7 +- server/backend/sync/memory/pubsub.go | 52 +- server/backend/sync/memory/pubsub_test.go | 15 +- server/backend/sync/pubsub.go | 8 +- server/clients/clients.go | 34 +- server/documents/documents.go | 39 +- server/packs/history.go | 8 +- server/packs/packs.go | 21 +- server/packs/pushpull.go | 2 +- server/packs/snapshots.go | 22 +- server/rpc/admin_server.go | 12 +- server/rpc/interceptors/admin_auth.go | 2 +- server/rpc/server_test.go | 828 +-------------- server/rpc/testcases/testcases.go | 985 ++++++++++++++++++ server/rpc/yorkie_server.go | 129 ++- server/users/users.go | 8 +- test/bench/push_pull_bench_test.go | 29 +- test/helper/helper.go | 138 +++ test/integration/document_test.go | 4 +- test/integration/housekeeping_test.go | 6 +- test/integration/retention_test.go | 5 +- test/sharding/mongo_client_test.go | 166 +++ test/sharding/server_test.go | 204 ++++ 53 files changed, 2809 insertions(+), 1624 deletions(-) create mode 100644 api/types/resource_ref_key.go create mode 100644 build/docker/sharding/README.md create mode 100644 build/docker/sharding/docker-compose.yml create mode 100755 build/docker/sharding/scripts/deploy.sh create mode 100644 build/docker/sharding/scripts/init-config1.js create mode 100644 build/docker/sharding/scripts/init-mongos1.js create mode 100644 build/docker/sharding/scripts/init-shard1-1.js create mode 100644 build/docker/sharding/scripts/init-shard2-1.js create mode 100644 server/rpc/testcases/testcases.go create mode 100644 test/sharding/mongo_client_test.go create mode 100644 test/sharding/server_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c9f623f1..a5d937265 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,3 +91,43 @@ jobs: fail-on-alert: false github-token: ${{ secrets.GITHUB_TOKEN }} comment-always: true + + sharding_test: + name: sharding_test + runs-on: ubuntu-latest + steps: + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get tools dependencies + run: make tools + + - name: Check Docker Compose Version + run: docker compose --version + + - name: Run the Config server, Shard 1 and Shard 2 + run: docker compose -f build/docker/sharding/docker-compose.yml up --build -d --wait config1 shard1-1 shard2-1 + + - name: Initialize the Config server + run: docker compose -f build/docker/sharding/docker-compose.yml exec config1 mongosh test /scripts/init-config1.js + + - name: Initialize the Shard 1 + run: docker compose -f build/docker/sharding/docker-compose.yml exec shard1-1 mongosh test /scripts/init-shard1-1.js + + - name: Initialize the Shard 2 + run: docker compose -f build/docker/sharding/docker-compose.yml exec shard2-1 mongosh test /scripts/init-shard2-1.js + + - name: Run the Mongos + run: docker compose -f build/docker/sharding/docker-compose.yml up --build -d --wait mongos1 + + - name: Initialize the Mongos + run: docker compose -f build/docker/sharding/docker-compose.yml exec mongos1 mongosh test /scripts/init-mongos1.js + + - name: Run the tests with sharding tag + run: go test -tags sharding -race -v ./test/sharding/... diff --git a/api/types/project.go b/api/types/project.go index 420b0cf67..3ed8dd909 100644 --- a/api/types/project.go +++ b/api/types/project.go @@ -17,7 +17,9 @@ package types -import "time" +import ( + "time" +) // Project is a project that consists of multiple documents and clients. type Project struct { diff --git a/api/types/resource_ref_key.go b/api/types/resource_ref_key.go new file mode 100644 index 000000000..d295d829a --- /dev/null +++ b/api/types/resource_ref_key.go @@ -0,0 +1,54 @@ +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "fmt" +) + +// DocRefKey represents an identifier used to reference a document. +type DocRefKey struct { + ProjectID ID + DocID ID +} + +// String returns the string representation of the given DocRefKey. +func (r DocRefKey) String() string { + return fmt.Sprintf("Document (%s.%s)", r.ProjectID, r.DocID) +} + +// ClientRefKey represents an identifier used to reference a client. +type ClientRefKey struct { + ProjectID ID + ClientID ID +} + +// String returns the string representation of the given ClientRefKey. +func (r ClientRefKey) String() string { + return fmt.Sprintf("Client (%s.%s)", r.ProjectID, r.ClientID) +} + +// SnapshotRefKey represents an identifier used to reference a snapshot. +type SnapshotRefKey struct { + DocRefKey + ServerSeq int64 +} + +// String returns the string representation of the given SnapshotRefKey. +func (r SnapshotRefKey) String() string { + return fmt.Sprintf("Snapshot (%s.%s.%d)", r.ProjectID, r.DocID, r.ServerSeq) +} diff --git a/build/charts/yorkie-mongodb/values.yaml b/build/charts/yorkie-mongodb/values.yaml index a02c0fca7..b9e01f5dc 100644 --- a/build/charts/yorkie-mongodb/values.yaml +++ b/build/charts/yorkie-mongodb/values.yaml @@ -32,6 +32,11 @@ sharded: restartPolicy: Never backoffLimit: 0 rules: + - collectionName: clients + shardKeys: + - name: project_id + method: "1" + unique: false - collectionName: documents shardKeys: - name: project_id diff --git a/build/docker/sharding/README.md b/build/docker/sharding/README.md new file mode 100644 index 000000000..d02abeeab --- /dev/null +++ b/build/docker/sharding/README.md @@ -0,0 +1,29 @@ +# Docker Compose File for Sharded Cluster + +These files deploy and set up a MongoDB sharded cluster using `docker compose`. + +The cluster consists of the following components and offers the minimum +configuration required for testing. +- Config Server (Primary Only) +- 2 Shards (Each Primary Only) +- 1 Mongos + +```bash +# Run the deploy.sh script to deploy and set up a sharded cluster. +./scripts/deploy.sh + +# Shut down the apps +docker compose -f docker-compose.yml down +``` + +The files we use are as follows: +- `docker-compose.yml`: This file is used to run Yorkie's integration tests with a + MongoDB sharded cluster. It runs a MongoDB sharded cluster. +- `scripts/init-config.yml`: This file is used to set up a replica set of the config + server. +- `scripts/init-shard1.yml`: This file is used to set up a replica set of the shard 1. +- `scripts/init-shard2.yml`: This file is used to set up a replica set of the shard 2. +- `scripts/init-mongos.yml`: This file is used to shard the `yorkie-meta` database and + the collections of it. +- `scripts/deploy.sh`: This script runs a MongoDB sharded cluster and sets up the cluster + step by step. diff --git a/build/docker/sharding/docker-compose.yml b/build/docker/sharding/docker-compose.yml new file mode 100644 index 000000000..387b33364 --- /dev/null +++ b/build/docker/sharding/docker-compose.yml @@ -0,0 +1,121 @@ +version: '3' +services: + + # Config Server + config1: + image: mongo:6.0 + container_name: mongo-config1 + command: mongod --port 27017 --configsvr --replSet config-rs --bind_ip_all + volumes: + - ./scripts:/scripts + ports: + - 27100:27017 + restart: always + networks: + - sharding + healthcheck: + test: + [ + "CMD", + "mongosh", + "--host", + "config1", + "--port", + "27017", + "--quiet", + "--eval", + "'db.runCommand(\"ping\").ok'" + ] + interval: 30s + + # Shards + # Shards 1 + shard1-1: + image: mongo:6.0 + container_name: mongo-shard1-1 + command: mongod --port 27017 --shardsvr --replSet shard-rs-1 --bind_ip_all + volumes: + - ./scripts:/scripts + ports: + - 27110:27017 + restart: always + networks: + - sharding + healthcheck: + test: + [ + "CMD", + "mongosh", + "--host", + "shard1-1", + "--port", + "27017", + "--quiet", + "--eval", + "'db.runCommand(\"ping\").ok'" + ] + interval: 30s + + # Shards 2 + shard2-1: + image: mongo:6.0 + container_name: mongo-shard2-1 + command: mongod --port 27017 --shardsvr --replSet shard-rs-2 --bind_ip_all + volumes: + - ./scripts:/scripts + ports: + - 27113:27017 + restart: always + networks: + - sharding + healthcheck: + test: + [ + "CMD", + "mongosh", + "--host", + "shard2-1", + "--port", + "27017", + "--quiet", + "--eval", + "'db.runCommand(\"ping\").ok'" + ] + interval: 30s + + # Mongos + mongos1: + image: mongo:6.0 + container_name: mongos1 + command: mongos --port 27017 --configdb config-rs/config1:27017 --bind_ip_all + ports: + - 27017:27017 + restart: always + volumes: + - ./scripts:/scripts + networks: + - sharding + healthcheck: + test: + [ + "CMD", + "mongosh", + "--host", + "mongos1", + "--port", + "27017", + "--quiet", + "--eval", + "'db.runCommand(\"ping\").ok'" + ] + interval: 30s + depends_on: + config1: + condition: service_healthy + shard1-1: + condition: service_healthy + shard2-1: + condition: service_healthy + +networks: + sharding: diff --git a/build/docker/sharding/scripts/deploy.sh b/build/docker/sharding/scripts/deploy.sh new file mode 100755 index 000000000..f362e2674 --- /dev/null +++ b/build/docker/sharding/scripts/deploy.sh @@ -0,0 +1,19 @@ +#!/bin/sh +echo "Run the Config server, Shard 1 and Shard 2" +docker compose -f docker-compose.yml up --build -d --wait config1 shard1-1 shard2-1 +echo $? +echo "Init config" +docker compose -f docker-compose.yml exec config1 mongosh test /scripts/init-config1.js +echo $? +echo "Init shard1" +docker compose -f docker-compose.yml exec shard1-1 mongosh test /scripts/init-shard1-1.js +echo $? +echo "Init shard2" +docker compose -f docker-compose.yml exec shard2-1 mongosh test /scripts/init-shard2-1.js +echo $? +echo "Run the Mongos" +docker compose -f docker-compose.yml up --build -d --wait mongos1 +echo $? +echo "Init mongos1" +docker compose -f docker-compose.yml exec mongos1 mongosh test /scripts/init-mongos1.js +echo $? diff --git a/build/docker/sharding/scripts/init-config1.js b/build/docker/sharding/scripts/init-config1.js new file mode 100644 index 000000000..e839816c7 --- /dev/null +++ b/build/docker/sharding/scripts/init-config1.js @@ -0,0 +1,9 @@ +rs.initiate( + { + _id: "config-rs", + configsvr: true, + members: [ + { _id: 0, host: "config1:27017" }, + ] + } +) diff --git a/build/docker/sharding/scripts/init-mongos1.js b/build/docker/sharding/scripts/init-mongos1.js new file mode 100644 index 000000000..8942d777f --- /dev/null +++ b/build/docker/sharding/scripts/init-mongos1.js @@ -0,0 +1,39 @@ +sh.addShard("shard-rs-1/shard1-1:27017") +sh.addShard("shard-rs-2/shard2-1:27017") + +function findAnotherShard(shard) { + if (shard == "shard-rs-1") { + return "shard-rs-2" + } else { + return "shard-rs-1" + } +} + +function shardOfChunk(minKeyOfChunk) { + return db.getSiblingDB("config").chunks.findOne({ min: { project_id: minKeyOfChunk } }).shard +} + +// Shard the database for the mongo client test +const mongoClientDB = "test-yorkie-meta-mongo-client" +sh.enableSharding(mongoClientDB) +sh.shardCollection(mongoClientDB + ".clients", { project_id: 1 }) +sh.shardCollection(mongoClientDB + ".documents", { project_id: 1 }) +sh.shardCollection(mongoClientDB + ".changes", { doc_id: 1 }) +sh.shardCollection(mongoClientDB + ".snapshots", { doc_id: 1 }) +sh.shardCollection(mongoClientDB + ".syncedseqs", { doc_id: 1 }) + +// Split the inital range at `splitPoint` to allow doc_ids duplicate in different shards. +const splitPoint = ObjectId("500000000000000000000000") +sh.splitAt(mongoClientDB + ".documents", { project_id: splitPoint }) +// Move the chunk to another shard. +db.adminCommand({ moveChunk: mongoClientDB + ".documents", find: { project_id: splitPoint }, to: findAnotherShard(shardOfChunk(splitPoint)) }) + +// Shard the database for the server test +const serverDB = "test-yorkie-meta-server" +sh.enableSharding(serverDB) +sh.shardCollection(serverDB + ".clients", { project_id: 1 }) +sh.shardCollection(serverDB + ".documents", { project_id: 1 }) +sh.shardCollection(serverDB + ".changes", { doc_id: 1 }) +sh.shardCollection(serverDB + ".snapshots", { doc_id: 1 }) +sh.shardCollection(serverDB + ".syncedseqs", { doc_id: 1 }) + diff --git a/build/docker/sharding/scripts/init-shard1-1.js b/build/docker/sharding/scripts/init-shard1-1.js new file mode 100644 index 000000000..673b1ca04 --- /dev/null +++ b/build/docker/sharding/scripts/init-shard1-1.js @@ -0,0 +1,8 @@ +rs.initiate( + { + _id: "shard-rs-1", + members: [ + { _id: 0, host: "shard1-1:27017" }, + ] + } +) diff --git a/build/docker/sharding/scripts/init-shard2-1.js b/build/docker/sharding/scripts/init-shard2-1.js new file mode 100644 index 000000000..874be2fda --- /dev/null +++ b/build/docker/sharding/scripts/init-shard2-1.js @@ -0,0 +1,8 @@ +rs.initiate( + { + _id: "shard-rs-2", + members: [ + { _id: 0, host: "shard2-1:27017" }, + ] + } +) diff --git a/client/client.go b/client/client.go index 0ec16183c..f1673a9ac 100644 --- a/client/client.go +++ b/client/client.go @@ -581,17 +581,6 @@ func handleResponse( return nil, ErrUnsupportedWatchResponseType } -// FindDocKey returns the document key of the given document id. -func (c *Client) FindDocKey(docID string) (key.Key, error) { - for _, attachment := range c.attachments { - if attachment.docID.String() == docID { - return attachment.doc.Key(), nil - } - } - - return "", ErrDocumentNotAttached -} - // ID returns the ID of this client. func (c *Client) ID() *time.ActorID { return c.id diff --git a/server/backend/database/change_info.go b/server/backend/database/change_info.go index 52526cd1c..f35635d6c 100644 --- a/server/backend/database/change_info.go +++ b/server/backend/database/change_info.go @@ -41,6 +41,7 @@ var ErrDecodeOperationFailed = errors.New("decode operations failed") // ChangeInfo is a structure representing information of a change. type ChangeInfo struct { ID types.ID `bson:"_id"` + ProjectID types.ID `bson:"project_id"` DocID types.ID `bson:"doc_id"` ServerSeq int64 `bson:"server_seq"` ClientSeq uint32 `bson:"client_seq"` diff --git a/server/backend/database/client_info.go b/server/backend/database/client_info.go index a33b9dac8..0c9e4f3e8 100644 --- a/server/backend/database/client_info.go +++ b/server/backend/database/client_info.go @@ -54,6 +54,9 @@ type ClientDocInfo struct { ClientSeq uint32 `bson:"client_seq"` } +// ClientDocInfoMap is a map that associates DocRefKey with ClientDocInfo instances. +type ClientDocInfoMap map[types.ID]*ClientDocInfo + // ClientInfo is a structure representing information of a client. type ClientInfo struct { // ID is the unique ID of the client. @@ -69,7 +72,7 @@ type ClientInfo struct { Status string `bson:"status"` // Documents is a map of document which is attached to the client. - Documents map[types.ID]*ClientDocInfo `bson:"documents"` + Documents ClientDocInfoMap `bson:"documents"` // CreatedAt is the time when the client was created. CreatedAt time.Time `bson:"created_at"` @@ -85,9 +88,9 @@ func (i *ClientInfo) CheckIfInProject(projectID types.ID) error { if i.ProjectID != projectID { return fmt.Errorf( "check client(%s,%s) in project(%s): %w", - i.ID.String(), - i.ProjectID.String(), - projectID.String(), + i.ID, + i.ProjectID, + projectID, ErrClientNotFound, ) } @@ -103,7 +106,8 @@ func (i *ClientInfo) Deactivate() { // AttachDocument attaches the given document to this client. func (i *ClientInfo) AttachDocument(docID types.ID) error { if i.Status != ClientActivated { - return fmt.Errorf("client(%s) attaches document(%s): %w", i.ID.String(), docID.String(), ErrClientNotActivated) + return fmt.Errorf("client(%s) attaches %s: %w", + i.ID, docID, ErrClientNotActivated) } if i.Documents == nil { @@ -111,7 +115,8 @@ func (i *ClientInfo) AttachDocument(docID types.ID) error { } if i.hasDocument(docID) && i.Documents[docID].Status == DocumentAttached { - return fmt.Errorf("client(%s) attaches document(%s): %w", i.ID.String(), docID.String(), ErrDocumentAlreadyAttached) + return fmt.Errorf("client(%s) attaches %s: %w", + i.ID, docID, ErrDocumentAlreadyAttached) } i.Documents[docID] = &ClientDocInfo{ @@ -155,7 +160,8 @@ func (i *ClientInfo) RemoveDocument(docID types.ID) error { // IsAttached returns whether the given document is attached to this client. func (i *ClientInfo) IsAttached(docID types.ID) (bool, error) { if !i.hasDocument(docID) { - return false, fmt.Errorf("check document(%s) is attached: %w", docID.String(), ErrDocumentNeverAttached) + return false, fmt.Errorf("check %s is attached: %w", + docID, ErrDocumentNeverAttached) } return i.Documents[docID].Status == DocumentAttached, nil @@ -177,7 +183,7 @@ func (i *ClientInfo) UpdateCheckpoint( cp change.Checkpoint, ) error { if !i.hasDocument(docID) { - return fmt.Errorf("update checkpoint in document(%s): %w", docID.String(), ErrDocumentNeverAttached) + return fmt.Errorf("update checkpoint in %s: %w", docID, ErrDocumentNeverAttached) } i.Documents[docID].ServerSeq = cp.ServerSeq @@ -190,19 +196,13 @@ func (i *ClientInfo) UpdateCheckpoint( // EnsureDocumentAttached ensures the given document is attached. func (i *ClientInfo) EnsureDocumentAttached(docID types.ID) error { if i.Status != ClientActivated { - return fmt.Errorf("ensure attached document(%s) in client(%s): %w", - docID.String(), - i.ID.String(), - ErrClientNotActivated, - ) + return fmt.Errorf("ensure attached %s in client(%s): %w", + docID, i.ID, ErrClientNotActivated) } if !i.hasDocument(docID) || i.Documents[docID].Status != DocumentAttached { - return fmt.Errorf("ensure attached document(%s) in client(%s): %w", - docID.String(), - i.ID.String(), - ErrDocumentNotAttached, - ) + return fmt.Errorf("ensure attached %s in client(%s): %w", + docID, i.ID, ErrDocumentNotAttached) } return nil @@ -215,11 +215,11 @@ func (i *ClientInfo) DeepCopy() *ClientInfo { } documents := make(map[types.ID]*ClientDocInfo, len(i.Documents)) - for k, v := range i.Documents { - documents[k] = &ClientDocInfo{ - Status: v.Status, - ServerSeq: v.ServerSeq, - ClientSeq: v.ClientSeq, + for docID, docInfo := range i.Documents { + documents[docID] = &ClientDocInfo{ + Status: docInfo.Status, + ServerSeq: docInfo.ServerSeq, + ClientSeq: docInfo.ClientSeq, } } @@ -237,3 +237,11 @@ func (i *ClientInfo) DeepCopy() *ClientInfo { func (i *ClientInfo) hasDocument(docID types.ID) bool { return i.Documents != nil && i.Documents[docID] != nil } + +// RefKey returns the refKey of the client. +func (i *ClientInfo) RefKey() types.ClientRefKey { + return types.ClientRefKey{ + ProjectID: i.ProjectID, + ClientID: i.ID, + } +} diff --git a/server/backend/database/client_info_test.go b/server/backend/database/client_info_test.go index ac126e622..83dd3f8a1 100644 --- a/server/backend/database/client_info_test.go +++ b/server/backend/database/client_info_test.go @@ -28,6 +28,8 @@ import ( func TestClientInfo(t *testing.T) { dummyDocID := types.ID("000000000000000000000000") + dummyProjectID := types.ID("000000000000000000000000") + otherProjectID := types.ID("000000000000000000000001") t.Run("attach/detach document test", func(t *testing.T) { clientInfo := database.ClientInfo{ @@ -61,7 +63,6 @@ func TestClientInfo(t *testing.T) { }) t.Run("check if in project test", func(t *testing.T) { - dummyProjectID := types.ID("000000000000000000000000") clientInfo := database.ClientInfo{ ProjectID: dummyProjectID, } @@ -71,8 +72,6 @@ func TestClientInfo(t *testing.T) { }) t.Run("check if in project error test", func(t *testing.T) { - dummyProjectID := types.ID("000000000000000000000000") - otherProjectID := types.ID("000000000000000000000001") clientInfo := database.ClientInfo{ ProjectID: dummyProjectID, } @@ -117,7 +116,6 @@ func TestClientInfo(t *testing.T) { clientInfo := database.ClientInfo{ Status: database.ClientActivated, } - err := clientInfo.DetachDocument(dummyDocID) assert.ErrorIs(t, err, database.ErrDocumentNotAttached) }) @@ -126,7 +124,6 @@ func TestClientInfo(t *testing.T) { clientInfo := database.ClientInfo{ Status: database.ClientActivated, } - _, err := clientInfo.IsAttached(dummyDocID) assert.ErrorIs(t, err, database.ErrDocumentNeverAttached) diff --git a/server/backend/database/database.go b/server/backend/database/database.go index ad8ded7de..a5016dce8 100644 --- a/server/backend/database/database.go +++ b/server/backend/database/database.go @@ -121,23 +121,23 @@ type Database interface { hashedPassword string, ) (*UserInfo, error) - // FindUserInfo returns a user by the given username. - FindUserInfo(ctx context.Context, username string) (*UserInfo, error) - - // FindUserInfoByID finds a user by the given id. + // FindUserInfoByID returns a user by the given ID. FindUserInfoByID(ctx context.Context, id types.ID) (*UserInfo, error) + // FindUserInfoByName returns a user by the given username. + FindUserInfoByName(ctx context.Context, username string) (*UserInfo, error) + // ListUserInfos returns all users. ListUserInfos(ctx context.Context) ([]*UserInfo, error) // ActivateClient activates the client of the given key. ActivateClient(ctx context.Context, projectID types.ID, key string) (*ClientInfo, error) - // DeactivateClient deactivates the client of the given ID. - DeactivateClient(ctx context.Context, projectID, clientID types.ID) (*ClientInfo, error) + // DeactivateClient deactivates the client of the given refKey. + DeactivateClient(ctx context.Context, refKey types.ClientRefKey) (*ClientInfo, error) - // FindClientInfoByID finds the client of the given ID. - FindClientInfoByID(ctx context.Context, projectID, clientID types.ID) (*ClientInfo, error) + // FindClientInfoByRefKey finds the client of the given refKey. + FindClientInfoByRefKey(ctx context.Context, refKey types.ClientRefKey) (*ClientInfo, error) // UpdateClientInfoAfterPushPull updates the client from the given clientInfo // after handling PushPull. @@ -169,24 +169,21 @@ type Database interface { // exist. FindDocInfoByKeyAndOwner( ctx context.Context, - projectID types.ID, - clientID types.ID, + clientRefKey types.ClientRefKey, docKey key.Key, createDocIfNotExist bool, ) (*DocInfo, error) - // FindDocInfoByID finds the document of the given ID. - FindDocInfoByID( + // FindDocInfoByRefKey finds the document of the given refKey. + FindDocInfoByRefKey( ctx context.Context, - projectID types.ID, - id types.ID, + refKey types.DocRefKey, ) (*DocInfo, error) // UpdateDocInfoStatusToRemoved updates the document status to removed. UpdateDocInfoStatusToRemoved( ctx context.Context, - projectID types.ID, - docID types.ID, + refKey types.DocRefKey, ) error // CreateChangeInfos stores the given changes then updates the given docInfo. @@ -203,13 +200,13 @@ type Database interface { // save storage. PurgeStaleChanges( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, ) error // FindChangesBetweenServerSeqs returns the changes between two server sequences. FindChangesBetweenServerSeqs( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*change.Change, error) @@ -217,34 +214,44 @@ type Database interface { // FindChangeInfosBetweenServerSeqs returns the changeInfos between two server sequences. FindChangeInfosBetweenServerSeqs( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*ChangeInfo, error) // CreateSnapshotInfo stores the snapshot of the given document. - CreateSnapshotInfo(ctx context.Context, docID types.ID, doc *document.InternalDocument) error + CreateSnapshotInfo( + ctx context.Context, + docRefKey types.DocRefKey, + doc *document.InternalDocument, + ) error - // FindSnapshotInfoByID returns the snapshot by the given id. - FindSnapshotInfoByID(ctx context.Context, id types.ID) (*SnapshotInfo, error) + // FindSnapshotInfoByRefKey returns the snapshot by the given refKey. + FindSnapshotInfoByRefKey( + ctx context.Context, + refKey types.SnapshotRefKey, + ) (*SnapshotInfo, error) // FindClosestSnapshotInfo finds the closest snapshot info in a given serverSeq. FindClosestSnapshotInfo( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, includeSnapshot bool, ) (*SnapshotInfo, error) // FindMinSyncedSeqInfo finds the minimum synced sequence info. - FindMinSyncedSeqInfo(ctx context.Context, docID types.ID) (*SyncedSeqInfo, error) + FindMinSyncedSeqInfo( + ctx context.Context, + docRefKey types.DocRefKey, + ) (*SyncedSeqInfo, error) // UpdateAndFindMinSyncedTicket updates the given serverSeq of the given client // and returns the min synced ticket. UpdateAndFindMinSyncedTicket( ctx context.Context, clientInfo *ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) (*time.Ticket, error) @@ -252,7 +259,7 @@ type Database interface { UpdateSyncedSeq( ctx context.Context, clientInfo *ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) error @@ -274,8 +281,7 @@ type Database interface { // IsDocumentAttached returns true if the document is attached to clients. IsDocumentAttached( ctx context.Context, - projectID types.ID, - docID types.ID, + docRefKey types.DocRefKey, excludeClientID types.ID, ) (bool, error) } diff --git a/server/backend/database/doc_info.go b/server/backend/database/doc_info.go index 847458458..088efcaff 100644 --- a/server/backend/database/doc_info.go +++ b/server/backend/database/doc_info.go @@ -82,3 +82,11 @@ func (info *DocInfo) DeepCopy() *DocInfo { RemovedAt: info.RemovedAt, } } + +// RefKey returns the refKey of the document. +func (info *DocInfo) RefKey() types.DocRefKey { + return types.DocRefKey{ + ProjectID: info.ProjectID, + DocID: info.ID, + } +} diff --git a/server/backend/database/memory/database.go b/server/backend/database/memory/database.go index 1e35c0fed..deb14b4e9 100644 --- a/server/backend/database/memory/database.go +++ b/server/backend/database/memory/database.go @@ -402,33 +402,33 @@ func (d *DB) CreateUserInfo( return info, nil } -// FindUserInfo finds a user by the given username. -func (d *DB) FindUserInfo(_ context.Context, username string) (*database.UserInfo, error) { +// FindUserInfoByID finds a user by the given ID. +func (d *DB) FindUserInfoByID(_ context.Context, clientID types.ID) (*database.UserInfo, error) { txn := d.db.Txn(false) defer txn.Abort() - raw, err := txn.First(tblUsers, "username", username) + raw, err := txn.First(tblUsers, "id", clientID.String()) if err != nil { - return nil, fmt.Errorf("find user by username: %w", err) + return nil, fmt.Errorf("find user by id: %w", err) } if raw == nil { - return nil, fmt.Errorf("%s: %w", username, database.ErrUserNotFound) + return nil, fmt.Errorf("%s: %w", clientID, database.ErrUserNotFound) } return raw.(*database.UserInfo).DeepCopy(), nil } -// FindUserInfoByID finds a user by the given ID. -func (d *DB) FindUserInfoByID(_ context.Context, clientID types.ID) (*database.UserInfo, error) { +// FindUserInfoByName finds a user by the given username. +func (d *DB) FindUserInfoByName(_ context.Context, username string) (*database.UserInfo, error) { txn := d.db.Txn(false) defer txn.Abort() - raw, err := txn.First(tblUsers, "id", clientID.String()) + raw, err := txn.First(tblUsers, "username", username) if err != nil { - return nil, fmt.Errorf("find user by id: %w", err) + return nil, fmt.Errorf("find user by username: %w", err) } if raw == nil { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrUserNotFound) + return nil, fmt.Errorf("%s: %w", username, database.ErrUserNotFound) } return raw.(*database.UserInfo).DeepCopy(), nil @@ -497,25 +497,25 @@ func (d *DB) ActivateClient( } // DeactivateClient deactivates a client. -func (d *DB) DeactivateClient(_ context.Context, projectID, clientID types.ID) (*database.ClientInfo, error) { - if err := clientID.Validate(); err != nil { +func (d *DB) DeactivateClient(_ context.Context, refKey types.ClientRefKey) (*database.ClientInfo, error) { + if err := refKey.ClientID.Validate(); err != nil { return nil, err } txn := d.db.Txn(true) defer txn.Abort() - raw, err := txn.First(tblClients, "id", clientID.String()) + raw, err := txn.First(tblClients, "id", refKey.ClientID.String()) if err != nil { return nil, fmt.Errorf("find client by id: %w", err) } if raw == nil { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrClientNotFound) + return nil, fmt.Errorf("%s: %w", refKey.ClientID, database.ErrClientNotFound) } clientInfo := raw.(*database.ClientInfo) - if err := clientInfo.CheckIfInProject(projectID); err != nil { + if err := clientInfo.CheckIfInProject(refKey.ProjectID); err != nil { return nil, err } @@ -532,25 +532,25 @@ func (d *DB) DeactivateClient(_ context.Context, projectID, clientID types.ID) ( return clientInfo, nil } -// FindClientInfoByID finds a client by ID. -func (d *DB) FindClientInfoByID(_ context.Context, projectID, clientID types.ID) (*database.ClientInfo, error) { - if err := clientID.Validate(); err != nil { +// FindClientInfoByRefKey finds a client by the given refKey. +func (d *DB) FindClientInfoByRefKey(_ context.Context, refKey types.ClientRefKey) (*database.ClientInfo, error) { + if err := refKey.ClientID.Validate(); err != nil { return nil, err } txn := d.db.Txn(false) defer txn.Abort() - raw, err := txn.First(tblClients, "id", clientID.String()) + raw, err := txn.First(tblClients, "id", refKey.ClientID.String()) if err != nil { return nil, fmt.Errorf("find client by id: %w", err) } if raw == nil { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrClientNotFound) + return nil, fmt.Errorf("%s: %w", refKey.ClientID, database.ErrClientNotFound) } clientInfo := raw.(*database.ClientInfo) - if err := clientInfo.CheckIfInProject(projectID); err != nil { + if err := clientInfo.CheckIfInProject(refKey.ProjectID); err != nil { return nil, err } @@ -564,8 +564,9 @@ func (d *DB) UpdateClientInfoAfterPushPull( clientInfo *database.ClientInfo, docInfo *database.DocInfo, ) error { - clientDocInfo := clientInfo.Documents[docInfo.ID] - attached, err := clientInfo.IsAttached(docInfo.ID) + docRefKey := docInfo.RefKey() + clientDocInfo := clientInfo.Documents[docRefKey.DocID] + attached, err := clientInfo.IsAttached(docRefKey.DocID) if err != nil { return err } @@ -584,16 +585,16 @@ func (d *DB) UpdateClientInfoAfterPushPull( loaded := raw.(*database.ClientInfo).DeepCopy() if !attached { - loaded.Documents[docInfo.ID] = &database.ClientDocInfo{ + loaded.Documents[docRefKey.DocID] = &database.ClientDocInfo{ Status: clientDocInfo.Status, } loaded.UpdatedAt = gotime.Now() } else { - if _, ok := loaded.Documents[docInfo.ID]; !ok { - loaded.Documents[docInfo.ID] = &database.ClientDocInfo{} + if _, ok := loaded.Documents[docRefKey.DocID]; !ok { + loaded.Documents[docRefKey.DocID] = &database.ClientDocInfo{} } - loadedClientDocInfo := loaded.Documents[docInfo.ID] + loadedClientDocInfo := loaded.Documents[docRefKey.DocID] serverSeq := loadedClientDocInfo.ServerSeq if clientDocInfo.ServerSeq > loadedClientDocInfo.ServerSeq { serverSeq = clientDocInfo.ServerSeq @@ -602,7 +603,7 @@ func (d *DB) UpdateClientInfoAfterPushPull( if clientDocInfo.ClientSeq > loadedClientDocInfo.ClientSeq { clientSeq = clientDocInfo.ClientSeq } - loaded.Documents[docInfo.ID] = &database.ClientDocInfo{ + loaded.Documents[docRefKey.DocID] = &database.ClientDocInfo{ ServerSeq: serverSeq, ClientSeq: clientSeq, Status: clientDocInfo.Status, @@ -667,8 +668,7 @@ func (d *DB) FindDeactivateCandidatesPerProject( // exist. func (d *DB) FindDocInfoByKeyAndOwner( _ context.Context, - projectID types.ID, - clientID types.ID, + clientRefKey types.ClientRefKey, key key.Key, createDocIfNotExist bool, ) (*database.DocInfo, error) { @@ -678,7 +678,13 @@ func (d *DB) FindDocInfoByKeyAndOwner( // TODO(hackerwins): Removed documents should be filtered out by the query, but // somehow it does not work. This is a workaround. // val, err := txn.First(tblDocuments, "project_id_key_removed_at", projectID.String(), key.String(), gotime.Time{}) - iter, err := txn.Get(tblDocuments, "project_id_key_removed_at", projectID.String(), key.String(), gotime.Time{}) + iter, err := txn.Get( + tblDocuments, + "project_id_key_removed_at", + clientRefKey.ProjectID.String(), + key.String(), + gotime.Time{}, + ) if err != nil { return nil, fmt.Errorf("find document by key: %w", err) } @@ -693,7 +699,12 @@ func (d *DB) FindDocInfoByKeyAndOwner( return nil, fmt.Errorf("find document by key: %w", err) } if !createDocIfNotExist && raw == nil { - raw, err = txn.First(tblDocuments, "project_id_key", projectID.String(), key.String()) + raw, err = txn.First( + tblDocuments, + "project_id_key", + clientRefKey.ProjectID.String(), + key.String(), + ) if err != nil { return nil, fmt.Errorf("find document by key: %w", err) } @@ -707,9 +718,9 @@ func (d *DB) FindDocInfoByKeyAndOwner( if raw == nil { docInfo = &database.DocInfo{ ID: newID(), - ProjectID: projectID, + ProjectID: clientRefKey.ProjectID, Key: key, - Owner: clientID, + Owner: clientRefKey.ClientID, ServerSeq: 0, CreatedAt: now, AccessedAt: now, @@ -745,27 +756,26 @@ func (d *DB) FindDocInfoByKey( return raw.(*database.DocInfo).DeepCopy(), nil } -// FindDocInfoByID finds a docInfo of the given ID. -func (d *DB) FindDocInfoByID( +// FindDocInfoByRefKey finds a docInfo of the given refKey. +func (d *DB) FindDocInfoByRefKey( _ context.Context, - projectID types.ID, - id types.ID, + refKey types.DocRefKey, ) (*database.DocInfo, error) { txn := d.db.Txn(true) defer txn.Abort() - raw, err := txn.First(tblDocuments, "id", id.String()) + raw, err := txn.First(tblDocuments, "id", refKey.DocID.String()) if err != nil { return nil, fmt.Errorf("find document by id: %w", err) } if raw == nil { - return nil, fmt.Errorf("finding doc info by ID(%s): %w", id, database.ErrDocumentNotFound) + return nil, fmt.Errorf("finding doc info by ID(%s): %w", refKey.DocID, database.ErrDocumentNotFound) } docInfo := raw.(*database.DocInfo) - if docInfo.ProjectID != projectID { - return nil, fmt.Errorf("finding doc info by ID(%s): %w", id, database.ErrDocumentNotFound) + if docInfo.ProjectID != refKey.ProjectID { + return nil, fmt.Errorf("finding doc info by ID(%s): %w", refKey.DocID, database.ErrDocumentNotFound) } return docInfo.DeepCopy(), nil @@ -774,24 +784,23 @@ func (d *DB) FindDocInfoByID( // UpdateDocInfoStatusToRemoved updates the status of the document to removed. func (d *DB) UpdateDocInfoStatusToRemoved( _ context.Context, - projectID types.ID, - id types.ID, + refKey types.DocRefKey, ) error { txn := d.db.Txn(true) defer txn.Abort() - raw, err := txn.First(tblDocuments, "id", id.String()) + raw, err := txn.First(tblDocuments, "id", refKey.DocID.String()) if err != nil { return fmt.Errorf("find document by id: %w", err) } if raw == nil { - return fmt.Errorf("finding doc info by ID(%s): %w", id, database.ErrDocumentNotFound) + return fmt.Errorf("finding doc info by ID(%s): %w", refKey.DocID, database.ErrDocumentNotFound) } docInfo := raw.(*database.DocInfo) - if docInfo.ProjectID != projectID { - return fmt.Errorf("finding doc info by ID(%s): %w", id, database.ErrDocumentNotFound) + if docInfo.ProjectID != refKey.ProjectID { + return fmt.Errorf("finding doc info by ID(%s): %w", refKey.DocID, database.ErrDocumentNotFound) } docInfo.RemovedAt = gotime.Now() @@ -833,6 +842,7 @@ func (d *DB) CreateChangeInfos( if err := txn.Insert(tblChanges, &database.ChangeInfo{ ID: newID(), + ProjectID: docInfo.ProjectID, DocID: docInfo.ID, ServerSeq: cn.ServerSeq(), ActorID: types.ID(cn.ID().ActorID().String()), @@ -885,7 +895,7 @@ func (d *DB) CreateChangeInfos( // save storage. func (d *DB) PurgeStaleChanges( _ context.Context, - docID types.ID, + docRefKey types.DocRefKey, ) error { txn := d.db.Txn(true) defer txn.Abort() @@ -900,7 +910,7 @@ func (d *DB) PurgeStaleChanges( minSyncedServerSeq := change.MaxServerSeq for raw := it.Next(); raw != nil; raw = it.Next() { info := raw.(*database.SyncedSeqInfo) - if info.DocID == docID && info.ServerSeq < minSyncedServerSeq { + if info.DocID == docRefKey.DocID && info.ServerSeq < minSyncedServerSeq { minSyncedServerSeq = info.ServerSeq } } @@ -912,7 +922,7 @@ func (d *DB) PurgeStaleChanges( iterator, err := txn.ReverseLowerBound( tblChanges, "doc_id_server_seq", - docID.String(), + docRefKey.DocID.String(), minSyncedServerSeq, ) if err != nil { @@ -931,11 +941,11 @@ func (d *DB) PurgeStaleChanges( // FindChangesBetweenServerSeqs returns the changes between two server sequences. func (d *DB) FindChangesBetweenServerSeqs( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*change.Change, error) { - infos, err := d.FindChangeInfosBetweenServerSeqs(ctx, docID, from, to) + infos, err := d.FindChangeInfosBetweenServerSeqs(ctx, docRefKey, from, to) if err != nil { return nil, err } @@ -956,7 +966,7 @@ func (d *DB) FindChangesBetweenServerSeqs( // FindChangeInfosBetweenServerSeqs returns the changeInfos between two server sequences. func (d *DB) FindChangeInfosBetweenServerSeqs( _ context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*database.ChangeInfo, error) { @@ -968,7 +978,7 @@ func (d *DB) FindChangeInfosBetweenServerSeqs( iterator, err := txn.LowerBound( tblChanges, "doc_id_server_seq", - docID.String(), + docRefKey.DocID.String(), from, ) if err != nil { @@ -977,7 +987,7 @@ func (d *DB) FindChangeInfosBetweenServerSeqs( for raw := iterator.Next(); raw != nil; raw = iterator.Next() { info := raw.(*database.ChangeInfo) - if info.DocID != docID || info.ServerSeq > to { + if info.DocID != docRefKey.DocID || info.ServerSeq > to { break } infos = append(infos, info.DeepCopy()) @@ -988,7 +998,7 @@ func (d *DB) FindChangeInfosBetweenServerSeqs( // CreateSnapshotInfo stores the snapshot of the given document. func (d *DB) CreateSnapshotInfo( _ context.Context, - docID types.ID, + docRefKey types.DocRefKey, doc *document.InternalDocument, ) error { snapshot, err := converter.SnapshotToBytes(doc.RootObject(), doc.AllPresences()) @@ -1001,7 +1011,8 @@ func (d *DB) CreateSnapshotInfo( if err := txn.Insert(tblSnapshots, &database.SnapshotInfo{ ID: newID(), - DocID: docID, + ProjectID: docRefKey.ProjectID, + DocID: docRefKey.DocID, ServerSeq: doc.Checkpoint().ServerSeq, Lamport: doc.Lamport(), Snapshot: snapshot, @@ -1013,16 +1024,22 @@ func (d *DB) CreateSnapshotInfo( return nil } -// FindSnapshotInfoByID returns the snapshot by the given id. -func (d *DB) FindSnapshotInfoByID(_ context.Context, id types.ID) (*database.SnapshotInfo, error) { +// FindSnapshotInfoByRefKey returns the snapshot by the given refKey. +func (d *DB) FindSnapshotInfoByRefKey( + _ context.Context, + refKey types.SnapshotRefKey, +) (*database.SnapshotInfo, error) { txn := d.db.Txn(false) defer txn.Abort() - raw, err := txn.First(tblSnapshots, "id", id.String()) + raw, err := txn.First(tblSnapshots, "doc_id_server_seq", + refKey.DocID.String(), + refKey.ServerSeq, + ) if err != nil { return nil, fmt.Errorf("find snapshot by id: %w", err) } if raw == nil { - return nil, fmt.Errorf("%s: %w", id, database.ErrSnapshotNotFound) + return nil, fmt.Errorf("%s: %w", refKey, database.ErrSnapshotNotFound) } return raw.(*database.SnapshotInfo).DeepCopy(), nil @@ -1031,7 +1048,7 @@ func (d *DB) FindSnapshotInfoByID(_ context.Context, id types.ID) (*database.Sna // FindClosestSnapshotInfo finds the last snapshot of the given document. func (d *DB) FindClosestSnapshotInfo( _ context.Context, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, includeSnapshot bool, ) (*database.SnapshotInfo, error) { @@ -1041,7 +1058,7 @@ func (d *DB) FindClosestSnapshotInfo( iterator, err := txn.ReverseLowerBound( tblSnapshots, "doc_id_server_seq", - docID.String(), + docRefKey.DocID.String(), serverSeq, ) if err != nil { @@ -1051,9 +1068,10 @@ func (d *DB) FindClosestSnapshotInfo( var snapshotInfo *database.SnapshotInfo for raw := iterator.Next(); raw != nil; raw = iterator.Next() { info := raw.(*database.SnapshotInfo) - if info.DocID == docID { + if info.DocID == docRefKey.DocID { snapshotInfo = &database.SnapshotInfo{ ID: info.ID, + ProjectID: info.ProjectID, DocID: info.DocID, ServerSeq: info.ServerSeq, Lamport: info.Lamport, @@ -1076,7 +1094,7 @@ func (d *DB) FindClosestSnapshotInfo( // FindMinSyncedSeqInfo finds the minimum synced sequence info. func (d *DB) FindMinSyncedSeqInfo( _ context.Context, - docID types.ID, + docRefKey types.DocRefKey, ) (*database.SyncedSeqInfo, error) { txn := d.db.Txn(false) defer txn.Abort() @@ -1090,7 +1108,7 @@ func (d *DB) FindMinSyncedSeqInfo( minSyncedServerSeq := change.MaxServerSeq for raw := it.Next(); raw != nil; raw = it.Next() { info := raw.(*database.SyncedSeqInfo) - if info.DocID == docID && info.ServerSeq < minSyncedServerSeq { + if info.DocID == docRefKey.DocID && info.ServerSeq < minSyncedServerSeq { minSyncedServerSeq = info.ServerSeq syncedSeqInfo = info } @@ -1107,10 +1125,10 @@ func (d *DB) FindMinSyncedSeqInfo( func (d *DB) UpdateAndFindMinSyncedTicket( ctx context.Context, clientInfo *database.ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) (*time.Ticket, error) { - if err := d.UpdateSyncedSeq(ctx, clientInfo, docID, serverSeq); err != nil { + if err := d.UpdateSyncedSeq(ctx, clientInfo, docRefKey, serverSeq); err != nil { return nil, err } @@ -1120,18 +1138,18 @@ func (d *DB) UpdateAndFindMinSyncedTicket( iterator, err := txn.LowerBound( tblSyncedSeqs, "doc_id_lamport_actor_id", - docID.String(), + docRefKey.DocID.String(), int64(0), time.InitialActorID.String(), ) if err != nil { - return nil, fmt.Errorf("fetch smallest syncedseq of %s: %w", docID.String(), err) + return nil, fmt.Errorf("fetch smallest syncedseq of %s: %w", docRefKey.DocID.String(), err) } var syncedSeqInfo *database.SyncedSeqInfo if raw := iterator.Next(); raw != nil { info := raw.(*database.SyncedSeqInfo) - if info.DocID == docID { + if info.DocID == docRefKey.DocID { syncedSeqInfo = info } } @@ -1156,13 +1174,13 @@ func (d *DB) UpdateAndFindMinSyncedTicket( func (d *DB) UpdateSyncedSeq( _ context.Context, clientInfo *database.ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) error { txn := d.db.Txn(true) defer txn.Abort() - isAttached, err := clientInfo.IsAttached(docID) + isAttached, err := clientInfo.IsAttached(docRefKey.DocID) if err != nil { return err } @@ -1171,16 +1189,16 @@ func (d *DB) UpdateSyncedSeq( if _, err = txn.DeleteAll( tblSyncedSeqs, "doc_id_client_id", - docID.String(), + docRefKey.DocID.String(), clientInfo.ID.String(), ); err != nil { - return fmt.Errorf("delete syncedseqs of %s: %w", docID.String(), err) + return fmt.Errorf("delete syncedseqs of %s: %w", docRefKey.DocID.String(), err) } txn.Commit() return nil } - ticket, err := d.findTicketByServerSeq(txn, docID, serverSeq) + ticket, err := d.findTicketByServerSeq(txn, docRefKey, serverSeq) if err != nil { return err } @@ -1188,15 +1206,16 @@ func (d *DB) UpdateSyncedSeq( raw, err := txn.First( tblSyncedSeqs, "doc_id_client_id", - docID.String(), + docRefKey.DocID.String(), clientInfo.ID.String(), ) if err != nil { - return fmt.Errorf("fetch syncedseqs of %s: %w", docID.String(), err) + return fmt.Errorf("fetch syncedseqs of %s: %w", docRefKey.DocID.String(), err) } syncedSeqInfo := &database.SyncedSeqInfo{ - DocID: docID, + ProjectID: docRefKey.ProjectID, + DocID: docRefKey.DocID, ClientID: clientInfo.ID, Lamport: ticket.Lamport(), ActorID: types.ID(ticket.ActorID().String()), @@ -1209,7 +1228,7 @@ func (d *DB) UpdateSyncedSeq( } if err := txn.Insert(tblSyncedSeqs, syncedSeqInfo); err != nil { - return fmt.Errorf("insert syncedseqs of %s: %w", docID.String(), err) + return fmt.Errorf("insert syncedseqs of %s: %w", docRefKey.DocID.String(), err) } txn.Commit() @@ -1300,14 +1319,13 @@ func (d *DB) FindDocInfosByQuery( // IsDocumentAttached returns whether the document is attached to clients. func (d *DB) IsDocumentAttached( _ context.Context, - projectID types.ID, - docID types.ID, + refKey types.DocRefKey, excludeClientID types.ID, ) (bool, error) { txn := d.db.Txn(false) defer txn.Abort() - it, err := txn.Get(tblClients, "project_id", projectID.String()) + it, err := txn.Get(tblClients, "project_id", refKey.ProjectID.String()) if err != nil { return false, fmt.Errorf("%w", err) } @@ -1320,7 +1338,7 @@ func (d *DB) IsDocumentAttached( if clientInfo.ID == excludeClientID { continue } - clientDocInfo := clientInfo.Documents[docID] + clientDocInfo := clientInfo.Documents[refKey.DocID] if clientDocInfo == nil { continue } @@ -1334,7 +1352,7 @@ func (d *DB) IsDocumentAttached( func (d *DB) findTicketByServerSeq( txn *memdb.Txn, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) (*time.Ticket, error) { if serverSeq == change.InitialServerSeq { @@ -1344,16 +1362,16 @@ func (d *DB) findTicketByServerSeq( raw, err := txn.First( tblChanges, "doc_id_server_seq", - docID.String(), + docRefKey.DocID.String(), serverSeq, ) if err != nil { - return nil, fmt.Errorf("fetch change of %s: %w", docID.String(), err) + return nil, fmt.Errorf("fetch change of %s: %w", docRefKey.DocID.String(), err) } if raw == nil { return nil, fmt.Errorf( "docID %s, serverSeq %d: %w", - docID.String(), + docRefKey.DocID.String(), serverSeq, database.ErrDocumentNotFound, ) diff --git a/server/backend/database/memory/database_test.go b/server/backend/database/memory/database_test.go index 9e5cb3512..cc36ce8f7 100644 --- a/server/backend/database/memory/database_test.go +++ b/server/backend/database/memory/database_test.go @@ -68,6 +68,10 @@ func TestDB(t *testing.T) { testcases.RunFindUserInfoByIDTest(t, db) }) + t.Run("FindUserInfoByName test", func(t *testing.T) { + testcases.RunFindUserInfoByNameTest(t, db) + }) + t.Run("FindProjectInfoBySecretKey test", func(t *testing.T) { testcases.RunFindProjectInfoBySecretKeyTest(t, db) }) diff --git a/server/backend/database/mongo/client.go b/server/backend/database/mongo/client.go index 17cff8d41..6a9f40e38 100644 --- a/server/backend/database/mongo/client.go +++ b/server/backend/database/mongo/client.go @@ -60,7 +60,7 @@ func Dial(conf *Config) (*Client, error) { ctx, options.Client(). ApplyURI(conf.ConnectionURI). - SetRegistry(newRegistryBuilder().Build()), + SetRegistry(NewRegistryBuilder().Build()), ) if err != nil { return nil, fmt.Errorf("connect to mongo: %w", err) @@ -131,7 +131,7 @@ func (c *Client) ensureDefaultUserInfo( hashedPassword, ) - _, err = c.collection(colUsers).UpdateOne(ctx, bson.M{ + _, err = c.collection(ColUsers).UpdateOne(ctx, bson.M{ "username": candidate.Username, }, bson.M{ "$setOnInsert": bson.M{ @@ -144,7 +144,7 @@ func (c *Client) ensureDefaultUserInfo( return nil, fmt.Errorf("upsert default user info: %w", err) } - result := c.collection(colUsers).FindOne(ctx, bson.M{ + result := c.collection(ColUsers).FindOne(ctx, bson.M{ "username": candidate.Username, }) @@ -167,21 +167,13 @@ func (c *Client) ensureDefaultProjectInfo( ) (*database.ProjectInfo, error) { candidate := database.NewProjectInfo(database.DefaultProjectName, defaultUserID, defaultClientDeactivateThreshold) candidate.ID = database.DefaultProjectID - encodedID, err := encodeID(candidate.ID) - if err != nil { - return nil, err - } - encodedDefaultUserID, err := encodeID(defaultUserID) - if err != nil { - return nil, err - } - _, err = c.collection(colProjects).UpdateOne(ctx, bson.M{ - "_id": encodedID, + _, err := c.collection(ColProjects).UpdateOne(ctx, bson.M{ + "_id": candidate.ID, }, bson.M{ "$setOnInsert": bson.M{ "name": candidate.Name, - "owner": encodedDefaultUserID, + "owner": candidate.Owner, "client_deactivate_threshold": candidate.ClientDeactivateThreshold, "public_key": candidate.PublicKey, "secret_key": candidate.SecretKey, @@ -192,8 +184,8 @@ func (c *Client) ensureDefaultProjectInfo( return nil, fmt.Errorf("create default project: %w", err) } - result := c.collection(colProjects).FindOne(ctx, bson.M{ - "_id": encodedID, + result := c.collection(ColProjects).FindOne(ctx, bson.M{ + "_id": candidate.ID, }) info := database.ProjectInfo{} @@ -214,15 +206,10 @@ func (c *Client) CreateProjectInfo( owner types.ID, clientDeactivateThreshold string, ) (*database.ProjectInfo, error) { - encodedOwner, err := encodeID(owner) - if err != nil { - return nil, err - } - info := database.NewProjectInfo(name, owner, clientDeactivateThreshold) - result, err := c.collection(colProjects).InsertOne(ctx, bson.M{ + result, err := c.collection(ColProjects).InsertOne(ctx, bson.M{ "name": info.Name, - "owner": encodedOwner, + "owner": owner, "client_deactivate_threshold": info.ClientDeactivateThreshold, "public_key": info.PublicKey, "secret_key": info.SecretKey, @@ -246,17 +233,12 @@ func (c *Client) FindNextNCyclingProjectInfos( pageSize int, lastProjectID types.ID, ) ([]*database.ProjectInfo, error) { - encodedID, err := encodeID(lastProjectID) - if err != nil { - return nil, err - } - opts := options.Find() opts.SetLimit(int64(pageSize)) - cursor, err := c.collection(colProjects).Find(ctx, bson.M{ + cursor, err := c.collection(ColProjects).Find(ctx, bson.M{ "_id": bson.M{ - "$gt": encodedID, + "$gt": lastProjectID, }, }, opts) if err != nil { @@ -271,9 +253,9 @@ func (c *Client) FindNextNCyclingProjectInfos( if len(infos) < pageSize { opts.SetLimit(int64(pageSize - len(infos))) - cursor, err := c.collection(colProjects).Find(ctx, bson.M{ + cursor, err := c.collection(ColProjects).Find(ctx, bson.M{ "_id": bson.M{ - "$lte": encodedID, + "$lte": lastProjectID, }, }, opts) if err != nil { @@ -295,13 +277,8 @@ func (c *Client) ListProjectInfos( ctx context.Context, owner types.ID, ) ([]*database.ProjectInfo, error) { - encodedOwnerID, err := encodeID(owner) - if err != nil { - return nil, err - } - - cursor, err := c.collection(colProjects).Find(ctx, bson.M{ - "owner": encodedOwnerID, + cursor, err := c.collection(ColProjects).Find(ctx, bson.M{ + "owner": owner, }) if err != nil { return nil, fmt.Errorf("fetch project infos: %w", err) @@ -317,7 +294,7 @@ func (c *Client) ListProjectInfos( // FindProjectInfoByPublicKey returns a project by public key. func (c *Client) FindProjectInfoByPublicKey(ctx context.Context, publicKey string) (*database.ProjectInfo, error) { - result := c.collection(colProjects).FindOne(ctx, bson.M{ + result := c.collection(ColProjects).FindOne(ctx, bson.M{ "public_key": publicKey, }) @@ -334,7 +311,7 @@ func (c *Client) FindProjectInfoByPublicKey(ctx context.Context, publicKey strin // FindProjectInfoBySecretKey returns a project by secret key. func (c *Client) FindProjectInfoBySecretKey(ctx context.Context, secretKey string) (*database.ProjectInfo, error) { - result := c.collection(colProjects).FindOne(ctx, bson.M{ + result := c.collection(ColProjects).FindOne(ctx, bson.M{ "secret_key": secretKey, }) @@ -355,14 +332,9 @@ func (c *Client) FindProjectInfoByName( owner types.ID, name string, ) (*database.ProjectInfo, error) { - encodedOwner, err := encodeID(owner) - if err != nil { - return nil, err - } - - result := c.collection(colProjects).FindOne(ctx, bson.M{ + result := c.collection(ColProjects).FindOne(ctx, bson.M{ "name": name, - "owner": encodedOwner, + "owner": owner, }) projectInfo := database.ProjectInfo{} @@ -378,13 +350,8 @@ func (c *Client) FindProjectInfoByName( // FindProjectInfoByID returns a project by the given id. func (c *Client) FindProjectInfoByID(ctx context.Context, id types.ID) (*database.ProjectInfo, error) { - encodedID, err := encodeID(id) - if err != nil { - return nil, err - } - - result := c.collection(colProjects).FindOne(ctx, bson.M{ - "_id": encodedID, + result := c.collection(ColProjects).FindOne(ctx, bson.M{ + "_id": id, }) projectInfo := database.ProjectInfo{} @@ -405,15 +372,6 @@ func (c *Client) UpdateProjectInfo( id types.ID, fields *types.UpdatableProjectFields, ) (*database.ProjectInfo, error) { - encodedOwner, err := encodeID(owner) - if err != nil { - return nil, err - } - encodedID, err := encodeID(id) - if err != nil { - return nil, err - } - // Convert UpdatableProjectFields to bson.M updatableFields := bson.M{} data, err := bson.Marshal(fields) @@ -425,9 +383,9 @@ func (c *Client) UpdateProjectInfo( } updatableFields["updated_at"] = gotime.Now() - res := c.collection(colProjects).FindOneAndUpdate(ctx, bson.M{ - "_id": encodedID, - "owner": encodedOwner, + res := c.collection(ColProjects).FindOneAndUpdate(ctx, bson.M{ + "_id": id, + "owner": owner, }, bson.M{ "$set": updatableFields, }, options.FindOneAndUpdate().SetReturnDocument(options.After)) @@ -453,7 +411,7 @@ func (c *Client) CreateUserInfo( hashedPassword string, ) (*database.UserInfo, error) { info := database.NewUserInfo(username, hashedPassword) - result, err := c.collection(colUsers).InsertOne(ctx, bson.M{ + result, err := c.collection(ColUsers).InsertOne(ctx, bson.M{ "username": info.Username, "hashed_password": info.HashedPassword, "created_at": info.CreatedAt, @@ -470,9 +428,26 @@ func (c *Client) CreateUserInfo( return info, nil } -// FindUserInfo returns a user by username. -func (c *Client) FindUserInfo(ctx context.Context, username string) (*database.UserInfo, error) { - result := c.collection(colUsers).FindOne(ctx, bson.M{ +// FindUserInfoByID returns a user by ID. +func (c *Client) FindUserInfoByID(ctx context.Context, clientID types.ID) (*database.UserInfo, error) { + result := c.collection(ColUsers).FindOne(ctx, bson.M{ + "_id": clientID, + }) + + userInfo := database.UserInfo{} + if err := result.Decode(&userInfo); err != nil { + if err == mongo.ErrNoDocuments { + return nil, fmt.Errorf("%s: %w", clientID, database.ErrUserNotFound) + } + return nil, fmt.Errorf("decode user info: %w", err) + } + + return &userInfo, nil +} + +// FindUserInfoByName returns a user by username. +func (c *Client) FindUserInfoByName(ctx context.Context, username string) (*database.UserInfo, error) { + result := c.collection(ColUsers).FindOne(ctx, bson.M{ "username": username, }) @@ -491,7 +466,7 @@ func (c *Client) FindUserInfo(ctx context.Context, username string) (*database.U func (c *Client) ListUserInfos( ctx context.Context, ) ([]*database.UserInfo, error) { - cursor, err := c.collection(colUsers).Find(ctx, bson.M{}) + cursor, err := c.collection(ColUsers).Find(ctx, bson.M{}) if err != nil { return nil, fmt.Errorf("list user infos: %w", err) } @@ -504,38 +479,11 @@ func (c *Client) ListUserInfos( return infos, nil } -// FindUserInfoByID returns a user by ID. -func (c *Client) FindUserInfoByID(ctx context.Context, clientID types.ID) (*database.UserInfo, error) { - encodedClientID, err := encodeID(clientID) - if err != nil { - return nil, err - } - - result := c.collection(colUsers).FindOne(ctx, bson.M{ - "_id": encodedClientID, - }) - - userInfo := database.UserInfo{} - if err := result.Decode(&userInfo); err != nil { - if err == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrUserNotFound) - } - return nil, fmt.Errorf("decode user info: %w", err) - } - - return &userInfo, nil -} - // ActivateClient activates the client of the given key. func (c *Client) ActivateClient(ctx context.Context, projectID types.ID, key string) (*database.ClientInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - now := gotime.Now() - res, err := c.collection(colClients).UpdateOne(ctx, bson.M{ - "project_id": encodedProjectID, + res, err := c.collection(ColClients).UpdateOne(ctx, bson.M{ + "project_id": projectID, "key": key, }, bson.M{ "$set": bson.M{ @@ -549,16 +497,18 @@ func (c *Client) ActivateClient(ctx context.Context, projectID types.ID, key str var result *mongo.SingleResult if res.UpsertedCount > 0 { - result = c.collection(colClients).FindOneAndUpdate(ctx, bson.M{ - "_id": res.UpsertedID, + result = c.collection(ColClients).FindOneAndUpdate(ctx, bson.M{ + "project_id": projectID, + "_id": res.UpsertedID, }, bson.M{ "$set": bson.M{ "created_at": now, }, }) } else { - result = c.collection(colClients).FindOne(ctx, bson.M{ - "key": key, + result = c.collection(ColClients).FindOne(ctx, bson.M{ + "project_id": projectID, + "key": key, }) } @@ -570,20 +520,11 @@ func (c *Client) ActivateClient(ctx context.Context, projectID types.ID, key str return &clientInfo, nil } -// DeactivateClient deactivates the client of the given ID. -func (c *Client) DeactivateClient(ctx context.Context, projectID, clientID types.ID) (*database.ClientInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - encodedClientID, err := encodeID(clientID) - if err != nil { - return nil, err - } - - res := c.collection(colClients).FindOneAndUpdate(ctx, bson.M{ - "_id": encodedClientID, - "project_id": encodedProjectID, +// DeactivateClient deactivates the client of the given refKey. +func (c *Client) DeactivateClient(ctx context.Context, refKey types.ClientRefKey) (*database.ClientInfo, error) { + res := c.collection(ColClients).FindOneAndUpdate(ctx, bson.M{ + "project_id": refKey.ProjectID, + "_id": refKey.ClientID, }, bson.M{ "$set": bson.M{ "status": database.ClientDeactivated, @@ -594,7 +535,7 @@ func (c *Client) DeactivateClient(ctx context.Context, projectID, clientID types clientInfo := database.ClientInfo{} if err := res.Decode(&clientInfo); err != nil { if err == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrClientNotFound) + return nil, fmt.Errorf("%s: %w", refKey, database.ErrClientNotFound) } return nil, fmt.Errorf("decode client info: %w", err) } @@ -602,20 +543,11 @@ func (c *Client) DeactivateClient(ctx context.Context, projectID, clientID types return &clientInfo, nil } -// FindClientInfoByID finds the client of the given ID. -func (c *Client) FindClientInfoByID(ctx context.Context, projectID, clientID types.ID) (*database.ClientInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - encodedClientID, err := encodeID(clientID) - if err != nil { - return nil, err - } - - result := c.collection(colClients).FindOneAndUpdate(ctx, bson.M{ - "_id": encodedClientID, - "project_id": encodedProjectID, +// FindClientInfoByRefKey finds the client of the given refKey. +func (c *Client) FindClientInfoByRefKey(ctx context.Context, refKey types.ClientRefKey) (*database.ClientInfo, error) { + result := c.collection(ColClients).FindOneAndUpdate(ctx, bson.M{ + "project_id": refKey.ProjectID, + "_id": refKey.ClientID, }, bson.M{ "$set": bson.M{ "updated_at": gotime.Now(), @@ -625,7 +557,7 @@ func (c *Client) FindClientInfoByID(ctx context.Context, projectID, clientID typ clientInfo := database.ClientInfo{} if err := result.Decode(&clientInfo); err != nil { if err == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%s: %w", clientID, database.ErrClientNotFound) + return nil, fmt.Errorf("%s: %w", refKey, database.ErrClientNotFound) } } @@ -639,12 +571,7 @@ func (c *Client) UpdateClientInfoAfterPushPull( clientInfo *database.ClientInfo, docInfo *database.DocInfo, ) error { - encodedClientID, err := encodeID(clientInfo.ID) - if err != nil { - return err - } - - clientDocInfoKey := "documents." + docInfo.ID.String() + "." + clientDocInfoKey := getClientDocInfoKey(docInfo.ID) clientDocInfo, ok := clientInfo.Documents[docInfo.ID] if !ok { return fmt.Errorf("client doc info: %w", database.ErrDocumentNeverAttached) @@ -677,8 +604,9 @@ func (c *Client) UpdateClientInfoAfterPushPull( } } - result := c.collection(colClients).FindOneAndUpdate(ctx, bson.M{ - "_id": encodedClientID, + result := c.collection(ColClients).FindOneAndUpdate(ctx, bson.M{ + "project_id": clientInfo.ProjectID, + "_id": clientInfo.ID, }, updater) if result.Err() != nil { @@ -697,18 +625,13 @@ func (c *Client) FindDeactivateCandidatesPerProject( project *database.ProjectInfo, candidatesLimit int, ) ([]*database.ClientInfo, error) { - encodedProjectID, err := encodeID(project.ID) - if err != nil { - return nil, err - } - clientDeactivateThreshold, err := project.ClientDeactivateThresholdAsTimeDuration() if err != nil { return nil, err } - cursor, err := c.collection(colClients).Find(ctx, bson.M{ - "project_id": encodedProjectID, + cursor, err := c.collection(ColClients).Find(ctx, bson.M{ + "project_id": project.ID, "status": database.ClientActivated, "updated_at": bson.M{ "$lte": gotime.Now().Add(-clientDeactivateThreshold), @@ -732,29 +655,19 @@ func (c *Client) FindDeactivateCandidatesPerProject( // exist. func (c *Client) FindDocInfoByKeyAndOwner( ctx context.Context, - projectID types.ID, - clientID types.ID, + clientRefKey types.ClientRefKey, docKey key.Key, createDocIfNotExist bool, ) (*database.DocInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - encodedOwnerID, err := encodeID(clientID) - if err != nil { - return nil, err - } - filter := bson.M{ - "project_id": encodedProjectID, + "project_id": clientRefKey.ProjectID, "key": docKey, "removed_at": bson.M{ "$exists": false, }, } now := gotime.Now() - res, err := c.collection(colDocuments).UpdateOne(ctx, filter, bson.M{ + res, err := c.collection(ColDocuments).UpdateOne(ctx, filter, bson.M{ "$set": bson.M{ "accessed_at": now, }, @@ -765,19 +678,25 @@ func (c *Client) FindDocInfoByKeyAndOwner( var result *mongo.SingleResult if res.UpsertedCount > 0 { - result = c.collection(colDocuments).FindOneAndUpdate(ctx, bson.M{ - "_id": res.UpsertedID, + result = c.collection(ColDocuments).FindOneAndUpdate(ctx, bson.M{ + "project_id": clientRefKey.ProjectID, + "_id": res.UpsertedID, }, bson.M{ "$set": bson.M{ - "owner": encodedOwnerID, + "owner": clientRefKey.ClientID, "server_seq": 0, "created_at": now, }, }) } else { - result = c.collection(colDocuments).FindOne(ctx, filter) + result = c.collection(ColDocuments).FindOne(ctx, filter) if result.Err() == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%s %s: %w", projectID, docKey, database.ErrDocumentNotFound) + return nil, fmt.Errorf( + "%s %s: %w", + clientRefKey.ProjectID, + docKey, + database.ErrDocumentNotFound, + ) } if result.Err() != nil { return nil, fmt.Errorf("find document: %w", result.Err()) @@ -798,13 +717,8 @@ func (c *Client) FindDocInfoByKey( projectID types.ID, docKey key.Key, ) (*database.DocInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - - result := c.collection(colDocuments).FindOne(ctx, bson.M{ - "project_id": encodedProjectID, + result := c.collection(ColDocuments).FindOne(ctx, bson.M{ + "project_id": projectID, "key": docKey, "removed_at": bson.M{ "$exists": false, @@ -825,28 +739,17 @@ func (c *Client) FindDocInfoByKey( return &docInfo, nil } -// FindDocInfoByID finds a docInfo of the given ID. -func (c *Client) FindDocInfoByID( +// FindDocInfoByRefKey finds a docInfo of the given refKey. +func (c *Client) FindDocInfoByRefKey( ctx context.Context, - projectID types.ID, - id types.ID, + refKey types.DocRefKey, ) (*database.DocInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - - encodedDocID, err := encodeID(id) - if err != nil { - return nil, err - } - - result := c.collection(colDocuments).FindOne(ctx, bson.M{ - "_id": encodedDocID, - "project_id": encodedProjectID, + result := c.collection(ColDocuments).FindOne(ctx, bson.M{ + "project_id": refKey.ProjectID, + "_id": refKey.DocID, }) if result.Err() == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%s: %w", id, database.ErrDocumentNotFound) + return nil, fmt.Errorf("%s: %w", refKey, database.ErrDocumentNotFound) } if result.Err() != nil { return nil, fmt.Errorf("find document: %w", result.Err()) @@ -863,22 +766,11 @@ func (c *Client) FindDocInfoByID( // UpdateDocInfoStatusToRemoved updates the document status to removed. func (c *Client) UpdateDocInfoStatusToRemoved( ctx context.Context, - projectID types.ID, - id types.ID, + refKey types.DocRefKey, ) error { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return err - } - - encodedDocID, err := encodeID(id) - if err != nil { - return err - } - - result := c.collection(colDocuments).FindOneAndUpdate(ctx, bson.M{ - "_id": encodedDocID, - "project_id": encodedProjectID, + result := c.collection(ColDocuments).FindOneAndUpdate(ctx, bson.M{ + "project_id": refKey.ProjectID, + "_id": refKey.DocID, }, bson.M{ "$set": bson.M{ "removed_at": gotime.Now(), @@ -886,7 +778,7 @@ func (c *Client) UpdateDocInfoStatusToRemoved( }, options.FindOneAndUpdate().SetReturnDocument(options.After)) if result.Err() == mongo.ErrNoDocuments { - return fmt.Errorf("%s: %w", id, database.ErrDocumentNotFound) + return fmt.Errorf("%s: %w", refKey, database.ErrDocumentNotFound) } if result.Err() != nil { return fmt.Errorf("update document info status to removed: %w", result.Err()) @@ -904,10 +796,7 @@ func (c *Client) CreateChangeInfos( changes []*change.Change, isRemoved bool, ) error { - encodedDocID, err := encodeID(docInfo.ID) - if err != nil { - return err - } + docRefKey := docInfo.RefKey() var models []mongo.WriteModel for _, cn := range changes { @@ -921,10 +810,11 @@ func (c *Client) CreateChangeInfos( } models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{ - "doc_id": encodedDocID, + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": cn.ServerSeq(), }).SetUpdate(bson.M{"$set": bson.M{ - "actor_id": encodeActorID(cn.ID().ActorID()), + "actor_id": cn.ID().ActorID(), "client_seq": cn.ID().ClientSeq(), "lamport": cn.ID().Lamport(), "message": cn.Message(), @@ -936,7 +826,7 @@ func (c *Client) CreateChangeInfos( // TODO(hackerwins): We need to handle the updates for the two collections // below atomically. if len(changes) > 0 { - if _, err = c.collection(colChanges).BulkWrite( + if _, err := c.collection(ColChanges).BulkWrite( ctx, models, options.BulkWrite().SetOrdered(true), @@ -954,8 +844,9 @@ func (c *Client) CreateChangeInfos( updateFields["removed_at"] = now } - res, err := c.collection(colDocuments).UpdateOne(ctx, bson.M{ - "_id": encodedDocID, + res, err := c.collection(ColDocuments).UpdateOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "_id": docRefKey.DocID, "server_seq": initialServerSeq, }, bson.M{ "$set": updateFields, @@ -964,7 +855,7 @@ func (c *Client) CreateChangeInfos( return fmt.Errorf("update document: %w", err) } if res.MatchedCount == 0 { - return fmt.Errorf("%s: %w", docInfo.ID, database.ErrConflictOnUpdate) + return fmt.Errorf("%s: %w", docRefKey, database.ErrConflictOnUpdate) } if isRemoved { docInfo.RemovedAt = now @@ -977,18 +868,16 @@ func (c *Client) CreateChangeInfos( // save storage. func (c *Client) PurgeStaleChanges( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, ) error { - encodedDocID, err := encodeID(docID) - if err != nil { - return err - } - // Find the smallest server seq in `syncedseqs`. // Because offline client can pull changes when it becomes online. - result := c.collection(colSyncedSeqs).FindOne( + result := c.collection(ColSyncedSeqs).FindOne( ctx, - bson.M{"doc_id": encodedDocID}, + bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, + }, options.FindOne().SetSort(bson.M{"server_seq": 1}), ) if result.Err() == mongo.ErrNoDocuments { @@ -1003,10 +892,11 @@ func (c *Client) PurgeStaleChanges( } // Delete all changes before the smallest server seq. - if _, err := c.collection(colChanges).DeleteMany( + if _, err := c.collection(ColChanges).DeleteMany( ctx, bson.M{ - "doc_id": encodedDocID, + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": bson.M{"$lt": minSyncedSeqInfo.ServerSeq}, }, options.Delete(), @@ -1020,11 +910,11 @@ func (c *Client) PurgeStaleChanges( // FindChangesBetweenServerSeqs returns the changes between two server sequences. func (c *Client) FindChangesBetweenServerSeqs( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*change.Change, error) { - infos, err := c.FindChangeInfosBetweenServerSeqs(ctx, docID, from, to) + infos, err := c.FindChangeInfosBetweenServerSeqs(ctx, docRefKey, from, to) if err != nil { return nil, err } @@ -1044,17 +934,13 @@ func (c *Client) FindChangesBetweenServerSeqs( // FindChangeInfosBetweenServerSeqs returns the changeInfos between two server sequences. func (c *Client) FindChangeInfosBetweenServerSeqs( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, from int64, to int64, ) ([]*database.ChangeInfo, error) { - encodedDocID, err := encodeID(docID) - if err != nil { - return nil, err - } - - cursor, err := c.collection(colChanges).Find(ctx, bson.M{ - "doc_id": encodedDocID, + cursor, err := c.collection(ColChanges).Find(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": bson.M{ "$gte": from, "$lte": to, @@ -1075,20 +961,17 @@ func (c *Client) FindChangeInfosBetweenServerSeqs( // CreateSnapshotInfo stores the snapshot of the given document. func (c *Client) CreateSnapshotInfo( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, doc *document.InternalDocument, ) error { - encodedDocID, err := encodeID(docID) - if err != nil { - return err - } snapshot, err := converter.SnapshotToBytes(doc.RootObject(), doc.AllPresences()) if err != nil { return err } - if _, err := c.collection(colSnapshots).InsertOne(ctx, bson.M{ - "doc_id": encodedDocID, + if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": doc.Checkpoint().ServerSeq, "lamport": doc.Lamport(), "snapshot": snapshot, @@ -1100,18 +983,15 @@ func (c *Client) CreateSnapshotInfo( return nil } -// FindSnapshotInfoByID returns the snapshot by the given id. -func (c *Client) FindSnapshotInfoByID( +// FindSnapshotInfoByRefKey returns the snapshot by the given refKey. +func (c *Client) FindSnapshotInfoByRefKey( ctx context.Context, - id types.ID, + refKey types.SnapshotRefKey, ) (*database.SnapshotInfo, error) { - encodedID, err := encodeID(id) - if err != nil { - return nil, err - } - - result := c.collection(colSnapshots).FindOne(ctx, bson.M{ - "_id": encodedID, + result := c.collection(ColSnapshots).FindOne(ctx, bson.M{ + "project_id": refKey.ProjectID, + "doc_id": refKey.DocID, + "server_seq": refKey.ServerSeq, }) snapshotInfo := &database.SnapshotInfo{} @@ -1132,15 +1012,10 @@ func (c *Client) FindSnapshotInfoByID( // FindClosestSnapshotInfo finds the last snapshot of the given document. func (c *Client) FindClosestSnapshotInfo( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, includeSnapshot bool, ) (*database.SnapshotInfo, error) { - encodedDocID, err := encodeID(docID) - if err != nil { - return nil, err - } - option := options.FindOne().SetSort(bson.M{ "server_seq": -1, }) @@ -1149,8 +1024,9 @@ func (c *Client) FindClosestSnapshotInfo( option.SetProjection(bson.M{"Snapshot": 0}) } - result := c.collection(colSnapshots).FindOne(ctx, bson.M{ - "doc_id": encodedDocID, + result := c.collection(ColSnapshots).FindOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": bson.M{ "$lte": serverSeq, }, @@ -1174,15 +1050,11 @@ func (c *Client) FindClosestSnapshotInfo( // FindMinSyncedSeqInfo finds the minimum synced sequence info. func (c *Client) FindMinSyncedSeqInfo( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, ) (*database.SyncedSeqInfo, error) { - encodedDocID, err := encodeID(docID) - if err != nil { - return nil, err - } - - syncedSeqResult := c.collection(colSyncedSeqs).FindOne(ctx, bson.M{ - "doc_id": encodedDocID, + syncedSeqResult := c.collection(ColSyncedSeqs).FindOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, }, options.FindOne().SetSort(bson.D{ {Key: "server_seq", Value: 1}, })) @@ -1207,21 +1079,17 @@ func (c *Client) FindMinSyncedSeqInfo( func (c *Client) UpdateAndFindMinSyncedTicket( ctx context.Context, clientInfo *database.ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) (*time.Ticket, error) { - if err := c.UpdateSyncedSeq(ctx, clientInfo, docID, serverSeq); err != nil { - return nil, err - } - - encodedDocID, err := encodeID(docID) - if err != nil { + if err := c.UpdateSyncedSeq(ctx, clientInfo, docRefKey, serverSeq); err != nil { return nil, err } // 02. find min synced seq of the given document. - result := c.collection(colSyncedSeqs).FindOne(ctx, bson.M{ - "doc_id": encodedDocID, + result := c.collection(ColSyncedSeqs).FindOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, }, options.FindOne().SetSort(bson.D{ {Key: "lamport", Value: 1}, {Key: "actor_id", Value: 1}, @@ -1259,31 +1127,21 @@ func (c *Client) FindDocInfosByPaging( projectID types.ID, paging types.Paging[types.ID], ) ([]*database.DocInfo, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - filter := bson.M{ "project_id": bson.M{ - "$eq": encodedProjectID, + "$eq": projectID, }, "removed_at": bson.M{ "$exists": false, }, } if paging.Offset != "" { - encodedOffset, err := encodeID(paging.Offset) - if err != nil { - return nil, err - } - k := "$lt" if paging.IsForward { k = "$gt" } filter["_id"] = bson.M{ - k: encodedOffset, + k: paging.Offset, } } @@ -1294,7 +1152,7 @@ func (c *Client) FindDocInfosByPaging( opts = opts.SetSort(map[string]int{"_id": -1}) } - cursor, err := c.collection(colDocuments).Find(ctx, filter, opts) + cursor, err := c.collection(ColDocuments).Find(ctx, filter, opts) if err != nil { return nil, fmt.Errorf("find documents: %w", err) } @@ -1314,13 +1172,8 @@ func (c *Client) FindDocInfosByQuery( query string, pageSize int, ) (*types.SearchResult[*database.DocInfo], error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return nil, err - } - - cursor, err := c.collection(colDocuments).Find(ctx, bson.M{ - "project_id": encodedProjectID, + cursor, err := c.collection(ColDocuments).Find(ctx, bson.M{ + "project_id": projectID, "key": bson.M{"$regex": primitive.Regex{ Pattern: "^" + escapeRegex(query), }}, @@ -1348,46 +1201,39 @@ func (c *Client) FindDocInfosByQuery( func (c *Client) UpdateSyncedSeq( ctx context.Context, clientInfo *database.ClientInfo, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) error { - encodedDocID, err := encodeID(docID) - if err != nil { - return err - } - encodedClientID, err := encodeID(clientInfo.ID) - if err != nil { - return err - } - // 01. update synced seq of the given client. - isAttached, err := clientInfo.IsAttached(docID) + isAttached, err := clientInfo.IsAttached(docRefKey.DocID) if err != nil { return err } if !isAttached { - if _, err = c.collection(colSyncedSeqs).DeleteOne(ctx, bson.M{ - "doc_id": encodedDocID, - "client_id": encodedClientID, + if _, err = c.collection(ColSyncedSeqs).DeleteOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, + "client_id": clientInfo.ID, }, options.Delete()); err != nil { return fmt.Errorf("delete synced seq: %w", err) } return nil } - ticket, err := c.findTicketByServerSeq(ctx, docID, serverSeq) + ticket, err := c.findTicketByServerSeq(ctx, docRefKey, serverSeq) if err != nil { return err } - if _, err = c.collection(colSyncedSeqs).UpdateOne(ctx, bson.M{ - "doc_id": encodedDocID, - "client_id": encodedClientID, + if _, err = c.collection(ColSyncedSeqs).UpdateOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, + "client_id": clientInfo.ID, }, bson.M{ "$set": bson.M{ "lamport": ticket.Lamport(), - "actor_id": encodeActorID(ticket.ActorID()), + "actor_id": ticket.ActorID(), "server_seq": serverSeq, }, }, options.Update().SetUpsert(true)); err != nil { @@ -1400,31 +1246,20 @@ func (c *Client) UpdateSyncedSeq( // IsDocumentAttached returns whether the given document is attached to clients. func (c *Client) IsDocumentAttached( ctx context.Context, - projectID types.ID, - docID types.ID, + docRefKey types.DocRefKey, excludeClientID types.ID, ) (bool, error) { - encodedProjectID, err := encodeID(projectID) - if err != nil { - return false, err - } - - clientDocInfoKey := "documents." + docID.String() + "." + clientDocInfoKey := getClientDocInfoKey(docRefKey.DocID) filter := bson.M{ - "project_id": encodedProjectID, + "project_id": docRefKey.ProjectID, clientDocInfoKey + "status": database.DocumentAttached, } if excludeClientID != "" { - encodedExcludeClientID, err := encodeID(excludeClientID) - if err != nil { - return false, err - } - - filter["_id"] = bson.M{"$ne": encodedExcludeClientID} + filter["_id"] = bson.M{"$ne": excludeClientID} } - result := c.collection(colClients).FindOne(ctx, filter) + result := c.collection(ColClients).FindOne(ctx, filter) if result.Err() == mongo.ErrNoDocuments { return false, nil } @@ -1434,26 +1269,22 @@ func (c *Client) IsDocumentAttached( func (c *Client) findTicketByServerSeq( ctx context.Context, - docID types.ID, + docRefKey types.DocRefKey, serverSeq int64, ) (*time.Ticket, error) { if serverSeq == change.InitialServerSeq { return time.InitialTicket, nil } - encodedDocID, err := encodeID(docID) - if err != nil { - return nil, err - } - - result := c.collection(colChanges).FindOne(ctx, bson.M{ - "doc_id": encodedDocID, + result := c.collection(ColChanges).FindOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, "server_seq": serverSeq, }) if result.Err() == mongo.ErrNoDocuments { return nil, fmt.Errorf( - "change docID=%s serverSeq=%d: %w", - docID.String(), + "change %s serverSeq=%d: %w", + docRefKey, serverSeq, database.ErrDocumentNotFound, ) @@ -1505,3 +1336,7 @@ func escapeRegex(str string) string { } return buf.String() } + +func getClientDocInfoKey(docID types.ID) string { + return fmt.Sprintf("documents.%s.", docID) +} diff --git a/server/backend/database/mongo/client_test.go b/server/backend/database/mongo/client_test.go index 9ed49d7af..f59eca9b3 100644 --- a/server/backend/database/mongo/client_test.go +++ b/server/backend/database/mongo/client_test.go @@ -85,6 +85,10 @@ func TestClient(t *testing.T) { testcases.RunFindUserInfoByIDTest(t, cli) }) + t.Run("FindUserInfoByName test", func(t *testing.T) { + testcases.RunFindUserInfoByNameTest(t, cli) + }) + t.Run("FindProjectInfoBySecretKey test", func(t *testing.T) { testcases.RunFindProjectInfoBySecretKeyTest(t, cli) }) diff --git a/server/backend/database/mongo/indexes.go b/server/backend/database/mongo/indexes.go index 953659b3f..0e1e2a5a5 100644 --- a/server/backend/database/mongo/indexes.go +++ b/server/backend/database/mongo/indexes.go @@ -26,24 +26,42 @@ import ( ) const ( - colProjects = "projects" - colUsers = "users" - colClients = "clients" - colDocuments = "documents" - colChanges = "changes" - colSnapshots = "snapshots" - colSyncedSeqs = "syncedseqs" + // ColProjects represents the projects collection in the database. + ColProjects = "projects" + // ColUsers represents the users collection in the database. + ColUsers = "users" + // ColClients represents the clients collection in the database. + ColClients = "clients" + // ColDocuments represents the documents collection in the database. + ColDocuments = "documents" + // ColChanges represents the changes collection in the database. + ColChanges = "changes" + // ColSnapshots represents the snapshots collection in the database. + ColSnapshots = "snapshots" + // ColSyncedSeqs represents the syncedseqs collection in the database. + ColSyncedSeqs = "syncedseqs" ) +// Collections represents the list of all collections in the database. +var Collections = []string{ + ColProjects, + ColUsers, + ColClients, + ColDocuments, + ColChanges, + ColSnapshots, + ColSyncedSeqs, +} + type collectionInfo struct { name string indexes []mongo.IndexModel } -// Below are names and indexes information of collections that stores Yorkie data. +// Below are names and indexes information of Collections that stores Yorkie data. var collectionInfos = []collectionInfo{ { - name: colProjects, + name: ColProjects, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ {Key: "owner", Value: bsonx.Int32(1)}, @@ -59,17 +77,17 @@ var collectionInfos = []collectionInfo{ }}, }, { - name: colUsers, + name: ColUsers, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{{Key: "username", Value: bsonx.Int32(1)}}, Options: options.Index().SetUnique(true), }}, }, { - name: colClients, + name: ColClients, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ - {Key: "project_id", Value: bsonx.Int32(1)}, + {Key: "project_id", Value: bsonx.Int32(1)}, // shard key {Key: "key", Value: bsonx.Int32(1)}, }, Options: options.Index().SetUnique(true), @@ -86,10 +104,10 @@ var collectionInfos = []collectionInfo{ }}, }, { - name: colDocuments, + name: ColDocuments, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ - {Key: "project_id", Value: bsonx.Int32(1)}, + {Key: "project_id", Value: bsonx.Int32(1)}, // shard key {Key: "key", Value: bsonx.Int32(1)}, }, Options: options.Index().SetPartialFilterExpression( @@ -99,34 +117,38 @@ var collectionInfos = []collectionInfo{ ).SetUnique(true), }}, }, { - name: colChanges, + name: ColChanges, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ - {Key: "doc_id", Value: bsonx.Int32(1)}, + {Key: "doc_id", Value: bsonx.Int32(1)}, // shard key + {Key: "project_id", Value: bsonx.Int32(1)}, {Key: "server_seq", Value: bsonx.Int32(1)}, }, Options: options.Index().SetUnique(true), }}, }, { - name: colSnapshots, + name: ColSnapshots, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ - {Key: "doc_id", Value: bsonx.Int32(1)}, + {Key: "doc_id", Value: bsonx.Int32(1)}, // shard key + {Key: "project_id", Value: bsonx.Int32(1)}, {Key: "server_seq", Value: bsonx.Int32(1)}, }, Options: options.Index().SetUnique(true), }}, }, { - name: colSyncedSeqs, + name: ColSyncedSeqs, indexes: []mongo.IndexModel{{ Keys: bsonx.Doc{ - {Key: "doc_id", Value: bsonx.Int32(1)}, + {Key: "doc_id", Value: bsonx.Int32(1)}, // shard key + {Key: "project_id", Value: bsonx.Int32(1)}, {Key: "client_id", Value: bsonx.Int32(1)}, }, Options: options.Index().SetUnique(true), }, { Keys: bsonx.Doc{ {Key: "doc_id", Value: bsonx.Int32(1)}, + {Key: "project_id", Value: bsonx.Int32(1)}, {Key: "lamport", Value: bsonx.Int32(1)}, {Key: "actor_id", Value: bsonx.Int32(1)}, }, diff --git a/server/backend/database/mongo/registry.go b/server/backend/database/mongo/registry.go index 729059ccb..07e960cb9 100644 --- a/server/backend/database/mongo/registry.go +++ b/server/backend/database/mongo/registry.go @@ -17,26 +17,66 @@ package mongo import ( + "fmt" "reflect" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/bsoncodec" "go.mongodb.org/mongo-driver/bson/bsonoptions" + "go.mongodb.org/mongo-driver/bson/bsonrw" "github.com/yorkie-team/yorkie/api/types" + "github.com/yorkie-team/yorkie/pkg/document/time" + "github.com/yorkie-team/yorkie/server/backend/database" ) -func newRegistryBuilder() *bsoncodec.RegistryBuilder { +var tID = reflect.TypeOf(types.ID("")) +var tActorID = reflect.TypeOf(&time.ActorID{}) +var tClientDocInfoMap = reflect.TypeOf(make(database.ClientDocInfoMap)) + +// NewRegistryBuilder returns a new registry builder with the default encoder and decoder. +func NewRegistryBuilder() *bsoncodec.RegistryBuilder { rb := bsoncodec.NewRegistryBuilder() bsoncodec.DefaultValueEncoders{}.RegisterDefaultEncoders(rb) bsoncodec.DefaultValueDecoders{}.RegisterDefaultDecoders(rb) bson.PrimitiveCodecs{}.RegisterPrimitiveCodecs(rb) + // Register the decoder for ObjectID rb.RegisterCodec( - reflect.TypeOf(types.ID("")), + tID, bsoncodec.NewStringCodec(bsonoptions.StringCodec().SetDecodeObjectIDAsHex(true)), ) + // Register the encoder for types.ID + rb.RegisterTypeEncoder(tID, bsoncodec.ValueEncoderFunc(idEncoder)) + // Register the encoder for time.ActorID + rb.RegisterTypeEncoder(tActorID, bsoncodec.ValueEncoderFunc(actorIDEncoder)) + return rb } + +func idEncoder(_ bsoncodec.EncodeContext, vw bsonrw.ValueWriter, val reflect.Value) error { + if !val.IsValid() || val.Type() != tID { + return bsoncodec.ValueEncoderError{Name: "idEncoder", Types: []reflect.Type{tID}, Received: val} + } + objectID, err := encodeID(val.Interface().(types.ID)) + if err != nil { + return err + } + if err := vw.WriteObjectID(objectID); err != nil { + return fmt.Errorf("encode error: %w", err) + } + return nil +} + +func actorIDEncoder(_ bsoncodec.EncodeContext, vw bsonrw.ValueWriter, val reflect.Value) error { + if !val.IsValid() || val.Type() != tActorID { + return bsoncodec.ValueEncoderError{Name: "actorIDEncoder", Types: []reflect.Type{tActorID}, Received: val} + } + objectID := encodeActorID(val.Interface().(*time.ActorID)) + if err := vw.WriteObjectID(objectID); err != nil { + return fmt.Errorf("encode error: %w", err) + } + return nil +} diff --git a/server/backend/database/mongo/registry_test.go b/server/backend/database/mongo/registry_test.go index de9bd482b..3b4e00afb 100644 --- a/server/backend/database/mongo/registry_test.go +++ b/server/backend/database/mongo/registry_test.go @@ -17,18 +17,23 @@ package mongo import ( + "bytes" + "reflect" "testing" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/bsoncodec" + "go.mongodb.org/mongo-driver/bson/bsonrw" "go.mongodb.org/mongo-driver/bson/primitive" "github.com/yorkie-team/yorkie/api/types" + "github.com/yorkie-team/yorkie/pkg/document/time" "github.com/yorkie-team/yorkie/server/backend/database" ) func TestRegistry(t *testing.T) { - registry := newRegistryBuilder().Build() + registry := NewRegistryBuilder().Build() id := types.ID(primitive.NewObjectID().Hex()) data, err := bson.MarshalWithRegistry(registry, bson.M{ @@ -41,3 +46,44 @@ func TestRegistry(t *testing.T) { assert.Equal(t, id, info.ID) } + +func TestEncoder(t *testing.T) { + t.Run("idEncoder test", func(t *testing.T) { + field := "id" + id := types.ID(primitive.NewObjectID().Hex()) + + buf := new(bytes.Buffer) + vw, err := bsonrw.NewBSONValueWriter(buf) + assert.NoError(t, err) + dw, err := vw.WriteDocument() + assert.NoError(t, err) + vw, err = dw.WriteDocumentElement(field) + assert.NoError(t, err) + + assert.NoError(t, idEncoder(bsoncodec.EncodeContext{}, vw, reflect.ValueOf(id))) + assert.NoError(t, dw.WriteDocumentEnd()) + result := make(map[string]string) + assert.NoError(t, bson.Unmarshal(buf.Bytes(), &result)) + assert.Equal(t, id.String(), result[field]) + }) + + t.Run("actorIDEncoder test", func(t *testing.T) { + field := "actor_id" + actorID, err := time.ActorIDFromHex(primitive.NewObjectID().Hex()) + assert.NoError(t, err) + + buf := new(bytes.Buffer) + vw, err := bsonrw.NewBSONValueWriter(buf) + assert.NoError(t, err) + dw, err := vw.WriteDocument() + assert.NoError(t, err) + vw, err = dw.WriteDocumentElement(field) + assert.NoError(t, err) + + assert.NoError(t, actorIDEncoder(bsoncodec.EncodeContext{}, vw, reflect.ValueOf(actorID))) + assert.NoError(t, dw.WriteDocumentEnd()) + result := make(map[string]string) + assert.NoError(t, bson.Unmarshal(buf.Bytes(), &result)) + assert.Equal(t, actorID.String(), result[field]) + }) +} diff --git a/server/backend/database/snapshot_info.go b/server/backend/database/snapshot_info.go index efa88ea29..ca0255f96 100644 --- a/server/backend/database/snapshot_info.go +++ b/server/backend/database/snapshot_info.go @@ -27,6 +27,9 @@ type SnapshotInfo struct { // ID is the unique ID of the snapshot. ID types.ID `bson:"_id"` + // ProjectID is the ID of the project which the snapshot belongs to. + ProjectID types.ID `bson:"project_id"` + // DocID is the ID of the document which the snapshot belongs to. DocID types.ID `bson:"doc_id"` @@ -51,6 +54,7 @@ func (i *SnapshotInfo) DeepCopy() *SnapshotInfo { return &SnapshotInfo{ ID: i.ID, + ProjectID: i.ProjectID, DocID: i.DocID, ServerSeq: i.ServerSeq, Lamport: i.Lamport, @@ -58,3 +62,14 @@ func (i *SnapshotInfo) DeepCopy() *SnapshotInfo { CreatedAt: i.CreatedAt, } } + +// RefKey returns the refKey of the snapshot. +func (i *SnapshotInfo) RefKey() types.SnapshotRefKey { + return types.SnapshotRefKey{ + DocRefKey: types.DocRefKey{ + ProjectID: i.ProjectID, + DocID: i.DocID, + }, + ServerSeq: i.ServerSeq, + } +} diff --git a/server/backend/database/synced_seq_info.go b/server/backend/database/synced_seq_info.go index 608521a76..489275859 100644 --- a/server/backend/database/synced_seq_info.go +++ b/server/backend/database/synced_seq_info.go @@ -16,12 +16,15 @@ package database -import "github.com/yorkie-team/yorkie/api/types" +import ( + "github.com/yorkie-team/yorkie/api/types" +) // SyncedSeqInfo is a structure representing information about the synchronized // sequence for each client. type SyncedSeqInfo struct { ID types.ID `bson:"_id"` + ProjectID types.ID `bson:"project_id"` DocID types.ID `bson:"doc_id"` ClientID types.ID `bson:"client_id"` Lamport int64 `bson:"lamport"` diff --git a/server/backend/database/testcases/testcases.go b/server/backend/database/testcases/testcases.go index 4ae70dabb..bddbfb9d7 100644 --- a/server/backend/database/testcases/testcases.go +++ b/server/backend/database/testcases/testcases.go @@ -57,14 +57,17 @@ func RunFindDocInfoTest( clientInfo, err := db.ActivateClient(ctx, projectID, t.Name()) assert.NoError(t, err) - _, err = db.FindDocInfoByID(context.Background(), projectID, dummyClientID) + _, err = db.FindDocInfoByRefKey(context.Background(), types.DocRefKey{ + ProjectID: projectID, + DocID: dummyClientID, + }) assert.ErrorIs(t, err, database.ErrDocumentNotFound) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - _, err = db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, false) + _, err = db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, false) assert.ErrorIs(t, err, database.ErrDocumentNotFound) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.Equal(t, docKey, docInfo.Key) }) @@ -100,7 +103,12 @@ func RunFindProjectInfoByNameTest( ctx := context.Background() suffixes := []int{0, 1, 2} for _, suffix := range suffixes { - _, err := db.CreateProjectInfo(ctx, fmt.Sprintf("%s-%d", t.Name(), suffix), dummyOwnerID, clientDeactivateThreshold) + _, err := db.CreateProjectInfo( + ctx, + fmt.Sprintf("%s-%d", t.Name(), suffix), + dummyOwnerID, + clientDeactivateThreshold, + ) assert.NoError(t, err) } @@ -158,7 +166,7 @@ func RunFindDocInfosByQueryTest( "test0", "test1", "test2", "test3", "test10", "test11", "test20", "test21", "test22", "test23"} for _, docKey := range docKeys { - _, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, key.Key(docKey), true) + _, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), key.Key(docKey), true) assert.NoError(t, err) } @@ -189,7 +197,8 @@ func RunFindChangesBetweenServerSeqsTest( docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) - docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) + docRefKey := docInfo.RefKey() assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) @@ -219,7 +228,7 @@ func RunFindChangesBetweenServerSeqsTest( // Find changes loadedChanges, err := db.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, 6, 10, ) @@ -237,7 +246,7 @@ func RunFindClosestSnapshotInfoTest(t *testing.T, db database.Database, projectI clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) bytesID, _ := clientInfo.ID.Bytes() actorID, _ := time.ActorIDFromBytes(bytesID) - docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) doc := document.New(key.Key(t.Name())) doc.SetActor(actorID) @@ -247,26 +256,28 @@ func RunFindClosestSnapshotInfoTest(t *testing.T, db database.Database, projectI return nil })) - assert.NoError(t, db.CreateSnapshotInfo(ctx, docInfo.ID, doc.InternalDocument())) - snapshot, err := db.FindClosestSnapshotInfo(ctx, docInfo.ID, change.MaxCheckpoint.ServerSeq, true) + docRefKey := docInfo.RefKey() + + assert.NoError(t, db.CreateSnapshotInfo(ctx, docRefKey, doc.InternalDocument())) + snapshot, err := db.FindClosestSnapshotInfo(ctx, docRefKey, change.MaxCheckpoint.ServerSeq, true) assert.NoError(t, err) assert.Equal(t, int64(0), snapshot.ServerSeq) pack := change.NewPack(doc.Key(), doc.Checkpoint().NextServerSeq(1), nil, nil) assert.NoError(t, doc.ApplyChangePack(pack)) - assert.NoError(t, db.CreateSnapshotInfo(ctx, docInfo.ID, doc.InternalDocument())) - snapshot, err = db.FindClosestSnapshotInfo(ctx, docInfo.ID, change.MaxCheckpoint.ServerSeq, true) + assert.NoError(t, db.CreateSnapshotInfo(ctx, docRefKey, doc.InternalDocument())) + snapshot, err = db.FindClosestSnapshotInfo(ctx, docRefKey, change.MaxCheckpoint.ServerSeq, true) assert.NoError(t, err) assert.Equal(t, int64(1), snapshot.ServerSeq) pack = change.NewPack(doc.Key(), doc.Checkpoint().NextServerSeq(2), nil, nil) assert.NoError(t, doc.ApplyChangePack(pack)) - assert.NoError(t, db.CreateSnapshotInfo(ctx, docInfo.ID, doc.InternalDocument())) - snapshot, err = db.FindClosestSnapshotInfo(ctx, docInfo.ID, change.MaxCheckpoint.ServerSeq, true) + assert.NoError(t, db.CreateSnapshotInfo(ctx, docRefKey, doc.InternalDocument())) + snapshot, err = db.FindClosestSnapshotInfo(ctx, docRefKey, change.MaxCheckpoint.ServerSeq, true) assert.NoError(t, err) assert.Equal(t, int64(2), snapshot.ServerSeq) - snapshot, err = db.FindClosestSnapshotInfo(ctx, docInfo.ID, 1, true) + snapshot, err = db.FindClosestSnapshotInfo(ctx, docRefKey, 1, true) assert.NoError(t, err) assert.Equal(t, int64(1), snapshot.ServerSeq) }) @@ -311,17 +322,38 @@ func RunFindUserInfoByIDTest(t *testing.T, db database.Database) { }) } +// RunFindUserInfoByNameTest runs the FindUserInfoByName test for the given db. +func RunFindUserInfoByNameTest(t *testing.T, db database.Database) { + t.Run("RunFindUserInfoByName test", func(t *testing.T) { + ctx := context.Background() + + username := "findUserInfoTestAccount" + password := "temporary-password" + + user, _, err := db.EnsureDefaultUserAndProject(ctx, username, password, clientDeactivateThreshold) + assert.NoError(t, err) + + info1, err := db.FindUserInfoByName(ctx, user.Username) + assert.NoError(t, err) + + assert.Equal(t, user.ID, info1.ID) + }) +} + // RunActivateClientDeactivateClientTest runs the ActivateClient and DeactivateClient tests for the given db. func RunActivateClientDeactivateClientTest(t *testing.T, db database.Database, projectID types.ID) { t.Run("activate and find client test", func(t *testing.T) { ctx := context.Background() - _, err := db.FindClientInfoByID(ctx, projectID, dummyOwnerID) + _, err := db.FindClientInfoByRefKey(ctx, types.ClientRefKey{ + ProjectID: projectID, + ClientID: dummyClientID, + }) assert.ErrorIs(t, err, database.ErrClientNotFound) clientInfo, err := db.ActivateClient(ctx, projectID, t.Name()) assert.NoError(t, err) - found, err := db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + found, err := db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.NoError(t, err) assert.Equal(t, clientInfo.Key, found.Key) }) @@ -330,7 +362,10 @@ func RunActivateClientDeactivateClientTest(t *testing.T, db database.Database, p ctx := context.Background() // try to deactivate the client with not exists ID. - _, err := db.DeactivateClient(ctx, projectID, dummyOwnerID) + _, err := db.DeactivateClient(ctx, types.ClientRefKey{ + ProjectID: projectID, + ClientID: dummyClientID, + }) assert.ErrorIs(t, err, database.ErrClientNotFound) clientInfo, err := db.ActivateClient(ctx, projectID, t.Name()) @@ -345,15 +380,13 @@ func RunActivateClientDeactivateClientTest(t *testing.T, db database.Database, p assert.Equal(t, t.Name(), clientInfo.Key) assert.Equal(t, database.ClientActivated, clientInfo.Status) - clientID := clientInfo.ID - - clientInfo, err = db.DeactivateClient(ctx, projectID, clientID) + clientInfo, err = db.DeactivateClient(ctx, clientInfo.RefKey()) assert.NoError(t, err) assert.Equal(t, t.Name(), clientInfo.Key) assert.Equal(t, database.ClientDeactivated, clientInfo.Status) // try to deactivate the client twice. - clientInfo, err = db.DeactivateClient(ctx, projectID, clientID) + clientInfo, err = db.DeactivateClient(ctx, clientInfo.RefKey()) assert.NoError(t, err) assert.Equal(t, t.Name(), clientInfo.Key) assert.Equal(t, database.ClientDeactivated, clientInfo.Status) @@ -463,26 +496,27 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t t.Run("simple FindDocInfosByPaging test", func(t *testing.T) { ctx := context.Background() - assertKeys := func(expectedKeys []key.Key, infos []*database.DocInfo) { - var keys []key.Key - for _, info := range infos { - keys = append(keys, info.Key) - } - assert.EqualValues(t, expectedKeys, keys) - } - pageSize := 5 totalSize := 9 clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) + docInfos := make([]*database.DocInfo, 0, totalSize) for i := 0; i < totalSize; i++ { - _, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, key.Key(fmt.Sprintf("%d", i)), true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), key.Key(fmt.Sprintf("%d", i)), true) assert.NoError(t, err) + docInfos = append(docInfos, docInfo) + } + + docKeys := make([]key.Key, 0, totalSize) + docKeysInReverse := make([]key.Key, 0, totalSize) + for _, docInfo := range docInfos { + docKeys = append(docKeys, docInfo.Key) + docKeysInReverse = append([]key.Key{docInfo.Key}, docKeysInReverse...) } // initial page, offset is empty infos, err := db.FindDocInfosByPaging(ctx, projectID, types.Paging[types.ID]{PageSize: pageSize}) assert.NoError(t, err) - assertKeys([]key.Key{"8", "7", "6", "5", "4"}, infos) + AssertKeys(t, docKeysInReverse[:pageSize], infos) // backward infos, err = db.FindDocInfosByPaging(ctx, projectID, types.Paging[types.ID]{ @@ -490,7 +524,7 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t PageSize: pageSize, }) assert.NoError(t, err) - assertKeys([]key.Key{"3", "2", "1", "0"}, infos) + AssertKeys(t, docKeysInReverse[pageSize:], infos) // backward again emptyInfos, err := db.FindDocInfosByPaging(ctx, projectID, types.Paging[types.ID]{ @@ -498,7 +532,7 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t PageSize: pageSize, }) assert.NoError(t, err) - assertKeys(nil, emptyInfos) + AssertKeys(t, nil, emptyInfos) // forward infos, err = db.FindDocInfosByPaging(ctx, projectID, types.Paging[types.ID]{ @@ -507,7 +541,7 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t IsForward: true, }) assert.NoError(t, err) - assertKeys([]key.Key{"4", "5", "6", "7", "8"}, infos) + AssertKeys(t, docKeys[totalSize-pageSize:], infos) // forward again emptyInfos, err = db.FindDocInfosByPaging(ctx, projectID, types.Paging[types.ID]{ @@ -516,7 +550,7 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t IsForward: true, }) assert.NoError(t, err) - assertKeys(nil, emptyInfos) + AssertKeys(t, nil, emptyInfos) }) t.Run("complex FindDocInfosByPaging test", func(t *testing.T) { @@ -530,15 +564,18 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t // dummy document setup var dummyDocInfos []*database.DocInfo for i := 0; i <= testDocCnt; i++ { - testDocKey := key.Key("testdockey" + strconv.Itoa(i)) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, testProjectInfo.ID, dummyClientID, testDocKey, true) + testDocKey := key.Key(fmt.Sprintf("%s%02d", "testdockey", i)) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, types.ClientRefKey{ + ProjectID: testProjectInfo.ID, + ClientID: dummyClientID, + }, testDocKey, true) assert.NoError(t, err) dummyDocInfos = append(dummyDocInfos, docInfo) } cases := []struct { name string - offset string + offset types.ID pageSize int isForward bool testResult []int @@ -573,28 +610,28 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t }, { name: "FindDocInfosByPaging --offset test", - offset: dummyDocInfos[13].ID.String(), + offset: dummyDocInfos[13].ID, pageSize: 0, isForward: false, testResult: helper.NewRangeSlice(12, 0), }, { name: "FindDocInfosByPaging --forward --offset test", - offset: dummyDocInfos[13].ID.String(), + offset: dummyDocInfos[13].ID, pageSize: 0, isForward: true, testResult: helper.NewRangeSlice(14, testDocCnt), }, { name: "FindDocInfosByPaging --size --offset test", - offset: dummyDocInfos[13].ID.String(), + offset: dummyDocInfos[13].ID, pageSize: 10, isForward: false, testResult: helper.NewRangeSlice(12, 3), }, { name: "FindDocInfosByPaging --size --forward --offset test", - offset: dummyDocInfos[13].ID.String(), + offset: dummyDocInfos[13].ID, pageSize: 10, isForward: true, testResult: helper.NewRangeSlice(14, 23), @@ -605,7 +642,7 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t t.Run(c.name, func(t *testing.T) { ctx := context.Background() testPaging := types.Paging[types.ID]{ - Offset: types.ID(c.offset), + Offset: c.offset, PageSize: c.pageSize, IsForward: c.isForward, } @@ -615,16 +652,16 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t for idx, docInfo := range docInfos { resultIdx := c.testResult[idx] - assert.Equal(t, docInfo.Key, dummyDocInfos[resultIdx].Key) - assert.Equal(t, docInfo.ID, dummyDocInfos[resultIdx].ID) - assert.Equal(t, docInfo.ProjectID, dummyDocInfos[resultIdx].ProjectID) + assert.Equal(t, dummyDocInfos[resultIdx].Key, docInfo.Key) + assert.Equal(t, dummyDocInfos[resultIdx].ID, docInfo.ID) + assert.Equal(t, dummyDocInfos[resultIdx].ProjectID, docInfo.ProjectID) } }) } }) t.Run("FindDocInfosByPaging with docInfoRemovedAt test", func(t *testing.T) { - const testDocCnt = 3 + const testDocCnt = 5 ctx := context.Background() // 01. Initialize a project and create documents. @@ -634,11 +671,19 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t var docInfos []*database.DocInfo for i := 0; i < testDocCnt; i++ { testDocKey := key.Key("key" + strconv.Itoa(i)) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectInfo.ID, dummyClientID, testDocKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, types.ClientRefKey{ + ProjectID: projectInfo.ID, + ClientID: dummyClientID, + }, testDocKey, true) assert.NoError(t, err) docInfos = append(docInfos, docInfo) } + docKeysInReverse := make([]key.Key, 0, testDocCnt) + for _, docInfo := range docInfos { + docKeysInReverse = append([]key.Key{docInfo.Key}, docKeysInReverse...) + } + // 02. List the documents. result, err := db.FindDocInfosByPaging(ctx, projectInfo.ID, types.Paging[types.ID]{ PageSize: 10, @@ -646,9 +691,12 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t }) assert.NoError(t, err) assert.Len(t, result, len(docInfos)) + AssertKeys(t, docKeysInReverse, result) - // 03. Remove a document. - err = db.CreateChangeInfos(ctx, projectInfo.ID, docInfos[0], 0, []*change.Change{}, true) + // 03. Remove some documents. + err = db.CreateChangeInfos(ctx, projectInfo.ID, docInfos[1], 0, []*change.Change{}, true) + assert.NoError(t, err) + err = db.CreateChangeInfos(ctx, projectInfo.ID, docInfos[3], 0, []*change.Change{}, true) assert.NoError(t, err) // 04. List the documents again and check the filtered result. @@ -657,7 +705,8 @@ func RunFindDocInfosByPagingTest(t *testing.T, db database.Database, projectID t IsForward: false, }) assert.NoError(t, err) - assert.Len(t, result, len(docInfos)-1) + assert.Len(t, result, len(docInfos)-2) + AssertKeys(t, []key.Key{docKeysInReverse[0], docKeysInReverse[2], docKeysInReverse[4]}, result) }) } @@ -669,14 +718,15 @@ func RunCreateChangeInfosTest(t *testing.T, db database.Database, projectID type // 01. Create a client and a document then attach the document to the client. clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) - docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) + docRefKey := docInfo.RefKey() assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) // 02. Remove the document and check the document is removed. err := db.CreateChangeInfos(ctx, projectID, docInfo, 0, []*change.Change{}, true) assert.NoError(t, err) - docInfo, err = db.FindDocInfoByID(ctx, projectID, docInfo.ID) + docInfo, err = db.FindDocInfoByRefKey(ctx, docRefKey) assert.NoError(t, err) assert.Equal(t, false, docInfo.RemovedAt.IsZero()) }) @@ -687,18 +737,20 @@ func RunCreateChangeInfosTest(t *testing.T, db database.Database, projectID type // 01. Create a client and a document then attach the document to the client. clientInfo1, _ := db.ActivateClient(ctx, projectID, t.Name()) - docInfo1, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo1.ID, docKey, true) - assert.NoError(t, clientInfo1.AttachDocument(docInfo1.ID)) + docInfo1, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo1.RefKey(), docKey, true) + docRefKey1 := docInfo1.RefKey() + assert.NoError(t, clientInfo1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo1, docInfo1)) // 02. Remove the document. - assert.NoError(t, clientInfo1.RemoveDocument(docInfo1.ID)) + assert.NoError(t, clientInfo1.RemoveDocument(docRefKey1.DocID)) err := db.CreateChangeInfos(ctx, projectID, docInfo1, 0, []*change.Change{}, true) assert.NoError(t, err) // 03. Create a document with same key and check they have same key but different id. - docInfo2, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo1.ID, docKey, true) - assert.NoError(t, clientInfo1.AttachDocument(docInfo2.ID)) + docInfo2, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo1.RefKey(), docKey, true) + docRefKey2 := docInfo2.RefKey() + assert.NoError(t, clientInfo1.AttachDocument(docRefKey2.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo1, docInfo2)) assert.Equal(t, docInfo1.Key, docInfo2.Key) assert.NotEqual(t, docInfo1.ID, docInfo2.ID) @@ -709,7 +761,8 @@ func RunCreateChangeInfosTest(t *testing.T, db database.Database, projectID type docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) - docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) + docRefKey := docInfo.RefKey() assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) @@ -722,12 +775,12 @@ func RunCreateChangeInfosTest(t *testing.T, db database.Database, projectID type assert.NoError(t, err) // Check whether removed_at is set in docInfo - docInfo, err = db.FindDocInfoByID(ctx, projectID, docInfo.ID) + docInfo, err = db.FindDocInfoByRefKey(ctx, docRefKey) assert.NoError(t, err) assert.NotEqual(t, gotime.Time{}, docInfo.RemovedAt) // Check whether DocumentRemoved status is set in clientInfo - clientInfo, err = db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + clientInfo, err = db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.NoError(t, err) assert.NotEqual(t, database.DocumentRemoved, clientInfo.Documents[docInfo.ID].Status) }) @@ -743,7 +796,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) err = db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo) @@ -757,13 +810,13 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err := db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err := db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(0)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(0)) @@ -775,7 +828,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) @@ -783,7 +836,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr clientInfo.Documents[docInfo.ID].ClientSeq = 1 assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err := db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err := db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(1)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(1)) @@ -794,7 +847,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr clientInfo.Documents[docInfo.ID].ClientSeq = 5 assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err = db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err = db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(3)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(5)) @@ -805,7 +858,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr clientInfo.Documents[docInfo.ID].ClientSeq = 3 assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err = db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err = db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(3)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(5)) @@ -817,7 +870,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) @@ -825,7 +878,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr clientInfo.Documents[docInfo.ID].ClientSeq = 1 assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err := db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err := db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(1)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(1)) @@ -834,7 +887,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, clientInfo.DetachDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err = db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err = db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentDetached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(0)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(0)) @@ -846,7 +899,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) @@ -854,7 +907,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr clientInfo.Documents[docInfo.ID].ClientSeq = 1 assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err := db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err := db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentAttached) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(1)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(1)) @@ -863,7 +916,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, clientInfo.RemoveDocument(docInfo.ID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) - result, err = db.FindClientInfoByID(ctx, projectID, clientInfo.ID) + result, err = db.FindClientInfoByRefKey(ctx, clientInfo.RefKey()) assert.Equal(t, result.Documents[docInfo.ID].Status, database.DocumentRemoved) assert.Equal(t, result.Documents[docInfo.ID].ServerSeq, int64(0)) assert.Equal(t, result.Documents[docInfo.ID].ClientSeq, uint32(0)) @@ -875,7 +928,7 @@ func RunUpdateClientInfoAfterPushPullTest(t *testing.T, db database.Database, pr assert.NoError(t, err) docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) - docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true) + docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(t, err) assert.NoError(t, clientInfo.AttachDocument(docInfo.ID)) @@ -899,48 +952,49 @@ func RunIsDocumentAttachedTest(t *testing.T, db database.Database, projectID typ assert.NoError(t, err) c2, err := db.ActivateClient(ctx, projectID, t.Name()+"2") assert.NoError(t, err) - d1, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, c1.ID, helper.TestDocKey(t), true) + d1, err := db.FindDocInfoByKeyAndOwner(ctx, c1.RefKey(), helper.TestDocKey(t), true) assert.NoError(t, err) // 01. Check if document is attached without attaching - attached, err := db.IsDocumentAttached(ctx, projectID, d1.ID, "") + docRefKey1 := d1.RefKey() + attached, err := db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) // 02. Check if document is attached after attaching - assert.NoError(t, c1.AttachDocument(d1.ID)) + assert.NoError(t, c1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) // 03. Check if document is attached after detaching - assert.NoError(t, c1.DetachDocument(d1.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) // 04. Check if document is attached after two clients attaching - assert.NoError(t, c1.AttachDocument(d1.ID)) + assert.NoError(t, c1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - assert.NoError(t, c2.AttachDocument(d1.ID)) + assert.NoError(t, c2.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c2, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) // 05. Check if document is attached after a client detaching - assert.NoError(t, c1.DetachDocument(d1.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) // 06. Check if document is attached after another client detaching - assert.NoError(t, c2.DetachDocument(d1.ID)) + assert.NoError(t, c2.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c2, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) }) @@ -951,38 +1005,40 @@ func RunIsDocumentAttachedTest(t *testing.T, db database.Database, projectID typ // 00. Create a client and two documents c1, err := db.ActivateClient(ctx, projectID, t.Name()+"1") assert.NoError(t, err) - d1, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, c1.ID, helper.TestDocKey(t)+"1", true) + d1, err := db.FindDocInfoByKeyAndOwner(ctx, c1.RefKey(), helper.TestDocKey(t)+"1", true) assert.NoError(t, err) - d2, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, c1.ID, helper.TestDocKey(t)+"2", true) + d2, err := db.FindDocInfoByKeyAndOwner(ctx, c1.RefKey(), helper.TestDocKey(t)+"2", true) assert.NoError(t, err) // 01. Check if documents are attached after attaching - assert.NoError(t, c1.AttachDocument(d1.ID)) + docRefKey1 := d1.RefKey() + assert.NoError(t, c1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err := db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err := db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) - assert.NoError(t, c1.AttachDocument(d2.ID)) + docRefKey2 := d2.RefKey() + assert.NoError(t, c1.AttachDocument(docRefKey2.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d2)) - attached, err = db.IsDocumentAttached(ctx, projectID, d2.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey2, "") assert.NoError(t, err) assert.True(t, attached) // 02. Check if a document is attached after detaching another document - assert.NoError(t, c1.DetachDocument(d2.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey2.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d2)) - attached, err = db.IsDocumentAttached(ctx, projectID, d2.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey2, "") assert.NoError(t, err) assert.False(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) // 03. Check if a document is attached after detaching remaining document - assert.NoError(t, c1.DetachDocument(d1.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) }) @@ -995,72 +1051,73 @@ func RunIsDocumentAttachedTest(t *testing.T, db database.Database, projectID typ assert.NoError(t, err) c2, err := db.ActivateClient(ctx, projectID, t.Name()+"2") assert.NoError(t, err) - d1, err := db.FindDocInfoByKeyAndOwner(ctx, projectID, c1.ID, helper.TestDocKey(t), true) + d1, err := db.FindDocInfoByKeyAndOwner(ctx, c1.RefKey(), helper.TestDocKey(t), true) assert.NoError(t, err) // 01. Check if document is attached without attaching - attached, err := db.IsDocumentAttached(ctx, projectID, d1.ID, "") + docRefKey1 := d1.RefKey() + attached, err := db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) // 02. Check if document is attached after attaching - assert.NoError(t, c1.AttachDocument(d1.ID)) + assert.NoError(t, c1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c1.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c1.ID) assert.NoError(t, err) assert.False(t, attached) // 03. Check if document is attached after detaching - assert.NoError(t, c1.DetachDocument(d1.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c1.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c1.ID) assert.NoError(t, err) assert.False(t, attached) // 04. Check if document is attached after two clients attaching - assert.NoError(t, c1.AttachDocument(d1.ID)) + assert.NoError(t, c1.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - assert.NoError(t, c2.AttachDocument(d1.ID)) + assert.NoError(t, c2.AttachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c2, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c1.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c1.ID) assert.NoError(t, err) assert.True(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c2.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c2.ID) assert.NoError(t, err) assert.True(t, attached) // 05. Check if document is attached after a client detaching - assert.NoError(t, c1.DetachDocument(d1.ID)) + assert.NoError(t, c1.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c1, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.True(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c1.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c1.ID) assert.NoError(t, err) assert.True(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c2.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c2.ID) assert.NoError(t, err) assert.False(t, attached) // 06. Check if document is attached after another client detaching - assert.NoError(t, c2.DetachDocument(d1.ID)) + assert.NoError(t, c2.DetachDocument(docRefKey1.DocID)) assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, c2, d1)) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, "") + attached, err = db.IsDocumentAttached(ctx, docRefKey1, "") assert.NoError(t, err) assert.False(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c1.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c1.ID) assert.NoError(t, err) assert.False(t, attached) - attached, err = db.IsDocumentAttached(ctx, projectID, d1.ID, c2.ID) + attached, err = db.IsDocumentAttached(ctx, docRefKey1, c2.ID) assert.NoError(t, err) assert.False(t, attached) }) @@ -1148,3 +1205,12 @@ func RunFindDeactivateCandidatesPerProjectTest(t *testing.T, db database.Databas assert.Contains(t, idList, c2.ID) }) } + +// AssertKeys checks the equivalence between the provided expectedKeys and the keys in the given infos. +func AssertKeys(t *testing.T, expectedKeys []key.Key, infos []*database.DocInfo) { + var keys []key.Key + for _, info := range infos { + keys = append(keys, info.Key) + } + assert.EqualValues(t, expectedKeys, keys) +} diff --git a/server/backend/housekeeping/housekeeping.go b/server/backend/housekeeping/housekeeping.go index 0d8934012..65c26f5fd 100644 --- a/server/backend/housekeeping/housekeeping.go +++ b/server/backend/housekeeping/housekeeping.go @@ -163,8 +163,7 @@ func (h *Housekeeping) deactivateCandidates( if _, err := clients.Deactivate( ctx, h.database, - clientInfo.ProjectID, - clientInfo.ID, + clientInfo.RefKey(), ); err != nil { return database.DefaultProjectID, err } diff --git a/server/backend/sync/coordinator.go b/server/backend/sync/coordinator.go index 764a942d6..4fa3dd2a4 100644 --- a/server/backend/sync/coordinator.go +++ b/server/backend/sync/coordinator.go @@ -41,13 +41,13 @@ type Coordinator interface { Subscribe( ctx context.Context, subscriber *time.ActorID, - documentID types.ID, + documentRefKey types.DocRefKey, ) (*Subscription, []*time.ActorID, error) // Unsubscribe unsubscribes from the given documents. Unsubscribe( ctx context.Context, - documentID types.ID, + documentRefKey types.DocRefKey, sub *Subscription, ) error diff --git a/server/backend/sync/memory/coordinator.go b/server/backend/sync/memory/coordinator.go index dba6ed0e7..17b93790a 100644 --- a/server/backend/sync/memory/coordinator.go +++ b/server/backend/sync/memory/coordinator.go @@ -58,24 +58,24 @@ func (c *Coordinator) NewLocker( func (c *Coordinator) Subscribe( ctx context.Context, subscriber *time.ActorID, - documentID types.ID, + documentRefKey types.DocRefKey, ) (*sync.Subscription, []*time.ActorID, error) { - sub, err := c.pubSub.Subscribe(ctx, subscriber, documentID) + sub, err := c.pubSub.Subscribe(ctx, subscriber, documentRefKey) if err != nil { return nil, nil, err } - ids := c.pubSub.ClientIDs(documentID) + ids := c.pubSub.ClientIDs(documentRefKey) return sub, ids, nil } // Unsubscribe unsubscribes the given documents. func (c *Coordinator) Unsubscribe( ctx context.Context, - documentID types.ID, + documentRefKey types.DocRefKey, sub *sync.Subscription, ) error { - c.pubSub.Unsubscribe(ctx, documentID, sub) + c.pubSub.Unsubscribe(ctx, documentRefKey, sub) return nil } diff --git a/server/backend/sync/memory/coordinator_test.go b/server/backend/sync/memory/coordinator_test.go index 02e2e5ce4..9f0dcc7db 100644 --- a/server/backend/sync/memory/coordinator_test.go +++ b/server/backend/sync/memory/coordinator_test.go @@ -30,14 +30,17 @@ import ( func TestCoordinator(t *testing.T) { t.Run("subscriptions map test", func(t *testing.T) { coordinator := memory.NewCoordinator(nil) - docID := types.ID(t.Name() + "id") + docRefKey := types.DocRefKey{ + ProjectID: types.ID("000000000000000000000000"), + DocID: types.ID("000000000000000000000000"), + } ctx := context.Background() for i := 0; i < 5; i++ { id, err := time.ActorIDFromBytes([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, byte(i)}) assert.NoError(t, err) - _, clientIDs, err := coordinator.Subscribe(ctx, id, docID) + _, clientIDs, err := coordinator.Subscribe(ctx, id, docRefKey) assert.NoError(t, err) assert.Len(t, clientIDs, i+1) } diff --git a/server/backend/sync/memory/pubsub.go b/server/backend/sync/memory/pubsub.go index a01703784..2330036ca 100644 --- a/server/backend/sync/memory/pubsub.go +++ b/server/backend/sync/memory/pubsub.go @@ -65,15 +65,15 @@ func (s *subscriptions) Len() int { // PubSub is the memory implementation of PubSub, used for single server. type PubSub struct { - subscriptionsMapMu *gosync.RWMutex - subscriptionsMapByDocID map[types.ID]*subscriptions + subscriptionsMapMu *gosync.RWMutex + subscriptionsMapByDocRefKey map[types.DocRefKey]*subscriptions } // NewPubSub creates an instance of PubSub. func NewPubSub() *PubSub { return &PubSub{ - subscriptionsMapMu: &gosync.RWMutex{}, - subscriptionsMapByDocID: make(map[types.ID]*subscriptions), + subscriptionsMapMu: &gosync.RWMutex{}, + subscriptionsMapByDocRefKey: make(map[types.DocRefKey]*subscriptions), } } @@ -81,13 +81,13 @@ func NewPubSub() *PubSub { func (m *PubSub) Subscribe( ctx context.Context, subscriber *time.ActorID, - documentID types.ID, + documentRefKey types.DocRefKey, ) (*sync.Subscription, error) { if logging.Enabled(zap.DebugLevel) { logging.From(ctx).Debugf( `Subscribe(%s,%s) Start`, - documentID.String(), - subscriber.String(), + documentRefKey, + subscriber, ) } @@ -95,15 +95,15 @@ func (m *PubSub) Subscribe( defer m.subscriptionsMapMu.Unlock() sub := sync.NewSubscription(subscriber) - if _, ok := m.subscriptionsMapByDocID[documentID]; !ok { - m.subscriptionsMapByDocID[documentID] = newSubscriptions() + if _, ok := m.subscriptionsMapByDocRefKey[documentRefKey]; !ok { + m.subscriptionsMapByDocRefKey[documentRefKey] = newSubscriptions() } - m.subscriptionsMapByDocID[documentID].Add(sub) + m.subscriptionsMapByDocRefKey[documentRefKey].Add(sub) if logging.Enabled(zap.DebugLevel) { logging.From(ctx).Debugf( `Subscribe(%s,%s) End`, - documentID.String(), + documentRefKey, subscriber.String(), ) } @@ -113,7 +113,7 @@ func (m *PubSub) Subscribe( // Unsubscribe unsubscribes the given docKeys. func (m *PubSub) Unsubscribe( ctx context.Context, - documentID types.ID, + documentRefKey types.DocRefKey, sub *sync.Subscription, ) { m.subscriptionsMapMu.Lock() @@ -122,25 +122,25 @@ func (m *PubSub) Unsubscribe( if logging.Enabled(zap.DebugLevel) { logging.From(ctx).Debugf( `Unsubscribe(%s,%s) Start`, - documentID, + documentRefKey, sub.Subscriber().String(), ) } sub.Close() - if subs, ok := m.subscriptionsMapByDocID[documentID]; ok { + if subs, ok := m.subscriptionsMapByDocRefKey[documentRefKey]; ok { subs.Delete(sub.ID()) if subs.Len() == 0 { - delete(m.subscriptionsMapByDocID, documentID) + delete(m.subscriptionsMapByDocRefKey, documentRefKey) } } if logging.Enabled(zap.DebugLevel) { logging.From(ctx).Debugf( `Unsubscribe(%s,%s) End`, - documentID, + documentRefKey, sub.Subscriber().String(), ) } @@ -155,12 +155,14 @@ func (m *PubSub) Publish( m.subscriptionsMapMu.RLock() defer m.subscriptionsMapMu.RUnlock() - documentID := event.DocumentID + documentRefKey := event.DocumentRefKey if logging.Enabled(zap.DebugLevel) { - logging.From(ctx).Debugf(`Publish(%s,%s) Start`, documentID.String(), publisherID.String()) + logging.From(ctx).Debugf(`Publish(%s,%s) Start`, + documentRefKey, + publisherID.String()) } - if subs, ok := m.subscriptionsMapByDocID[documentID]; ok { + if subs, ok := m.subscriptionsMapByDocRefKey[documentRefKey]; ok { for _, sub := range subs.Map() { if sub.Subscriber().Compare(publisherID) == 0 { continue @@ -170,7 +172,7 @@ func (m *PubSub) Publish( logging.From(ctx).Debugf( `Publish %s(%s,%s) to %s`, event.Type, - documentID.String(), + documentRefKey, publisherID.String(), sub.Subscriber().String(), ) @@ -183,7 +185,7 @@ func (m *PubSub) Publish( case <-gotime.After(100 * gotime.Millisecond): logging.From(ctx).Warnf( `Publish(%s,%s) to %s timeout`, - documentID.String(), + documentRefKey, publisherID.String(), sub.Subscriber().String(), ) @@ -191,17 +193,19 @@ func (m *PubSub) Publish( } } if logging.Enabled(zap.DebugLevel) { - logging.From(ctx).Debugf(`Publish(%s,%s) End`, documentID.String(), publisherID.String()) + logging.From(ctx).Debugf(`Publish(%s,%s) End`, + documentRefKey, + publisherID.String()) } } // ClientIDs returns the clients of the given document. -func (m *PubSub) ClientIDs(documentID types.ID) []*time.ActorID { +func (m *PubSub) ClientIDs(documentRefKey types.DocRefKey) []*time.ActorID { m.subscriptionsMapMu.RLock() defer m.subscriptionsMapMu.RUnlock() var ids []*time.ActorID - for _, sub := range m.subscriptionsMapByDocID[documentID].Map() { + for _, sub := range m.subscriptionsMapByDocRefKey[documentRefKey].Map() { ids = append(ids, sub.Subscriber()) } return ids diff --git a/server/backend/sync/memory/pubsub_test.go b/server/backend/sync/memory/pubsub_test.go index f1152bfcc..ea375f854 100644 --- a/server/backend/sync/memory/pubsub_test.go +++ b/server/backend/sync/memory/pubsub_test.go @@ -37,19 +37,22 @@ func TestPubSub(t *testing.T) { t.Run("publish subscribe test", func(t *testing.T) { pubSub := memory.NewPubSub() - id := types.ID(t.Name() + "id") + refKey := types.DocRefKey{ + ProjectID: types.ID("000000000000000000000000"), + DocID: types.ID("000000000000000000000000"), + } docEvent := sync.DocEvent{ - Type: types.DocumentWatchedEvent, - Publisher: idB, - DocumentID: id, + Type: types.DocumentWatchedEvent, + Publisher: idB, + DocumentRefKey: refKey, } ctx := context.Background() // subscribe the documents by actorA - subA, err := pubSub.Subscribe(ctx, idA, id) + subA, err := pubSub.Subscribe(ctx, idA, refKey) assert.NoError(t, err) defer func() { - pubSub.Unsubscribe(ctx, id, subA) + pubSub.Unsubscribe(ctx, refKey, subA) }() var wg gosync.WaitGroup diff --git a/server/backend/sync/pubsub.go b/server/backend/sync/pubsub.go index c4551b176..cf9547f27 100644 --- a/server/backend/sync/pubsub.go +++ b/server/backend/sync/pubsub.go @@ -47,10 +47,10 @@ func (s *Subscription) ID() string { // DocEvent represents events that occur related to the document. type DocEvent struct { - Type types.DocEventType - Publisher *time.ActorID - DocumentID types.ID - Body types.DocEventBody + Type types.DocEventType + Publisher *time.ActorID + DocumentRefKey types.DocRefKey + Body types.DocEventBody } // Events returns the DocEvent channel of this subscription. diff --git a/server/clients/clients.go b/server/clients/clients.go index 1803c7f17..9260147a9 100644 --- a/server/clients/clients.go +++ b/server/clients/clients.go @@ -22,7 +22,6 @@ import ( "errors" "github.com/yorkie-team/yorkie/api/types" - "github.com/yorkie-team/yorkie/pkg/document/time" "github.com/yorkie-team/yorkie/server/backend/database" ) @@ -48,20 +47,15 @@ func Activate( func Deactivate( ctx context.Context, db database.Database, - projectID types.ID, - clientID types.ID, + refKey types.ClientRefKey, ) (*database.ClientInfo, error) { - clientInfo, err := db.FindClientInfoByID( - ctx, - projectID, - clientID, - ) + clientInfo, err := db.FindClientInfoByRefKey(ctx, refKey) if err != nil { return nil, err } - for id, clientDocInfo := range clientInfo.Documents { - isAttached, err := clientInfo.IsAttached(id) + for docID, clientDocInfo := range clientInfo.Documents { + isAttached, err := clientInfo.IsAttached(docID) if err != nil { return nil, err } @@ -69,7 +63,7 @@ func Deactivate( continue } - if err := clientInfo.DetachDocument(id); err != nil { + if err := clientInfo.DetachDocument(docID); err != nil { return nil, err } @@ -81,26 +75,24 @@ func Deactivate( if err := db.UpdateSyncedSeq( ctx, clientInfo, - id, + types.DocRefKey{ + ProjectID: refKey.ProjectID, + DocID: docID, + }, clientDocInfo.ServerSeq, ); err != nil { return nil, err } } - return db.DeactivateClient(ctx, projectID, clientID) + return db.DeactivateClient(ctx, refKey) } -// FindClientInfo finds the client with the given id. +// FindClientInfo finds the client with the given refKey. func FindClientInfo( ctx context.Context, db database.Database, - project *types.Project, - clientID *time.ActorID, + refKey types.ClientRefKey, ) (*database.ClientInfo, error) { - return db.FindClientInfoByID( - ctx, - project.ID, - types.IDFromActorID(clientID), - ) + return db.FindClientInfoByRefKey(ctx, refKey) } diff --git a/server/documents/documents.go b/server/documents/documents.go index 68e46a9bf..f27be066b 100644 --- a/server/documents/documents.go +++ b/server/documents/documents.go @@ -100,8 +100,10 @@ func GetDocumentSummary( // TODO(hackerwins): Split FindDocInfoByKeyAndOwner into upsert and find. docInfo, err := be.DB.FindDocInfoByKeyAndOwner( ctx, - project.ID, - types.IDFromActorID(time.InitialActorID), + types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(time.InitialActorID), + }, k, false, ) @@ -134,8 +136,10 @@ func GetDocumentByServerSeq( ) (*document.InternalDocument, error) { docInfo, err := be.DB.FindDocInfoByKeyAndOwner( ctx, - project.ID, - types.IDFromActorID(time.InitialActorID), + types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(time.InitialActorID), + }, k, false, ) @@ -195,14 +199,13 @@ func FindDocInfoByKey( ) } -// FindDocInfo returns a document for the given document ID. -func FindDocInfo( +// FindDocInfoByRefKey returns a document for the given document refKey. +func FindDocInfoByRefKey( ctx context.Context, be *backend.Backend, - project *types.Project, - docID types.ID, + refkey types.DocRefKey, ) (*database.DocInfo, error) { - return be.DB.FindDocInfoByID(ctx, project.ID, docID) + return be.DB.FindDocInfoByRefKey(ctx, refkey) } // FindDocInfoByKeyAndOwner returns a document for the given document key. If @@ -210,15 +213,13 @@ func FindDocInfo( func FindDocInfoByKeyAndOwner( ctx context.Context, be *backend.Backend, - project *types.Project, clientInfo *database.ClientInfo, docKey key.Key, createDocIfNotExist bool, ) (*database.DocInfo, error) { return be.DB.FindDocInfoByKeyAndOwner( ctx, - project.ID, - clientInfo.ID, + clientInfo.RefKey(), docKey, createDocIfNotExist, ) @@ -229,15 +230,14 @@ func FindDocInfoByKeyAndOwner( func RemoveDocument( ctx context.Context, be *backend.Backend, - project *types.Project, - docID types.ID, + refKey types.DocRefKey, force bool, ) error { if force { - return be.DB.UpdateDocInfoStatusToRemoved(ctx, project.ID, docID) + return be.DB.UpdateDocInfoStatusToRemoved(ctx, refKey) } - isAttached, err := be.DB.IsDocumentAttached(ctx, project.ID, docID, "") + isAttached, err := be.DB.IsDocumentAttached(ctx, refKey, "") if err != nil { return err } @@ -245,16 +245,15 @@ func RemoveDocument( return ErrDocumentAttached } - return be.DB.UpdateDocInfoStatusToRemoved(ctx, project.ID, docID) + return be.DB.UpdateDocInfoStatusToRemoved(ctx, refKey) } // IsDocumentAttached returns true if the given document is attached to any client. func IsDocumentAttached( ctx context.Context, be *backend.Backend, - project *types.Project, - docID types.ID, + docRefKey types.DocRefKey, excludeClientID types.ID, ) (bool, error) { - return be.DB.IsDocumentAttached(ctx, project.ID, docID, excludeClientID) + return be.DB.IsDocumentAttached(ctx, docRefKey, excludeClientID) } diff --git a/server/packs/history.go b/server/packs/history.go index 84b0a402e..8d10f9bc2 100644 --- a/server/packs/history.go +++ b/server/packs/history.go @@ -32,14 +32,16 @@ func FindChanges( from int64, to int64, ) ([]*change.Change, error) { + docRefKey := docInfo.RefKey() if be.Config.SnapshotWithPurgingChanges { - minSyncedSeqInfo, err := be.DB.FindMinSyncedSeqInfo(ctx, docInfo.ID) + minSyncedSeqInfo, err := be.DB.FindMinSyncedSeqInfo(ctx, docRefKey) if err != nil { return nil, err } snapshotInfo, err := be.DB.FindClosestSnapshotInfo( - ctx, docInfo.ID, + ctx, + docRefKey, minSyncedSeqInfo.ServerSeq+be.Config.SnapshotInterval, false, ) @@ -54,7 +56,7 @@ func FindChanges( changes, err := be.DB.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, from, to, ) diff --git a/server/packs/packs.go b/server/packs/packs.go index 84099c17d..6015dbf68 100644 --- a/server/packs/packs.go +++ b/server/packs/packs.go @@ -84,7 +84,8 @@ func PushPull( be.Metrics.AddPushPullSentOperations(respPack.OperationsLen()) be.Metrics.AddPushPullSnapshotBytes(respPack.SnapshotLen()) - if err := clientInfo.UpdateCheckpoint(docInfo.ID, respPack.Checkpoint); err != nil { + docRefKey := docInfo.RefKey() + if err := clientInfo.UpdateCheckpoint(docRefKey.DocID, respPack.Checkpoint); err != nil { return nil, err } @@ -112,7 +113,7 @@ func PushPull( minSyncedTicket, err := be.DB.UpdateAndFindMinSyncedTicket( ctx, clientInfo, - docInfo.ID, + docRefKey, reqPack.Checkpoint.ServerSeq, ) if err != nil { @@ -147,9 +148,9 @@ func PushPull( ctx, publisherID, sync.DocEvent{ - Type: types.DocumentChangedEvent, - Publisher: publisherID, - DocumentID: docInfo.ID, + Type: types.DocumentChangedEvent, + Publisher: publisherID, + DocumentRefKey: docRefKey, }, ) @@ -196,7 +197,13 @@ func BuildDocumentForServerSeq( docInfo *database.DocInfo, serverSeq int64, ) (*document.InternalDocument, error) { - snapshotInfo, err := be.DB.FindClosestSnapshotInfo(ctx, docInfo.ID, serverSeq, true) + docRefKey := docInfo.RefKey() + snapshotInfo, err := be.DB.FindClosestSnapshotInfo( + ctx, + docRefKey, + serverSeq, + true, + ) if err != nil { return nil, err } @@ -216,7 +223,7 @@ func BuildDocumentForServerSeq( // certain size (e.g. 100) and read and gradually reflect it into the document. changes, err := be.DB.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, snapshotInfo.ServerSeq+1, serverSeq, ) diff --git a/server/packs/pushpull.go b/server/packs/pushpull.go index a8c979482..addea98a5 100644 --- a/server/packs/pushpull.go +++ b/server/packs/pushpull.go @@ -185,7 +185,7 @@ func pullChangeInfos( ) (change.Checkpoint, []*database.ChangeInfo, error) { pulledChanges, err := be.DB.FindChangeInfosBetweenServerSeqs( ctx, - docInfo.ID, + docInfo.RefKey(), reqPack.Checkpoint.ServerSeq+1, initialServerSeq, ) diff --git a/server/packs/snapshots.go b/server/packs/snapshots.go index ac56be161..39b0c915d 100644 --- a/server/packs/snapshots.go +++ b/server/packs/snapshots.go @@ -34,7 +34,12 @@ func storeSnapshot( minSyncedTicket *time.Ticket, ) error { // 01. get the closest snapshot's metadata of this docInfo - snapshotMetadata, err := be.DB.FindClosestSnapshotInfo(ctx, docInfo.ID, docInfo.ServerSeq, false) + docRefKey := docInfo.RefKey() + snapshotMetadata, err := be.DB.FindClosestSnapshotInfo( + ctx, + docRefKey, + docInfo.ServerSeq, + false) if err != nil { return err } @@ -48,7 +53,7 @@ func storeSnapshot( // 02. retrieve the changes between last snapshot and current docInfo changes, err := be.DB.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, snapshotMetadata.ServerSeq+1, docInfo.ServerSeq, ) @@ -59,7 +64,10 @@ func storeSnapshot( // 03. create document instance of the docInfo snapshotInfo := snapshotMetadata if snapshotMetadata.ID != "" { - snapshotInfo, err = be.DB.FindSnapshotInfoByID(ctx, snapshotInfo.ID) + snapshotInfo, err = be.DB.FindSnapshotInfoByRefKey( + ctx, + snapshotInfo.RefKey(), + ) if err != nil { return err } @@ -88,7 +96,11 @@ func storeSnapshot( } // 04. save the snapshot of the docInfo - if err := be.DB.CreateSnapshotInfo(ctx, docInfo.ID, doc); err != nil { + if err := be.DB.CreateSnapshotInfo( + ctx, + docRefKey, + doc, + ); err != nil { return err } @@ -96,7 +108,7 @@ func storeSnapshot( if be.Config.SnapshotWithPurgingChanges { if err := be.DB.PurgeStaleChanges( ctx, - docInfo.ID, + docRefKey, ); err != nil { logging.From(ctx).Error(err) } diff --git a/server/rpc/admin_server.go b/server/rpc/admin_server.go index 63f7fbe2b..dc57c66c9 100644 --- a/server/rpc/admin_server.go +++ b/server/rpc/admin_server.go @@ -327,7 +327,11 @@ func (s *adminServer) RemoveDocumentByAdmin( } }() - if err := documents.RemoveDocument(ctx, s.backend, project, docInfo.ID, req.Msg.Force); err != nil { + if err := documents.RemoveDocument( + ctx, s.backend, + docInfo.RefKey(), + req.Msg.Force, + ); err != nil { return nil, err } @@ -337,9 +341,9 @@ func (s *adminServer) RemoveDocumentByAdmin( ctx, publisherID, sync.DocEvent{ - Type: types.DocumentChangedEvent, - Publisher: publisherID, - DocumentID: docInfo.ID, + Type: types.DocumentChangedEvent, + Publisher: publisherID, + DocumentRefKey: docInfo.RefKey(), }, ) diff --git a/server/rpc/interceptors/admin_auth.go b/server/rpc/interceptors/admin_auth.go index f46d6cf39..39ee85119 100644 --- a/server/rpc/interceptors/admin_auth.go +++ b/server/rpc/interceptors/admin_auth.go @@ -165,7 +165,7 @@ func (i *AdminAuthInterceptor) authenticate( // NOTE(raararaara): If the token is access token, return the user of the token. claims, err := i.tokenManager.Verify(authorization) if err == nil { - user, err := users.GetUser(ctx, i.backend, claims.Username) + user, err := users.GetUserByName(ctx, i.backend, claims.Username) if err == nil { return user, nil } diff --git a/server/rpc/server_test.go b/server/rpc/server_test.go index c5382845b..745717ad8 100644 --- a/server/rpc/server_test.go +++ b/server/rpc/server_test.go @@ -18,7 +18,6 @@ package rpc_test import ( "context" - "encoding/hex" "fmt" "log" "net/http" @@ -27,7 +26,6 @@ import ( "connectrpc.com/connect" "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/types/known/wrapperspb" "github.com/yorkie-team/yorkie/admin" api "github.com/yorkie-team/yorkie/api/yorkie/v1" @@ -39,6 +37,7 @@ import ( "github.com/yorkie-team/yorkie/server/backend/housekeeping" "github.com/yorkie-team/yorkie/server/profiling/prometheus" "github.com/yorkie-team/yorkie/server/rpc" + "github.com/yorkie-team/yorkie/server/rpc/testcases" "github.com/yorkie-team/yorkie/test/helper" ) @@ -139,864 +138,73 @@ func TestMain(m *testing.M) { func TestSDKRPCServerBackend(t *testing.T) { t.Run("activate/deactivate client test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) - assert.NoError(t, err) - - // invalid argument - _, err = testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: ""})) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: emptyClientID})) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - // client not found - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: nilClientID})) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + testcases.RunActivateAndDeactivateClientTest(t, testClient) }) t.Run("attach/detach document test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - // try to attach with invalid client ID - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: invalidClientID, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - // try to attach with invalid client - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: nilClientID, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) - - // try to attach already attached document - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) - - // try to attach invalid change pack - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: invalidChangePack, - }, - )) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - // try to detach already detached document - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) - - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: invalidChangePack, - }, - )) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - // document not found - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: "000000000000000000000000", - ChangePack: &api.ChangePack{ - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - }, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) - - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) - assert.NoError(t, err) - - // try to attach the document with a deactivated client - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + testcases.RunAttachAndDetachDocumentTest(t, testClient) }) t.Run("attach/detach on removed document test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - packWithRemoveRequest := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - IsRemoved: true, - } - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.NoError(t, err) - - // try to detach document with same ID as removed document - // FailedPrecondition because document is not attached. - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) - - // try to create new document with same key as removed document - resPack, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.NoError(t, err) + testcases.RunAttachAndDetachRemovedDocumentTest(t, testClient) }) t.Run("push/pull changes test", func(t *testing.T) { - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: helper.TestDocKey(t).String()})) - assert.NoError(t, err) - - actorID, _ := hex.DecodeString(activateResp.Msg.ClientId) - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 1}, - Changes: []*api.Change{{ - Id: &api.ChangeID{ - ClientSeq: 1, - Lamport: 1, - ActorId: actorID, - }, - }}, - }, - }, - )) - assert.NoError(t, err) - - _, err = testClient.PushPullChanges( - context.Background(), - connect.NewRequest(&api.PushPullChangesRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 2}, - Changes: []*api.Change{{ - Id: &api.ChangeID{ - ClientSeq: 2, - Lamport: 2, - ActorId: actorID, - }, - }}, - }, - }, - )) - assert.NoError(t, err) - - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 3}, - Changes: []*api.Change{{ - Id: &api.ChangeID{ - ClientSeq: 3, - Lamport: 3, - ActorId: actorID, - }, - }}, - }, - }, - )) - assert.NoError(t, err) - - // try to push/pull with detached document - _, err = testClient.PushPullChanges( - context.Background(), - connect.NewRequest(&api.PushPullChangesRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) - - // try to push/pull with invalid pack - _, err = testClient.PushPullChanges( - context.Background(), - connect.NewRequest(&api.PushPullChangesRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: invalidChangePack, - }, - )) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) - - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) - assert.NoError(t, err) - - // try to push/pull with deactivated client - _, err = testClient.PushPullChanges( - context.Background(), - connect.NewRequest(&api.PushPullChangesRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + testcases.RunPushPullChangeTest(t, testClient) }) t.Run("push/pull on removed document test", func(t *testing.T) { - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - packWithRemoveRequest := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - IsRemoved: true, - } - - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: helper.TestDocKey(t).String()})) - assert.NoError(t, err) - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.NoError(t, err) - - // try to push/pull on removed document - _, err = testClient.PushPullChanges( - context.Background(), - connect.NewRequest(&api.PushPullChangesRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + testcases.RunPushPullChangeOnRemovedDocumentTest(t, testClient) }) t.Run("remove document test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - packWithRemoveRequest := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - IsRemoved: true, - } - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.NoError(t, err) - - // try to remove removed document - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + testcases.RunRemoveDocumentTest(t, testClient) }) t.Run("remove document with invalid client state test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - packWithRemoveRequest := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - IsRemoved: true, - } - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testClient.DetachDocument( - context.Background(), - connect.NewRequest(&api.DetachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - // try to remove detached document - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) - - _, err = testClient.DeactivateClient( - context.Background(), - connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) - assert.NoError(t, err) - - // try to remove document with a deactivated client - _, err = testClient.RemoveDocument( - context.Background(), - connect.NewRequest(&api.RemoveDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - ChangePack: packWithRemoveRequest, - }, - )) - assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + testcases.RunRemoveDocumentWithInvalidClientStateTest(t, testClient) }) t.Run("watch document test", func(t *testing.T) { - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: helper.TestDocKey(t).String(), - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - resPack, err := testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - // watch document - watchResp, err := testClient.WatchDocument( - context.Background(), - connect.NewRequest(&api.WatchDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - DocumentId: resPack.Msg.DocumentId, - }, - )) - assert.NoError(t, err) - - // check if stream is open - for watchResp.Receive() { - resp := watchResp.Msg() - assert.NotNil(t, resp) - break - } - - // TODO(krapie): find a way to set timeout for stream - //// wait for MaxConnectionAge + MaxConnectionAgeGrace - //time.Sleep(helper.RPCMaxConnectionAge + helper.RPCMaxConnectionAgeGrace) - // - //// check if stream has closed by server (EOF) - //_ = watchResp.Msg() - //assert.Equal(t, connect.CodeUnavailable, connect.CodeOf(err)) - //assert.Contains(t, err.Error(), "EOF") + testcases.RunWatchDocumentTest(t, testClient) }) } func TestAdminRPCServerBackend(t *testing.T) { t.Run("admin signup test", func(t *testing.T) { - adminUser := helper.TestSlugName(t) - adminPassword := helper.AdminPassword + "123!" - - _, err := testAdminClient.SignUp( - context.Background(), - connect.NewRequest(&api.SignUpRequest{ - Username: adminUser, - Password: adminPassword, - }, - )) - assert.NoError(t, err) - - // try to sign up with existing username - _, err = testAdminClient.SignUp( - context.Background(), - connect.NewRequest(&api.SignUpRequest{ - Username: adminUser, - Password: adminPassword, - }, - )) - assert.Equal(t, connect.CodeAlreadyExists, connect.CodeOf(err)) + testcases.RunAdminSignUpTest(t, testAdminClient) }) t.Run("admin login test", func(t *testing.T) { - _, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - // try to log in with invalid password - _, err = testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: invalidSlugName, - }, - )) - assert.Equal(t, connect.CodeUnauthenticated, connect.CodeOf(err)) + testcases.RunAdminLoginTest(t, testAdminClient) }) t.Run("admin create project test", func(t *testing.T) { - projectName := helper.TestSlugName(t) - - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - _, err = testAdminClient.CreateProject( - context.Background(), - connect.NewRequest(&api.CreateProjectRequest{ - Name: projectName, - }, - )) - assert.NoError(t, err) - - // try to create project with existing name - _, err = testAdminClient.CreateProject( - context.Background(), - connect.NewRequest(&api.CreateProjectRequest{ - Name: projectName, - }, - )) - assert.Equal(t, connect.CodeAlreadyExists, connect.CodeOf(err)) + testcases.RunAdminCreateProjectTest(t, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin list projects test", func(t *testing.T) { - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - _, err = testAdminClient.CreateProject( - context.Background(), - connect.NewRequest(&api.CreateProjectRequest{ - Name: helper.TestSlugName(t), - }, - )) - assert.NoError(t, err) - - _, err = testAdminClient.ListProjects( - context.Background(), - connect.NewRequest(&api.ListProjectsRequest{})) - assert.NoError(t, err) + testcases.RunAdminListProjectsTest(t, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin get project test", func(t *testing.T) { - projectName := helper.TestSlugName(t) - - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - _, err = testAdminClient.CreateProject( - context.Background(), - connect.NewRequest(&api.CreateProjectRequest{ - Name: projectName, - }, - )) - assert.NoError(t, err) - - _, err = testAdminClient.GetProject( - context.Background(), - connect.NewRequest(&api.GetProjectRequest{ - Name: projectName, - }, - )) - assert.NoError(t, err) - - // try to get project with non-existing name - _, err = testAdminClient.GetProject( - context.Background(), - connect.NewRequest(&api.GetProjectRequest{ - Name: invalidSlugName, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + testcases.RunAdminGetProjectTest(t, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin update project test", func(t *testing.T) { - projectName := helper.TestSlugName(t) - - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - createResp, err := testAdminClient.CreateProject( - context.Background(), - connect.NewRequest(&api.CreateProjectRequest{ - Name: projectName, - }, - )) - assert.NoError(t, err) - - _, err = testAdminClient.UpdateProject( - context.Background(), - connect.NewRequest(&api.UpdateProjectRequest{ - Id: createResp.Msg.Project.Id, - Fields: &api.UpdatableProjectFields{ - Name: &wrapperspb.StringValue{Value: "updated"}, - }, - }, - )) - assert.NoError(t, err) - - // try to update project with invalid field - _, err = testAdminClient.UpdateProject( - context.Background(), - connect.NewRequest(&api.UpdateProjectRequest{ - Id: projectName, - Fields: &api.UpdatableProjectFields{ - Name: &wrapperspb.StringValue{Value: invalidSlugName}, - }, - }, - )) - assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + testcases.RunAdminUpdateProjectTest(t, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin list documents test", func(t *testing.T) { - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - _, err = testAdminClient.ListDocuments( - context.Background(), - connect.NewRequest(&api.ListDocumentsRequest{ - ProjectName: defaultProjectName, - }, - )) - assert.NoError(t, err) - - // try to list documents with non-existing project name - _, err = testAdminClient.ListDocuments( - context.Background(), - connect.NewRequest(&api.ListDocumentsRequest{ - ProjectName: invalidSlugName, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + testcases.RunAdminListDocumentsTest(t, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin get document test", func(t *testing.T) { - testDocumentKey := helper.TestDocKey(t).String() - - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: testDocumentKey, - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testAdminClient.GetDocument( - context.Background(), - connect.NewRequest(&api.GetDocumentRequest{ - ProjectName: defaultProjectName, - DocumentKey: testDocumentKey, - }, - )) - assert.NoError(t, err) - - // try to get document with non-existing document name - _, err = testAdminClient.GetDocument( - context.Background(), - connect.NewRequest(&api.GetDocumentRequest{ - ProjectName: defaultProjectName, - DocumentKey: invalidChangePack.DocumentKey, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + testcases.RunAdminGetDocumentTest(t, testClient, testAdminClient, testAdminAuthInterceptor) }) t.Run("admin list changes test", func(t *testing.T) { - testDocumentKey := helper.TestDocKey(t).String() - - resp, err := testAdminClient.LogIn( - context.Background(), - connect.NewRequest(&api.LogInRequest{ - Username: helper.AdminUser, - Password: helper.AdminPassword, - }, - )) - assert.NoError(t, err) - - testAdminAuthInterceptor.SetToken(resp.Msg.Token) - - activateResp, err := testClient.ActivateClient( - context.Background(), - connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) - assert.NoError(t, err) - - packWithNoChanges := &api.ChangePack{ - DocumentKey: testDocumentKey, - Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, - } - - _, err = testClient.AttachDocument( - context.Background(), - connect.NewRequest(&api.AttachDocumentRequest{ - ClientId: activateResp.Msg.ClientId, - ChangePack: packWithNoChanges, - }, - )) - assert.NoError(t, err) - - _, err = testAdminClient.ListChanges( - context.Background(), - connect.NewRequest(&api.ListChangesRequest{ - ProjectName: defaultProjectName, - DocumentKey: testDocumentKey, - }, - )) - assert.NoError(t, err) - - // try to list changes with non-existing document name - _, err = testAdminClient.ListChanges( - context.Background(), - connect.NewRequest(&api.ListChangesRequest{ - ProjectName: defaultProjectName, - DocumentKey: invalidChangePack.DocumentKey, - }, - )) - assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + testcases.RunAdminListChangesTest(t, testClient, testAdminClient, testAdminAuthInterceptor) }) } diff --git a/server/rpc/testcases/testcases.go b/server/rpc/testcases/testcases.go new file mode 100644 index 000000000..c3712784b --- /dev/null +++ b/server/rpc/testcases/testcases.go @@ -0,0 +1,985 @@ +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package testcases contains testcases for server +package testcases + +import ( + "context" + "encoding/hex" + "testing" + + "connectrpc.com/connect" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/yorkie-team/yorkie/admin" + api "github.com/yorkie-team/yorkie/api/yorkie/v1" + "github.com/yorkie-team/yorkie/api/yorkie/v1/v1connect" + "github.com/yorkie-team/yorkie/test/helper" +) + +var ( + defaultProjectName = "default" + invalidSlugName = "@#$%^&*()_+" + + nilClientID = "000000000000000000000000" + emptyClientID = "" + invalidClientID = "invalid" + + invalidChangePack = &api.ChangePack{ + DocumentKey: "invalid", + Checkpoint: nil, + } +) + +// RunActivateAndDeactivateClientTest runs the ActivateClient and DeactivateClient test. +func RunActivateAndDeactivateClientTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) + assert.NoError(t, err) + + // invalid argument + _, err = testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: ""})) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: emptyClientID})) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + // client not found + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: nilClientID})) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) +} + +// RunAttachAndDetachDocumentTest runs the AttachDocument and DetachDocument test. +func RunAttachAndDetachDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + // try to attach with invalid client ID + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: invalidClientID, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + // try to attach with invalid client + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: nilClientID, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + + // try to attach already attached document + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + + // try to attach invalid change pack + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: invalidChangePack, + }, + )) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + // try to detach already detached document + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: invalidChangePack, + }, + )) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + // document not found + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: "000000000000000000000000", + ChangePack: &api.ChangePack{ + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + }, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) + + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) + assert.NoError(t, err) + + // try to attach the document with a deactivated client + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) +} + +// RunAttachAndDetachRemovedDocumentTest runs the AttachDocument and DetachDocument test on a removed document. +func RunAttachAndDetachRemovedDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + packWithRemoveRequest := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + IsRemoved: true, + } + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.NoError(t, err) + + // try to detach document with same ID as removed document + // FailedPrecondition because document is not attached. + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + + // try to create new document with same key as removed document + resPack, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.NoError(t, err) +} + +// RunPushPullChangeTest runs the PushChange and PullChange test. +func RunPushPullChangeTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: helper.TestDocKey(t).String()})) + assert.NoError(t, err) + + actorID, _ := hex.DecodeString(activateResp.Msg.ClientId) + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 1}, + Changes: []*api.Change{{ + Id: &api.ChangeID{ + ClientSeq: 1, + Lamport: 1, + ActorId: actorID, + }, + }}, + }, + }, + )) + assert.NoError(t, err) + + _, err = testClient.PushPullChanges( + context.Background(), + connect.NewRequest(&api.PushPullChangesRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 2}, + Changes: []*api.Change{{ + Id: &api.ChangeID{ + ClientSeq: 2, + Lamport: 2, + ActorId: actorID, + }, + }}, + }, + }, + )) + assert.NoError(t, err) + + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 3}, + Changes: []*api.Change{{ + Id: &api.ChangeID{ + ClientSeq: 3, + Lamport: 3, + ActorId: actorID, + }, + }}, + }, + }, + )) + assert.NoError(t, err) + + // try to push/pull with detached document + _, err = testClient.PushPullChanges( + context.Background(), + connect.NewRequest(&api.PushPullChangesRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + + // try to push/pull with invalid pack + _, err = testClient.PushPullChanges( + context.Background(), + connect.NewRequest(&api.PushPullChangesRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: invalidChangePack, + }, + )) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) + assert.NoError(t, err) + + // try to push/pull with deactivated client + _, err = testClient.PushPullChanges( + context.Background(), + connect.NewRequest(&api.PushPullChangesRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) +} + +// RunPushPullChangeOnRemovedDocumentTest runs the PushChange and PullChange test on a removed document. +func RunPushPullChangeOnRemovedDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + packWithRemoveRequest := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + IsRemoved: true, + } + + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: helper.TestDocKey(t).String()})) + assert.NoError(t, err) + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.NoError(t, err) + + // try to push/pull on removed document + _, err = testClient.PushPullChanges( + context.Background(), + connect.NewRequest(&api.PushPullChangesRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) +} + +// RunRemoveDocumentTest runs the RemoveDocument test. +func RunRemoveDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + packWithRemoveRequest := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + IsRemoved: true, + } + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.NoError(t, err) + + // try to remove removed document + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) +} + +// RunRemoveDocumentWithInvalidClientStateTest runs the RemoveDocument test with an invalid client state. +func RunRemoveDocumentWithInvalidClientStateTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + packWithRemoveRequest := &api.ChangePack{ + DocumentKey: helper.TestDocKey(t).String(), + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + IsRemoved: true, + } + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testClient.DetachDocument( + context.Background(), + connect.NewRequest(&api.DetachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + // try to remove detached document + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) + + _, err = testClient.DeactivateClient( + context.Background(), + connect.NewRequest(&api.DeactivateClientRequest{ClientId: activateResp.Msg.ClientId})) + assert.NoError(t, err) + + // try to remove document with a deactivated client + _, err = testClient.RemoveDocument( + context.Background(), + connect.NewRequest(&api.RemoveDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + ChangePack: packWithRemoveRequest, + }, + )) + assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err)) +} + +// RunWatchDocumentTest runs the WatchDocument test. +func RunWatchDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, +) { + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + docKey := helper.TestDocKey(t).String() + + packWithNoChanges := &api.ChangePack{ + DocumentKey: docKey, + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + resPack, err := testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + // watch document + watchResp, err := testClient.WatchDocument( + context.Background(), + connect.NewRequest(&api.WatchDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + DocumentId: resPack.Msg.DocumentId, + }, + )) + assert.NoError(t, err) + + // check if stream is open + for watchResp.Receive() { + resp := watchResp.Msg() + assert.NotNil(t, resp) + break + } + + // TODO(krapie): find a way to set timeout for stream + //// wait for MaxConnectionAge + MaxConnectionAgeGrace + //time.Sleep(helper.RPCMaxConnectionAge + helper.RPCMaxConnectionAgeGrace) + // + //// check if stream has closed by server (EOF) + //_ = watchResp.Msg() + //assert.Equal(t, connect.CodeUnavailable, connect.CodeOf(err)) + //assert.Contains(t, err.Error(), "EOF") +} + +// RunAdminSignUpTest runs the SignUp test in admin. +func RunAdminSignUpTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, +) { + adminUser := helper.TestSlugName(t) + adminPassword := helper.AdminPassword + "123!" + + _, err := testAdminClient.SignUp( + context.Background(), + connect.NewRequest(&api.SignUpRequest{ + Username: adminUser, + Password: adminPassword, + }, + )) + assert.NoError(t, err) + + // try to sign up with existing username + _, err = testAdminClient.SignUp( + context.Background(), + connect.NewRequest(&api.SignUpRequest{ + Username: adminUser, + Password: adminPassword, + }, + )) + assert.Equal(t, connect.CodeAlreadyExists, connect.CodeOf(err)) +} + +// RunAdminLoginTest runs the Admin Login test. +func RunAdminLoginTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, +) { + _, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + // try to log in with invalid password + _, err = testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: invalidSlugName, + }, + )) + assert.Equal(t, connect.CodeUnauthenticated, connect.CodeOf(err)) +} + +// RunAdminCreateProjectTest runs the CreateProject test in admin. +func RunAdminCreateProjectTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + projectName := helper.TestSlugName(t) + + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + _, err = testAdminClient.CreateProject( + context.Background(), + connect.NewRequest(&api.CreateProjectRequest{ + Name: projectName, + }, + )) + assert.NoError(t, err) + + // try to create project with existing name + _, err = testAdminClient.CreateProject( + context.Background(), + connect.NewRequest(&api.CreateProjectRequest{ + Name: projectName, + }, + )) + assert.Equal(t, connect.CodeAlreadyExists, connect.CodeOf(err)) +} + +// RunAdminListProjectsTest runs the ListProjects test in admin. +func RunAdminListProjectsTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + _, err = testAdminClient.CreateProject( + context.Background(), + connect.NewRequest(&api.CreateProjectRequest{ + Name: helper.TestSlugName(t), + }, + )) + assert.NoError(t, err) + + _, err = testAdminClient.ListProjects( + context.Background(), + connect.NewRequest(&api.ListProjectsRequest{})) + assert.NoError(t, err) +} + +// RunAdminGetProjectTest runs the GetProject test in admin. +func RunAdminGetProjectTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + projectName := helper.TestSlugName(t) + + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + _, err = testAdminClient.CreateProject( + context.Background(), + connect.NewRequest(&api.CreateProjectRequest{ + Name: projectName, + }, + )) + assert.NoError(t, err) + + _, err = testAdminClient.GetProject( + context.Background(), + connect.NewRequest(&api.GetProjectRequest{ + Name: projectName, + }, + )) + assert.NoError(t, err) + + // try to get project with non-existing name + _, err = testAdminClient.GetProject( + context.Background(), + connect.NewRequest(&api.GetProjectRequest{ + Name: invalidSlugName, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) +} + +// RunAdminUpdateProjectTest runs the UpdateProject test in admin. +func RunAdminUpdateProjectTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + projectName := helper.TestSlugName(t) + + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + createResp, err := testAdminClient.CreateProject( + context.Background(), + connect.NewRequest(&api.CreateProjectRequest{ + Name: projectName, + }, + )) + assert.NoError(t, err) + + _, err = testAdminClient.UpdateProject( + context.Background(), + connect.NewRequest(&api.UpdateProjectRequest{ + Id: createResp.Msg.Project.Id, + Fields: &api.UpdatableProjectFields{ + Name: &wrapperspb.StringValue{Value: "updated"}, + }, + }, + )) + assert.NoError(t, err) + + // try to update project with invalid field + _, err = testAdminClient.UpdateProject( + context.Background(), + connect.NewRequest(&api.UpdateProjectRequest{ + Id: projectName, + Fields: &api.UpdatableProjectFields{ + Name: &wrapperspb.StringValue{Value: invalidSlugName}, + }, + }, + )) + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) +} + +// RunAdminListDocumentsTest runs the ListDocuments test in admin. +func RunAdminListDocumentsTest( + t *testing.T, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + _, err = testAdminClient.ListDocuments( + context.Background(), + connect.NewRequest(&api.ListDocumentsRequest{ + ProjectName: defaultProjectName, + }, + )) + assert.NoError(t, err) + + // try to list documents with non-existing project name + _, err = testAdminClient.ListDocuments( + context.Background(), + connect.NewRequest(&api.ListDocumentsRequest{ + ProjectName: invalidSlugName, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) +} + +// RunAdminGetDocumentTest runs the GetDocument test in admin. +func RunAdminGetDocumentTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + testDocumentKey := helper.TestDocKey(t).String() + + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: testDocumentKey, + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testAdminClient.GetDocument( + context.Background(), + connect.NewRequest(&api.GetDocumentRequest{ + ProjectName: defaultProjectName, + DocumentKey: testDocumentKey, + }, + )) + assert.NoError(t, err) + + // try to get document with non-existing document name + _, err = testAdminClient.GetDocument( + context.Background(), + connect.NewRequest(&api.GetDocumentRequest{ + ProjectName: defaultProjectName, + DocumentKey: invalidChangePack.DocumentKey, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) +} + +// RunAdminListChangesTest runs the ListChanges test in admin. +func RunAdminListChangesTest( + t *testing.T, + testClient v1connect.YorkieServiceClient, + testAdminClient v1connect.AdminServiceClient, + testAdminAuthInterceptor *admin.AuthInterceptor, +) { + testDocumentKey := helper.TestDocKey(t).String() + + resp, err := testAdminClient.LogIn( + context.Background(), + connect.NewRequest(&api.LogInRequest{ + Username: helper.AdminUser, + Password: helper.AdminPassword, + }, + )) + assert.NoError(t, err) + + testAdminAuthInterceptor.SetToken(resp.Msg.Token) + + activateResp, err := testClient.ActivateClient( + context.Background(), + connect.NewRequest(&api.ActivateClientRequest{ClientKey: t.Name()})) + assert.NoError(t, err) + + packWithNoChanges := &api.ChangePack{ + DocumentKey: testDocumentKey, + Checkpoint: &api.Checkpoint{ServerSeq: 0, ClientSeq: 0}, + } + + _, err = testClient.AttachDocument( + context.Background(), + connect.NewRequest(&api.AttachDocumentRequest{ + ClientId: activateResp.Msg.ClientId, + ChangePack: packWithNoChanges, + }, + )) + assert.NoError(t, err) + + _, err = testAdminClient.ListChanges( + context.Background(), + connect.NewRequest(&api.ListChangesRequest{ + ProjectName: defaultProjectName, + DocumentKey: testDocumentKey, + }, + )) + assert.NoError(t, err) + + // try to list changes with non-existing document name + _, err = testAdminClient.ListChanges( + context.Background(), + connect.NewRequest(&api.ListChangesRequest{ + ProjectName: defaultProjectName, + DocumentKey: invalidChangePack.DocumentKey, + }, + )) + assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err)) +} diff --git a/server/rpc/yorkie_server.go b/server/rpc/yorkie_server.go index c5536efb8..989ca5401 100644 --- a/server/rpc/yorkie_server.go +++ b/server/rpc/yorkie_server.go @@ -93,7 +93,10 @@ func (s *yorkieServer) DeactivateClient( } project := projects.From(ctx) - _, err = clients.Deactivate(ctx, s.backend.DB, project.ID, types.IDFromActorID(actorID)) + _, err = clients.Deactivate(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(actorID), + }) if err != nil { return nil, err } @@ -141,11 +144,14 @@ func (s *yorkieServer) AttachDocument( } }() - clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, project, actorID) + clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(actorID), + }) if err != nil { return nil, err } - docInfo, err := documents.FindDocInfoByKeyAndOwner(ctx, s.backend, project, clientInfo, pack.DocumentKey, true) + docInfo, err := documents.FindDocInfoByKeyAndOwner(ctx, s.backend, clientInfo, pack.DocumentKey, true) if err != nil { return nil, err } @@ -184,8 +190,8 @@ func (s *yorkieServer) DetachDocument( if err != nil { return nil, err } - docID := types.ID(req.Msg.DocumentId) - if err := docID.Validate(); err != nil { + docID, err := converter.FromDocumentID(req.Msg.DocumentId) + if err != nil { return nil, err } @@ -211,16 +217,28 @@ func (s *yorkieServer) DetachDocument( } }() - clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, project, actorID) + clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(actorID), + }) if err != nil { return nil, err } - docInfo, err := documents.FindDocInfo(ctx, s.backend, project, docID) + + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } + docInfo, err := documents.FindDocInfoByRefKey(ctx, s.backend, docRefKey) if err != nil { return nil, err } - isAttached, err := documents.IsDocumentAttached(ctx, s.backend, project, docInfo.ID, clientInfo.ID) + isAttached, err := documents.IsDocumentAttached( + ctx, s.backend, + docRefKey, + clientInfo.ID, + ) if err != nil { return nil, err } @@ -266,8 +284,8 @@ func (s *yorkieServer) PushPullChanges( if err != nil { return nil, err } - docID := types.ID(req.Msg.DocumentId) - if err := docID.Validate(); err != nil { + docID, err := converter.FromDocumentID(req.Msg.DocumentId) + if err != nil { return nil, err } @@ -303,11 +321,19 @@ func (s *yorkieServer) PushPullChanges( syncMode = types.SyncModePushOnly } - clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, project, actorID) + clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(actorID), + }) if err != nil { return nil, err } - docInfo, err := documents.FindDocInfo(ctx, s.backend, project, docID) + + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } + docInfo, err := documents.FindDocInfoByRefKey(ctx, s.backend, docRefKey) if err != nil { return nil, err } @@ -342,16 +368,21 @@ func (s *yorkieServer) WatchDocument( if err != nil { return err } + + project := projects.From(ctx) docID, err := converter.FromDocumentID(req.Msg.DocumentId) if err != nil { return err } + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } - docInfo, err := documents.FindDocInfo( + docInfo, err := documents.FindDocInfoByRefKey( ctx, s.backend, - projects.From(ctx), - docID, + docRefKey, ) if err != nil { return nil @@ -364,8 +395,10 @@ func (s *yorkieServer) WatchDocument( return err } - project := projects.From(ctx) - if _, err = clients.FindClientInfo(ctx, s.backend.DB, project, clientID); err != nil { + if _, err = clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(clientID), + }); err != nil { return err } @@ -385,13 +418,13 @@ func (s *yorkieServer) WatchDocument( } }() - subscription, clientIDs, err := s.watchDoc(ctx, clientID, docID) + subscription, clientIDs, err := s.watchDoc(ctx, clientID, docRefKey) if err != nil { logging.From(ctx).Error(err) return err } defer func() { - s.unwatchDoc(subscription, docID) + s.unwatchDoc(subscription, docRefKey) }() var pbClientIDs []string @@ -452,8 +485,8 @@ func (s *yorkieServer) RemoveDocument( if err != nil { return nil, err } - docID := types.ID(req.Msg.DocumentId) - if err := docID.Validate(); err != nil { + docID, err := converter.FromDocumentID(req.Msg.DocumentId) + if err != nil { return nil, err } @@ -481,11 +514,19 @@ func (s *yorkieServer) RemoveDocument( }() } - clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, project, actorID) + clientInfo, err := clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(actorID), + }) if err != nil { return nil, err } - docInfo, err := documents.FindDocInfo(ctx, s.backend, project, docID) + + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } + docInfo, err := documents.FindDocInfoByRefKey(ctx, s.backend, docRefKey) if err != nil { return nil, err } @@ -512,9 +553,9 @@ func (s *yorkieServer) RemoveDocument( func (s *yorkieServer) watchDoc( ctx context.Context, clientID *time.ActorID, - documentID types.ID, + documentRefKey types.DocRefKey, ) (*sync.Subscription, []*time.ActorID, error) { - subscription, clientIDs, err := s.backend.Coordinator.Subscribe(ctx, clientID, documentID) + subscription, clientIDs, err := s.backend.Coordinator.Subscribe(ctx, clientID, documentRefKey) if err != nil { logging.From(ctx).Error(err) return nil, nil, err @@ -524,9 +565,9 @@ func (s *yorkieServer) watchDoc( ctx, subscription.Subscriber(), sync.DocEvent{ - Type: types.DocumentWatchedEvent, - Publisher: subscription.Subscriber(), - DocumentID: documentID, + Type: types.DocumentWatchedEvent, + Publisher: subscription.Subscriber(), + DocumentRefKey: documentRefKey, }, ) @@ -535,17 +576,17 @@ func (s *yorkieServer) watchDoc( func (s *yorkieServer) unwatchDoc( subscription *sync.Subscription, - documentID types.ID, + documentRefKey types.DocRefKey, ) { ctx := context.Background() - _ = s.backend.Coordinator.Unsubscribe(ctx, documentID, subscription) + _ = s.backend.Coordinator.Unsubscribe(ctx, documentRefKey, subscription) s.backend.Coordinator.Publish( ctx, subscription.Subscriber(), sync.DocEvent{ - Type: types.DocumentUnwatchedEvent, - Publisher: subscription.Subscriber(), - DocumentID: documentID, + Type: types.DocumentUnwatchedEvent, + Publisher: subscription.Subscriber(), + DocumentRefKey: documentRefKey, }, ) } @@ -559,16 +600,20 @@ func (s *yorkieServer) Broadcast( return nil, err } + project := projects.From(ctx) docID, err := converter.FromDocumentID(req.Msg.DocumentId) if err != nil { return nil, err } + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } - docInfo, err := documents.FindDocInfo( + docInfo, err := documents.FindDocInfoByRefKey( ctx, s.backend, - projects.From(ctx), - docID, + docRefKey, ) if err != nil { return nil, err @@ -582,8 +627,10 @@ func (s *yorkieServer) Broadcast( return nil, err } - project := projects.From(ctx) - if _, err = clients.FindClientInfo(ctx, s.backend.DB, project, clientID); err != nil { + if _, err = clients.FindClientInfo(ctx, s.backend.DB, types.ClientRefKey{ + ProjectID: project.ID, + ClientID: types.IDFromActorID(clientID), + }); err != nil { return nil, err } @@ -591,9 +638,9 @@ func (s *yorkieServer) Broadcast( ctx, clientID, sync.DocEvent{ - Type: types.DocumentBroadcastEvent, - Publisher: clientID, - DocumentID: docID, + Type: types.DocumentBroadcastEvent, + Publisher: clientID, + DocumentRefKey: docRefKey, Body: types.DocEventBody{ Topic: req.Msg.Topic, Payload: req.Msg.Payload, diff --git a/server/users/users.go b/server/users/users.go index 9f4019b43..e4adc6a66 100644 --- a/server/users/users.go +++ b/server/users/users.go @@ -53,7 +53,7 @@ func IsCorrectPassword( username, password string, ) (*types.User, error) { - info, err := be.DB.FindUserInfo(ctx, username) + info, err := be.DB.FindUserInfoByName(ctx, username) if err != nil { return nil, err } @@ -68,13 +68,13 @@ func IsCorrectPassword( return info.ToUser(), nil } -// GetUser returns a user by the given username. -func GetUser( +// GetUserByName returns a user by the given username. +func GetUserByName( ctx context.Context, be *backend.Backend, username string, ) (*types.User, error) { - info, err := be.DB.FindUserInfo(ctx, username) + info, err := be.DB.FindUserInfoByName(ctx, username) if err != nil { return nil, err } diff --git a/test/bench/push_pull_bench_test.go b/test/bench/push_pull_bench_test.go index 1dd6cc927..b6d981037 100644 --- a/test/bench/push_pull_bench_test.go +++ b/test/bench/push_pull_bench_test.go @@ -88,7 +88,7 @@ func setUpClientsAndDocs( for i := 0; i < n; i++ { clientInfo, err := be.DB.ActivateClient(ctx, database.DefaultProjectID, fmt.Sprintf("client-%d", i)) assert.NoError(b, err) - docInfo, err := be.DB.FindDocInfoByKeyAndOwner(ctx, database.DefaultProjectID, clientInfo.ID, docKey, true) + docInfo, err := be.DB.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) assert.NoError(b, err) assert.NoError(b, clientInfo.AttachDocument(docInfo.ID)) assert.NoError(b, be.DB.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) @@ -137,7 +137,10 @@ func benchmarkPushChanges( docKey := getDocKey(b, i) clientInfos, docID, docs := setUpClientsAndDocs(ctx, 1, docKey, b, be) pack := createChangePack(changeCnt, docs[0], b) - docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + docInfo, err := documents.FindDocInfoByRefKey(ctx, be, types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + }) assert.NoError(b, err) b.StartTimer() @@ -162,12 +165,16 @@ func benchmarkPullChanges( pushPack := createChangePack(changeCnt, pusherDoc, b) pullPack := createChangePack(0, pullerDoc, b) - docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } + docInfo, err := documents.FindDocInfoByRefKey(ctx, be, docRefKey) assert.NoError(b, err) _, err = packs.PushPull(ctx, be, project, pusherClientInfo, docInfo, pushPack, types.SyncModePushPull) assert.NoError(b, err) - docInfo, err = documents.FindDocInfo(ctx, be, project, docID) + docInfo, err = documents.FindDocInfoByRefKey(ctx, be, docRefKey) assert.NoError(b, err) b.StartTimer() @@ -188,12 +195,16 @@ func benchmarkPushSnapshots( ctx := context.Background() docKey := getDocKey(b, i) clientInfos, docID, docs := setUpClientsAndDocs(ctx, 1, docKey, b, be) + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } b.StartTimer() for j := 0; j < snapshotCnt; j++ { b.StopTimer() pushPack := createChangePack(changeCnt, docs[0], b) - docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + docInfo, err := documents.FindDocInfoByRefKey(ctx, be, docRefKey) assert.NoError(b, err) b.StartTimer() @@ -227,12 +238,16 @@ func benchmarkPullSnapshot( pushPack := createChangePack(changeCnt, pusherDoc, b) pullPack := createChangePack(0, pullerDoc, b) - docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + docRefKey := types.DocRefKey{ + ProjectID: project.ID, + DocID: docID, + } + docInfo, err := documents.FindDocInfoByRefKey(ctx, be, docRefKey) assert.NoError(b, err) _, err = packs.PushPull(ctx, be, project, pusherClientInfo, docInfo, pushPack, types.SyncModePushPull) assert.NoError(b, err) - docInfo, err = documents.FindDocInfo(ctx, be, project, docID) + docInfo, err = documents.FindDocInfoByRefKey(ctx, be, docRefKey) assert.NoError(b, err) b.StartTimer() diff --git a/test/helper/helper.go b/test/helper/helper.go index fc2f4d063..362270f06 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -26,8 +26,13 @@ import ( gotime "time" "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + gomongo "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" adminClient "github.com/yorkie-team/yorkie/admin" + "github.com/yorkie-team/yorkie/api/types" "github.com/yorkie-team/yorkie/internal/validation" "github.com/yorkie-team/yorkie/pkg/document" "github.com/yorkie-team/yorkie/pkg/document/change" @@ -39,8 +44,10 @@ import ( "github.com/yorkie-team/yorkie/pkg/index" "github.com/yorkie-team/yorkie/server" "github.com/yorkie-team/yorkie/server/backend" + "github.com/yorkie-team/yorkie/server/backend/database" "github.com/yorkie-team/yorkie/server/backend/database/mongo" "github.com/yorkie-team/yorkie/server/backend/housekeeping" + "github.com/yorkie-team/yorkie/server/logging" "github.com/yorkie-team/yorkie/server/profiling" "github.com/yorkie-team/yorkie/server/rpc" ) @@ -316,3 +323,134 @@ func NewRangeSlice(start, end int) []int { } return slice } + +// setupRawMongoClient returns the raw mongo client. +func setupRawMongoClient(databaseName string) (*gomongo.Client, error) { + conf := &mongo.Config{ + ConnectionTimeout: "5s", + ConnectionURI: "mongodb://localhost:27017", + YorkieDatabase: databaseName, + PingTimeout: "5s", + } + + ctx, cancel := context.WithTimeout(context.Background(), conf.ParseConnectionTimeout()) + defer cancel() + + client, err := gomongo.Connect( + ctx, + options.Client(). + ApplyURI(conf.ConnectionURI). + SetRegistry(mongo.NewRegistryBuilder().Build()), + ) + if err != nil { + return nil, fmt.Errorf("connect to mongo: %w", err) + } + + pingTimeout := conf.ParsePingTimeout() + ctxPing, cancel := context.WithTimeout(ctx, pingTimeout) + defer cancel() + + if err := client.Ping(ctxPing, readpref.Primary()); err != nil { + return nil, fmt.Errorf("ping mongo: %w", err) + } + + logging.DefaultLogger().Infof("MongoDB connected, URI: %s, DB: %s", conf.ConnectionURI, conf.YorkieDatabase) + + return client, nil +} + +// CleanUpAllCollections removes all data in every collection. +func CleanUpAllCollections(databaseName string) error { + cli, err := setupRawMongoClient(databaseName) + if err != nil { + return err + } + + for _, col := range mongo.Collections { + _, err := cli.Database(databaseName).Collection(col).DeleteMany(context.Background(), bson.D{}) + if err != nil { + return err + } + } + return nil +} + +// CreateDummyDocumentWithID creates a new dummy document with the given ID and key. +func CreateDummyDocumentWithID( + databaseName string, + projectID types.ID, + docID types.ID, + docKey key.Key, +) error { + cli, err := setupRawMongoClient(databaseName) + if err != nil { + return err + } + _, err = cli.Database(databaseName).Collection(mongo.ColDocuments).InsertOne( + context.Background(), + bson.M{ + "_id": docID, + "project_id": projectID, + "key": docKey, + }, + ) + if err != nil { + return err + } + + return nil +} + +// FindDocInfosWithID finds the docInfos of the given projectID and docID. +func FindDocInfosWithID( + databaseName string, + docID types.ID, +) ([]*database.DocInfo, error) { + ctx := context.Background() + cli, err := setupRawMongoClient(databaseName) + if err != nil { + return nil, err + } + + cursor, err := cli.Database(databaseName).Collection(mongo.ColDocuments).Find( + ctx, + bson.M{ + "_id": docID, + }, options.Find()) + if err != nil { + return nil, err + } + + var infos []*database.DocInfo + if err := cursor.All(ctx, &infos); err != nil { + return nil, err + } + + return infos, nil +} + +// CreateDummyClientWithID creates a new dummy document with the given ID and key. +func CreateDummyClientWithID( + databaseName string, + projectID types.ID, + clientKey string, + clientID types.ID, +) error { + cli, err := setupRawMongoClient(databaseName) + if err != nil { + return err + } + _, err = cli.Database(databaseName).Collection(mongo.ColClients).InsertOne( + context.Background(), + bson.M{ + "_id": clientID, + "project_id": projectID, + "key": clientKey, + }, + ) + if err != nil { + return err + } + + return nil +} diff --git a/test/integration/document_test.go b/test/integration/document_test.go index e69ae49a1..a163ab758 100644 --- a/test/integration/document_test.go +++ b/test/integration/document_test.go @@ -806,11 +806,11 @@ func TestDocumentWithProjects(t *testing.T) { assert.NoError(t, cli.Sync(ctx)) - docs, err := adminCli.ListDocuments(ctx, "default", "000000000000000000000000", 0, true, false) + docs, err := adminCli.ListDocuments(ctx, "default", "", 0, true, false) assert.NoError(t, err) assert.Equal(t, "", docs[0].Snapshot) - docs, err = adminCli.ListDocuments(ctx, "default", "000000000000000000000000", 0, true, true) + docs, err = adminCli.ListDocuments(ctx, "default", "", 0, true, true) assert.NoError(t, err) assert.NotEqual(t, 0, len(docs[0].Snapshot)) }) diff --git a/test/integration/housekeeping_test.go b/test/integration/housekeeping_test.go index 3e62c4cec..9b56d916d 100644 --- a/test/integration/housekeeping_test.go +++ b/test/integration/housekeeping_test.go @@ -1,7 +1,7 @@ //go:build integration && amd64 /* - * Copyright 2021 The Yorkie Authors. All rights reserved. + * Copyright 2024 The Yorkie Authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,8 @@ import ( ) const ( - dummyOwnerID = types.ID("000000000000000000000003") - otherOwnerID = types.ID("000000000000000000000004") + dummyOwnerID = types.ID("000000000000000000000000") + otherOwnerID = types.ID("000000000000000000000001") clientDeactivateThreshold = "23h" ) diff --git a/test/integration/retention_test.go b/test/integration/retention_test.go index 863005698..5864a9d3c 100644 --- a/test/integration/retention_test.go +++ b/test/integration/retention_test.go @@ -189,9 +189,10 @@ func TestRetention(t *testing.T) { ) assert.NoError(t, err) + docRefKey := docInfo.RefKey() changes, err := mongoCli.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, change.InitialServerSeq, change.MaxServerSeq, ) @@ -229,7 +230,7 @@ func TestRetention(t *testing.T) { changes, err = mongoCli.FindChangesBetweenServerSeqs( ctx, - docInfo.ID, + docRefKey, change.InitialServerSeq, change.MaxServerSeq, ) diff --git a/test/sharding/mongo_client_test.go b/test/sharding/mongo_client_test.go new file mode 100644 index 000000000..c041ec977 --- /dev/null +++ b/test/sharding/mongo_client_test.go @@ -0,0 +1,166 @@ +//go:build sharding + +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sharding + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/yorkie-team/yorkie/api/types" + "github.com/yorkie-team/yorkie/pkg/document/key" + "github.com/yorkie-team/yorkie/server/backend/database/mongo" + "github.com/yorkie-team/yorkie/server/backend/database/testcases" + "github.com/yorkie-team/yorkie/test/helper" +) + +const ( + shardedDBNameForMongoClient = "test-yorkie-meta-mongo-client" + dummyProjectID = types.ID("000000000000000000000000") + projectOneID = types.ID("000000000000000000000001") + projectTwoID = types.ID("000000000000000000000002") + dummyOwnerID = types.ID("000000000000000000000000") + dummyClientID = types.ID("000000000000000000000000") + clientDeactivateThreshold = "1h" +) + +func setupMongoClient(databaseName string) (*mongo.Client, error) { + config := &mongo.Config{ + ConnectionTimeout: "5s", + ConnectionURI: "mongodb://localhost:27017", + YorkieDatabase: databaseName, + PingTimeout: "5s", + } + if err := config.Validate(); err != nil { + return nil, err + } + + cli, err := mongo.Dial(config) + if err != nil { + return nil, err + } + + return cli, nil +} + +func TestClientWithShardedDB(t *testing.T) { + // Cleanup the previous data in DB + assert.NoError(t, helper.CleanUpAllCollections(shardedDBNameForMongoClient)) + + cli, err := setupMongoClient(shardedDBNameForMongoClient) + assert.NoError(t, err) + + t.Run("RunFindDocInfo test", func(t *testing.T) { + testcases.RunFindDocInfoTest(t, cli, dummyProjectID) + }) + + t.Run("RunFindDocInfosByQuery test", func(t *testing.T) { + t.Skip("TODO(hackerwins): the order of docInfos is different with memDB") + testcases.RunFindDocInfosByQueryTest(t, cli, projectOneID) + }) + + t.Run("RunFindChangesBetweenServerSeqs test", func(t *testing.T) { + testcases.RunFindChangesBetweenServerSeqsTest(t, cli, dummyProjectID) + }) + + t.Run("RunFindClosestSnapshotInfo test", func(t *testing.T) { + testcases.RunFindClosestSnapshotInfoTest(t, cli, dummyProjectID) + }) + + t.Run("ListUserInfos test", func(t *testing.T) { + t.Skip("TODO(hackerwins): time is returned as Local") + testcases.RunListUserInfosTest(t, cli) + }) + + t.Run("FindProjectInfoByName test", func(t *testing.T) { + testcases.RunFindProjectInfoByNameTest(t, cli) + }) + + t.Run("ActivateClientDeactivateClient test", func(t *testing.T) { + testcases.RunActivateClientDeactivateClientTest(t, cli, dummyProjectID) + }) + + t.Run("UpdateProjectInfo test", func(t *testing.T) { + testcases.RunUpdateProjectInfoTest(t, cli) + }) + + t.Run("FindDocInfosByPaging test", func(t *testing.T) { + testcases.RunFindDocInfosByPagingTest(t, cli, projectTwoID) + }) + + t.Run("CreateChangeInfo test", func(t *testing.T) { + testcases.RunCreateChangeInfosTest(t, cli, dummyProjectID) + }) + + t.Run("UpdateClientInfoAfterPushPull test", func(t *testing.T) { + testcases.RunUpdateClientInfoAfterPushPullTest(t, cli, dummyProjectID) + }) + + t.Run("IsDocumentAttached test", func(t *testing.T) { + testcases.RunIsDocumentAttachedTest(t, cli, dummyProjectID) + }) + + t.Run("FindDocInfoByRefKey with duplicate ID test", func(t *testing.T) { + ctx := context.Background() + + projectID1 := types.ID("000000000000000000000000") + projectID2 := types.ID("FFFFFFFFFFFFFFFFFFFFFFFF") + + docKey1 := key.Key(fmt.Sprintf("%s%d", t.Name(), 1)) + docKey2 := key.Key(fmt.Sprintf("%s%d", t.Name(), 2)) + + // 01. Initialize a project and create a document. + docInfo1, err := cli.FindDocInfoByKeyAndOwner(ctx, types.ClientRefKey{ + ProjectID: projectID1, + ClientID: dummyClientID, + }, docKey1, true) + assert.NoError(t, err) + + // 02. Create an extra document with duplicate ID. + err = helper.CreateDummyDocumentWithID( + shardedDBNameForMongoClient, + projectID2, + docInfo1.ID, + docKey2, + ) + assert.NoError(t, err) + + // 03. Check if there are two documents with the same ID. + infos, err := helper.FindDocInfosWithID( + shardedDBNameForMongoClient, + docInfo1.ID, + ) + assert.NoError(t, err) + assert.Len(t, infos, 2) + + // 04. Check if the document is correctly found using docKey and docID. + result, err := cli.FindDocInfoByRefKey( + ctx, + types.DocRefKey{ + ProjectID: projectID1, + DocID: docInfo1.ID, + }, + ) + assert.NoError(t, err) + assert.Equal(t, docInfo1.Key, result.Key) + assert.Equal(t, docInfo1.ID, result.ID) + }) +} diff --git a/test/sharding/server_test.go b/test/sharding/server_test.go new file mode 100644 index 000000000..6e906fb4a --- /dev/null +++ b/test/sharding/server_test.go @@ -0,0 +1,204 @@ +//go:build sharding + +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sharding + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "testing" + + "connectrpc.com/connect" + + "github.com/yorkie-team/yorkie/admin" + "github.com/yorkie-team/yorkie/api/yorkie/v1/v1connect" + "github.com/yorkie-team/yorkie/client" + "github.com/yorkie-team/yorkie/server/backend" + "github.com/yorkie-team/yorkie/server/backend/database" + "github.com/yorkie-team/yorkie/server/backend/database/mongo" + "github.com/yorkie-team/yorkie/server/backend/housekeeping" + "github.com/yorkie-team/yorkie/server/profiling/prometheus" + "github.com/yorkie-team/yorkie/server/rpc" + "github.com/yorkie-team/yorkie/server/rpc/testcases" + "github.com/yorkie-team/yorkie/test/helper" +) + +var ( + shardedDBNameForServer = "test-yorkie-meta-server" + testRPCServer *rpc.Server + testRPCAddr = fmt.Sprintf("localhost:%d", helper.RPCPort) + testClient v1connect.YorkieServiceClient + testAdminAuthInterceptor *admin.AuthInterceptor + testAdminClient v1connect.AdminServiceClient +) + +func TestMain(m *testing.M) { + // Cleanup the previous data in DB + err := helper.CleanUpAllCollections(shardedDBNameForServer) + if err != nil { + log.Fatal(err) + } + + met, err := prometheus.NewMetrics() + if err != nil { + log.Fatal(err) + } + + be, err := backend.New(&backend.Config{ + AdminUser: helper.AdminUser, + AdminPassword: helper.AdminPassword, + ClientDeactivateThreshold: helper.ClientDeactivateThreshold, + SnapshotThreshold: helper.SnapshotThreshold, + AuthWebhookCacheSize: helper.AuthWebhookSize, + ProjectInfoCacheSize: helper.ProjectInfoCacheSize, + ProjectInfoCacheTTL: helper.ProjectInfoCacheTTL.String(), + AdminTokenDuration: helper.AdminTokenDuration, + }, &mongo.Config{ + ConnectionURI: helper.MongoConnectionURI, + YorkieDatabase: shardedDBNameForServer, + ConnectionTimeout: helper.MongoConnectionTimeout, + PingTimeout: helper.MongoPingTimeout, + }, &housekeeping.Config{ + Interval: helper.HousekeepingInterval.String(), + CandidatesLimitPerProject: helper.HousekeepingCandidatesLimitPerProject, + ProjectFetchSize: helper.HousekeepingProjectFetchSize, + }, met) + if err != nil { + log.Fatal(err) + } + + project, err := be.DB.FindProjectInfoByID( + context.Background(), + database.DefaultProjectID, + ) + if err != nil { + log.Fatal(err) + } + + testRPCServer, err = rpc.NewServer(&rpc.Config{ + Port: helper.RPCPort, + }, be) + if err != nil { + log.Fatal(err) + } + + if err := testRPCServer.Start(); err != nil { + log.Fatalf("failed rpc listen: %s\n", err) + } + + authInterceptor := client.NewAuthInterceptor(project.PublicKey, "") + + conn := http.DefaultClient + testClient = v1connect.NewYorkieServiceClient( + conn, + "http://"+testRPCAddr, + connect.WithInterceptors(authInterceptor), + ) + + testAdminAuthInterceptor = admin.NewAuthInterceptor("") + + adminConn := http.DefaultClient + testAdminClient = v1connect.NewAdminServiceClient( + adminConn, + "http://"+testRPCAddr, + connect.WithInterceptors(testAdminAuthInterceptor), + ) + + code := m.Run() + + if err := be.Shutdown(); err != nil { + log.Fatal(err) + } + testRPCServer.Shutdown(true) + os.Exit(code) +} + +func TestSDKRPCServerBackendWithShardedDB(t *testing.T) { + t.Run("activate/deactivate client test", func(t *testing.T) { + testcases.RunActivateAndDeactivateClientTest(t, testClient) + }) + + t.Run("attach/detach document test", func(t *testing.T) { + testcases.RunAttachAndDetachDocumentTest(t, testClient) + }) + + t.Run("attach/detach on removed document test", func(t *testing.T) { + testcases.RunAttachAndDetachRemovedDocumentTest(t, testClient) + }) + + t.Run("push/pull changes test", func(t *testing.T) { + testcases.RunPushPullChangeTest(t, testClient) + }) + + t.Run("push/pull on removed document test", func(t *testing.T) { + testcases.RunPushPullChangeOnRemovedDocumentTest(t, testClient) + }) + + t.Run("remove document test", func(t *testing.T) { + testcases.RunRemoveDocumentTest(t, testClient) + }) + + t.Run("remove document with invalid client state test", func(t *testing.T) { + testcases.RunRemoveDocumentWithInvalidClientStateTest(t, testClient) + }) + + t.Run("watch document test", func(t *testing.T) { + testcases.RunWatchDocumentTest(t, testClient) + }) +} + +func TestAdminRPCServerBackendWithShardedDB(t *testing.T) { + t.Run("admin signup test", func(t *testing.T) { + testcases.RunAdminSignUpTest(t, testAdminClient) + }) + + t.Run("admin login test", func(t *testing.T) { + testcases.RunAdminLoginTest(t, testAdminClient) + }) + + t.Run("admin create project test", func(t *testing.T) { + testcases.RunAdminCreateProjectTest(t, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin list projects test", func(t *testing.T) { + testcases.RunAdminListProjectsTest(t, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin get project test", func(t *testing.T) { + testcases.RunAdminGetProjectTest(t, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin update project test", func(t *testing.T) { + testcases.RunAdminUpdateProjectTest(t, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin list documents test", func(t *testing.T) { + testcases.RunAdminListDocumentsTest(t, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin get document test", func(t *testing.T) { + testcases.RunAdminGetDocumentTest(t, testClient, testAdminClient, testAdminAuthInterceptor) + }) + + t.Run("admin list changes test", func(t *testing.T) { + testcases.RunAdminListChangesTest(t, testClient, testAdminClient, testAdminAuthInterceptor) + }) +}