From 5500afbe793c57e7aafbd7d663b3271553160bba Mon Sep 17 00:00:00 2001 From: Sejong Kim <46182768+sejongk@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:10:15 +0900 Subject: [PATCH] Add PushPull benchmark test (#658) --- .github/workflows/ci.yml | 29 ++- test/bench/push_pull_bench_test.go | 294 +++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 test/bench/push_pull_bench_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab8a5c8de..c00c8a632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,30 @@ jobs: - name: Test run: go test -tags integration -race -coverprofile=coverage.txt -covermode=atomic -v ./... + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.txt + + bench: + name: bench + runs-on: ubuntu-latest + steps: + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get tools dependencies + run: make tools + + - name: Stack + run: docker-compose -f build/docker/docker-compose.yml up --build -d + - name: Bench run: make bench @@ -66,8 +90,3 @@ jobs: fail-on-alert: false github-token: ${{ secrets.GITHUB_TOKEN }} comment-always: true - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.txt diff --git a/test/bench/push_pull_bench_test.go b/test/bench/push_pull_bench_test.go new file mode 100644 index 000000000..cab46ea8d --- /dev/null +++ b/test/bench/push_pull_bench_test.go @@ -0,0 +1,294 @@ +//go:build bench + +/* + * Copyright 2023 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bench + +import ( + "context" + "fmt" + "testing" + gotime "time" + + "github.com/stretchr/testify/assert" + "github.com/yorkie-team/yorkie/api/converter" + "github.com/yorkie-team/yorkie/api/types" + "github.com/yorkie-team/yorkie/pkg/document" + "github.com/yorkie-team/yorkie/pkg/document/change" + "github.com/yorkie-team/yorkie/pkg/document/json" + "github.com/yorkie-team/yorkie/pkg/document/key" + "github.com/yorkie-team/yorkie/pkg/document/presence" + "github.com/yorkie-team/yorkie/pkg/document/time" + "github.com/yorkie-team/yorkie/server/backend" + "github.com/yorkie-team/yorkie/server/backend/database" + "github.com/yorkie-team/yorkie/server/documents" + "github.com/yorkie-team/yorkie/server/packs" + "github.com/yorkie-team/yorkie/server/profiling/prometheus" + "github.com/yorkie-team/yorkie/test/helper" +) + +func getDocKey(b *testing.B, i int) key.Key { + return key.Key(fmt.Sprintf("tests$%s-%d-%d", b.Name(), i, gotime.Now().UnixMilli())) +} + +func setUpBackend( + b *testing.B, + snapshotInterval int64, + snapshotThreshold int64, +) *backend.Backend { + conf := helper.TestConfig() + conf.Backend.SnapshotInterval = snapshotInterval + conf.Backend.SnapshotThreshold = snapshotThreshold + + metrics, err := prometheus.NewMetrics() + assert.NoError(b, err) + + be, err := backend.New( + conf.Backend, + conf.Mongo, + conf.Housekeeping, + metrics, + ) + assert.NoError(b, err) + + return be +} + +func setUpDefaultProject(b *testing.B, be *backend.Backend) *types.Project { + projectInfo, err := be.DB.FindProjectInfoByID(context.Background(), database.DefaultProjectID) + assert.NoError(b, err) + return projectInfo.ToProject() +} + +func setUpClientsAndDocs( + ctx context.Context, + n int, + docKey key.Key, + b *testing.B, + be *backend.Backend, +) ([]*database.ClientInfo, types.ID, []*document.Document) { + var clientInfos []*database.ClientInfo + var docID types.ID + var docs []*document.Document + for i := 0; i < n; i++ { + clientInfo, err := be.DB.ActivateClient(ctx, database.DefaultProjectID, fmt.Sprintf("client-%d", i)) + assert.NoError(b, err) + docInfo, err := be.DB.FindDocInfoByKeyAndOwner(ctx, database.DefaultProjectID, clientInfo.ID, docKey, true) + assert.NoError(b, err) + assert.NoError(b, clientInfo.AttachDocument(docInfo.ID)) + assert.NoError(b, be.DB.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo)) + + bytesID, _ := clientInfo.ID.Bytes() + actorID, _ := time.ActorIDFromBytes(bytesID) + doc := document.New(docKey) + doc.SetActor(actorID) + assert.NoError(b, doc.Update(func(root *json.Object, _ *presence.Presence) error { + root.SetNewArray("array") + return nil + })) + + clientInfos = append(clientInfos, clientInfo) + if docID == "" { + docID = docInfo.ID + } + docs = append(docs, doc) + } + return clientInfos, docID, docs +} + +func createChangePack( + cnt int, + doc *document.Document, + b *testing.B, +) *change.Pack { + for idx := 0; idx < cnt; idx++ { + assert.NoError(b, doc.Update(func(root *json.Object, _ *presence.Presence) error { + root.GetArray("array").AddString("A") + return nil + })) + } + return doc.CreateChangePack() +} + +func benchmarkPushChanges( + changeCnt int, + b *testing.B, + be *backend.Backend, + project *types.Project, +) { + for i := 1; i < b.N; i++ { + b.StopTimer() + ctx := context.Background() + docKey := getDocKey(b, i) + clientInfos, docID, docs := setUpClientsAndDocs(ctx, 1, docKey, b, be) + pack := createChangePack(changeCnt, docs[0], b) + docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + b.StartTimer() + + _, err = packs.PushPull(ctx, be, project, clientInfos[0], docInfo, pack, types.SyncModePushPull) + assert.NoError(b, err) + } +} + +func benchmarkPullChanges( + changeCnt int, + b *testing.B, + be *backend.Backend, + project *types.Project, +) { + for i := 1; i < b.N; i++ { + b.StopTimer() + ctx := context.Background() + docKey := getDocKey(b, i) + clientInfos, docID, docs := setUpClientsAndDocs(ctx, 2, docKey, b, be) + pusherClientInfo, pullerClientInfo := clientInfos[0], clientInfos[1] + pusherDoc, pullerDoc := docs[0], docs[1] + pushPack := createChangePack(changeCnt, pusherDoc, b) + pullPack := createChangePack(0, pullerDoc, b) + + docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + _, err = packs.PushPull(ctx, be, project, pusherClientInfo, docInfo, pushPack, types.SyncModePushPull) + assert.NoError(b, err) + + docInfo, err = documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + b.StartTimer() + + _, err = packs.PushPull(ctx, be, project, pullerClientInfo, docInfo, pullPack, types.SyncModePushPull) + assert.NoError(b, err) + } +} + +func benchmarkPushSnapshots( + snapshotCnt int, + changeCnt int, + b *testing.B, + be *backend.Backend, + project *types.Project, +) { + for i := 1; i < b.N; i++ { + b.StopTimer() + ctx := context.Background() + docKey := getDocKey(b, i) + clientInfos, docID, docs := setUpClientsAndDocs(ctx, 1, docKey, b, be) + b.StartTimer() + + for j := 0; j < snapshotCnt; j++ { + b.StopTimer() + pushPack := createChangePack(changeCnt, docs[0], b) + docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + b.StartTimer() + + pulled, err := packs.PushPull(ctx, be, project, clientInfos[0], docInfo, pushPack, types.SyncModePushPull) + assert.NoError(b, err) + + b.StopTimer() + pbChangePack, err := pulled.ToPBChangePack() + assert.NoError(b, err) + pullPack, err := converter.FromChangePack(pbChangePack) + assert.NoError(b, err) + assert.NoError(b, docs[0].ApplyChangePack(pullPack)) + b.StartTimer() + } + } +} + +func benchmarkPullSnapshot( + changeCnt int, + b *testing.B, + be *backend.Backend, + project *types.Project, +) { + for i := 1; i < b.N; i++ { + b.StopTimer() + ctx := context.Background() + docKey := getDocKey(b, i) + clientInfos, docID, docs := setUpClientsAndDocs(ctx, 2, docKey, b, be) + pusherClientInfo, pullerClientInfo := clientInfos[0], clientInfos[1] + pusherDoc, pullerDoc := docs[0], docs[1] + pushPack := createChangePack(changeCnt, pusherDoc, b) + pullPack := createChangePack(0, pullerDoc, b) + + docInfo, err := documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + _, err = packs.PushPull(ctx, be, project, pusherClientInfo, docInfo, pushPack, types.SyncModePushPull) + assert.NoError(b, err) + + docInfo, err = documents.FindDocInfo(ctx, be, project, docID) + assert.NoError(b, err) + b.StartTimer() + + _, err = packs.PushPull(ctx, be, project, pullerClientInfo, docInfo, pullPack, types.SyncModePushPull) + assert.NoError(b, err) + } +} + +func BenchmarkChange(b *testing.B) { + // Disable to take a snapshot by making the interval and the threshold large. + be := setUpBackend(b, 100000, 100000) + project := setUpDefaultProject(b, be) + b.ResetTimer() + + b.Run("Push 10 Changes", func(b *testing.B) { + benchmarkPushChanges(10, b, be, project) + }) + + b.Run("Push 100 Changes", func(b *testing.B) { + benchmarkPushChanges(100, b, be, project) + }) + + b.Run("Push 1000 Changes", func(b *testing.B) { + benchmarkPushChanges(1000, b, be, project) + }) + + b.Run("Pull 10 Changes", func(b *testing.B) { + benchmarkPullChanges(10, b, be, project) + }) + + b.Run("Pull 100 Changes", func(b *testing.B) { + benchmarkPullChanges(100, b, be, project) + }) + + b.Run("Pull 1000 Changes", func(b *testing.B) { + benchmarkPullChanges(1000, b, be, project) + }) +} + +func BenchmarkSnapshot(b *testing.B) { + be := setUpBackend(b, 10, 10) + project := setUpDefaultProject(b, be) + b.ResetTimer() + + b.Run("Push 3KB snapshot", func(b *testing.B) { + benchmarkPushSnapshots(1, 100, b, be, project) + }) + + b.Run("Push 30KB snapshot", func(b *testing.B) { + benchmarkPushSnapshots(1, 1000, b, be, project) + }) + + b.Run("Pull 3KB snapshot", func(b *testing.B) { + benchmarkPullSnapshot(100, b, be, project) + }) + + b.Run("Pull 30KB snapshot", func(b *testing.B) { + benchmarkPullSnapshot(1000, b, be, project) + }) +}