Replace benchmark.js with vitest bench (#793)

.github/workflows/ci.yml

@@ -51,29 +51,4 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Run benchmark
-        run: npm run benchmark | tee output.txt
-      # Download previous benchmark result from cache (if exists)
-      - name: Download previous benchmark data
-        uses: actions/cache@v1
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-      # Run `github-action-benchmark` action
-      - name: Store benchmark result
-        uses: benchmark-action/github-action-benchmark@v1
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-        with:
-          # What benchmark tool the output.txt came from
-          tool: 'benchmarkjs'
-          # Where the output from the benchmark tool is stored
-          output-file-path: output.txt
-          # Where the previous data file is stored
-          external-data-json-path: ./cache/benchmark-data.json
-          # Workflow will fail when an alert happens
-          fail-on-alert: false
-          # GitHub API token to make a commit comment
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          # Enable comment always
-          comment-always: true
-          # Save only on the main to compare the benchmarks between PR and base branch
-          save-data-file: ${{ github.ref == 'refs/heads/main' }}
+        run: npm run test:bench

package.json

@@ -22,9 +22,9 @@
     "test:ci": "vitest run --coverage",
     "test:yorkie.dev": "TEST_RPC_ADDR=https://api.yorkie.dev vitest run --coverage",
     "test:watch": "vitest",
+    "test:bench": "vitest bench",
     "lint": "eslint . --fix --max-warnings=0 --ext .ts",
     "prepare": "ts-patch install && npm run build",
-    "benchmark": "ts-node-script ./test/bench",
     "publish:prepare": "mkdir -p bundle && cp -r dist package.json README.md LICENSE CHANGELOG.md bundle && cp package.publish.json bundle/package.json",
     "profile:bundle": "webpack --config ./config/webpack.dev.config.js --profile"
   },
@@ -38,7 +38,7 @@
   },
   "author": {
     "name": "hackerwins",
-    "email": "<[email protected]>"
+    "email": "[email protected]"
   },
   "license": "Apache-2.0",
   "bugs": {
@@ -51,7 +51,6 @@
     "@connectrpc/protoc-gen-connect-es": "^1.2.0",
     "@microsoft/api-documenter": "^7.15.1",
     "@microsoft/api-extractor": "^7.19.4",
-    "@types/benchmark": "^2.1.1",
     "@types/google-protobuf": "^3.15.5",
     "@types/jsdom": "^21.1.3",
     "@types/long": "^4.0.1",
@@ -61,7 +60,6 @@
     "@vitest/coverage-istanbul": "^0.34.5",
     "@vitest/coverage-v8": "^0.34.5",
     "@webpack-cli/serve": "^1.6.0",
-    "benchmark": "^2.1.4",
     "copy-webpack-plugin": "^11.0.0",
     "eslint": "^8.19.0",
     "eslint-plugin-jsdoc": "^39.3.3",

test/bench/counter.bench.ts

@@ -0,0 +1,60 @@
+import { Document, Counter } from '@yorkie-js-sdk/src/yorkie';
+import { CounterType } from '@yorkie-js-sdk/src/document/crdt/counter';
+import { describe, bench, assert } from 'vitest';
+
+const benchmarkCounter = (size: number) => {
+  const doc = new Document<{ counter: Counter }>('test-doc');
+
+  doc.update((root) => {
+    root.counter = new Counter(CounterType.IntegerCnt, 0);
+    for (let i = 0; i < size; i++) {
+      root.counter.increase(i);
+    }
+  });
+};
+
+describe('Counter', () => {
+  bench('counter', () => {
+    const doc = new Document<{ age: Counter; price: Counter }>('test-doc');
+    const integer = 10;
+    const long = 5;
+    const uinteger = 100;
+    const float = 3.14;
+    const double = 5.66;
+    doc.update((root) => {
+      root.age = new Counter(CounterType.IntegerCnt, 5);
+      root.age.increase(long);
+      root.age.increase(double);
+      root.age.increase(float);
+      root.age.increase(uinteger);
+      root.age.increase(integer);
+    });
+    assert.equal('{"age":128}', doc.toJSON());
+    doc.update((root) => {
+      root.price = new Counter(CounterType.LongCnt, 9000000000000000000);
+      root.price.increase(long);
+      root.price.increase(double);
+      root.price.increase(float);
+      root.price.increase(uinteger);
+      root.price.increase(integer);
+    });
+    assert.equal('{"age":128,"price":9000000000000000123}', doc.toJSON());
+    doc.update((root) => {
+      root.age.increase(-5);
+      root.age.increase(-3.14);
+      root.price.increase(-100);
+      root.price.increase(-20.5);
+    });
+    assert.equal('{"age":120,"price":9000000000000000003}', doc.toJSON());
+    // TODO: We need to filter not-allowed type
+    // counter.increase() method doesn't filter not-allowed type
+  });
+
+  bench('counter 1000', () => {
+    benchmarkCounter(1000);
+  });
+
+  bench('counter 10000', () => {
+    benchmarkCounter(10000);
+  });
+});

test/bench/document.bench.ts

@@ -0,0 +1,192 @@
+import { Document, JSONArray } from '@yorkie-js-sdk/src/yorkie';
+import { MaxTimeTicket } from '@yorkie-js-sdk/src/document/time/ticket';
+import { InitialCheckpoint } from '@yorkie-js-sdk/src/document/change/checkpoint';
+import { DocumentStatus } from '@yorkie-js-sdk/src/document/document';
+import { describe, bench, assert } from 'vitest';
+
+const benchmarkObject = (size: number) => {
+  const doc = new Document<{ k1: number }>('test-doc');
+
+  doc.update((root) => {
+    for (let i = 0; i < size; i++) {
+      root.k1 = i;
+    }
+  });
+};
+
+const benchmarkArray = (size: number) => {
+  const doc = new Document<{ k1: JSONArray<number> }>('test-doc');
+
+  doc.update((root) => {
+    root.k1 = [];
+
+    for (let i = 0; i < size; i++) {
+      root.k1.push(i);
+    }
+  });
+};
+
+const benchmarkArrayGC = (size: number) => {
+  const doc = new Document<{ k1?: JSONArray<number> }>('test-doc');
+
+  doc.update((root) => {
+    root.k1 = [];
+
+    for (let i = 0; i < size; i++) {
+      root.k1.push(i);
+    }
+  });
+  doc.update((root) => {
+    delete root.k1;
+  });
+
+  assert.equal(size + 1, doc.garbageCollect(MaxTimeTicket));
+};
+
+describe('Document', () => {
+  bench('constructor', () => {
+    const doc = new Document<{ text: JSONArray<string> }>(`test-doc`);
+    assert.equal('{}', doc.toJSON());
+    assert.equal(doc.getCheckpoint(), InitialCheckpoint);
+    assert.isFalse(doc.hasLocalChanges());
+  });
+
+  bench('status', () => {
+    const doc = new Document<{ text: JSONArray<string> }>(`test-doc`);
+    assert.equal(doc.getStatus(), DocumentStatus.Detached);
+    doc.applyStatus(DocumentStatus.Attached);
+    assert.equal(doc.getStatus(), DocumentStatus.Attached);
+  });
+
+  bench('equals', () => {
+    const doc1 = new Document<{ text: string }>('d1');
+    const doc2 = new Document<{ text: string }>('d2');
+    const doc3 = new Document<{ text: string }>('d3');
+    doc1.update((root) => {
+      root.text = 'value';
+    }, 'update text');
+    assert.notEqual(doc1.toJSON(), doc2.toJSON());
+    assert.equal(doc2.toJSON(), doc3.toJSON());
+  });
+
+  bench('nested update', () => {
+    const expected = `{"k1":"v1","k2":{"k4":"v4"},"k3":["v5","v6"]}`;
+    const doc = new Document<{
+      k1: string;
+      k2: { k4: string };
+      k3: Array<string>;
+    }>('test-doc');
+    assert.equal('{}', doc.toJSON());
+    assert.isFalse(doc.hasLocalChanges());
+    doc.update((root) => {
+      root.k1 = 'v1';
+      root.k2 = { k4: 'v4' };
+      root.k3 = ['v5', 'v6'];
+    }, 'updates k1,k2,k3');
+    assert.equal(expected, doc.toJSON());
+    assert.isTrue(doc.hasLocalChanges());
+  });
+
+  bench('delete', () => {
+    const doc = new Document<{
+      k1?: string;
+      k2?: { k4: string };
+      k3?: Array<string>;
+    }>('test-doc');
+    assert.equal('{}', doc.toJSON());
+    assert.isFalse(doc.hasLocalChanges());
+    let expected = `{"k1":"v1","k2":{"k4":"v4"},"k3":["v5","v6"]}`;
+    doc.update((root) => {
+      root.k1 = 'v1';
+      root.k2 = { k4: 'v4' };
+      root.k3 = ['v5', 'v6'];
+    }, 'updates k1,k2,k3');
+    assert.equal(expected, doc.toJSON());
+    expected = `{"k1":"v1","k3":["v5","v6"]}`;
+    doc.update((root) => {
+      delete root.k2;
+    }, 'deletes k2');
+    assert.equal(expected, doc.toJSON());
+  });
+
+  bench('object', () => {
+    const doc = new Document<{ k1: string }>('test-doc');
+    doc.update((root) => {
+      root.k1 = 'v1';
+      root.k1 = 'v2';
+    });
+    assert.equal(`{"k1":"v2"}`, doc.toJSON());
+  });
+
+  bench('array', () => {
+    const doc = new Document<{ k1: JSONArray<number> }>('test-doc');
+
+    doc.update((root) => {
+      root.k1 = [];
+      root.k1.push(1);
+      root.k1.push(2);
+      root.k1.push(3);
+
+      assert.equal('{"k1":[1,2,3]}', root.toJSON!());
+      assert.equal(root.k1.length, 3);
+      assert.equal(
+        '[1:000000000000000000000000:2:1][1:000000000000000000000000:3:2][1:000000000000000000000000:4:3]',
+        root.k1.toTestString!(),
+      );
+
+      root.k1.splice(1, 1);
+      assert.equal('{"k1":[1,3]}', root.toJSON!());
+      assert.equal(root.k1.length, 2);
+      assert.equal(
+        '[1:000000000000000000000000:2:1]{1:000000000000000000000000:3:2}[1:000000000000000000000000:4:3]',
+        root.k1.toTestString!(),
+      );
+
+      const first = root.k1.getElementByIndex!(0);
+      root.k1.insertAfter!(first.getID!(), 2);
+      assert.equal('{"k1":[1,2,3]}', root.toJSON!());
+      assert.equal(root.k1.length, 3);
+      assert.equal(
+        '[1:000000000000000000000000:2:1][1:000000000000000000000000:6:2]{1:000000000000000000000000:3:2}[1:000000000000000000000000:4:3]',
+        root.k1.toTestString!(),
+      );
+
+      const third = root.k1.getElementByIndex!(2);
+      root.k1.insertAfter!(third.getID!(), 4);
+      assert.equal('{"k1":[1,2,3,4]}', root.toJSON!());
+      assert.equal(root.k1.length, 4);
+      assert.equal(
+        '[1:000000000000000000000000:2:1][1:000000000000000000000000:6:2]{1:000000000000000000000000:3:2}[1:000000000000000000000000:4:3][1:000000000000000000000000:7:4]',
+        root.k1.toTestString!(),
+      );
+
+      for (let i = 0; i < root.k1.length; i++) {
+        assert.equal(i + 1, root.k1[i]);
+      }
+    });
+  });
+
+  bench('array 1000', () => {
+    benchmarkArray(1000);
+  });
+
+  bench('array 10000', () => {
+    benchmarkArray(10000);
+  });
+
+  bench('array GC 1000', () => {
+    benchmarkArrayGC(1000);
+  });
+
+  bench('array GC 10000', () => {
+    benchmarkArrayGC(10000);
+  });
+
+  bench('object 1000', () => {
+    benchmarkObject(1000);
+  });
+
+  bench('object 10000', () => {
+    benchmarkObject(10000);
+  });
+});