wip: preliminary DB fixes.

MatrixAI · Aug 9, 2022 · 873f8d4 · 873f8d4
1 parent 3c1b8c8
commit 873f8d4
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 75 deletions.
diff --git a/benches/gitgc.ts b/benches/gitgc.ts
@@ -2,7 +2,7 @@ import path from 'path';
 import b from 'benny';
 import { suiteCommon } from './utils';
 
-async function main () {
+async function main() {
   let map = new Map();
   let obj = {};
   let arr: any = [];
@@ -18,10 +18,10 @@ async function main () {
         for (let i = 0; i < 1000; i++) {
           map.delete(i);
         }
-        for (const i of map) {
+        for (const _i of map) {
           // NOOP
         }
-      }
+      };
     }),
     b.add('obj', async () => {
       obj = {};
@@ -32,26 +32,26 @@ async function main () {
         for (let i = 0; i < 1000; i++) {
           delete obj[i];
         }
-        for (const i in obj) {
+        for (const _i in obj) {
           // NOOP
         }
       };
     }),
     b.add('arr', async () => {
-      // you first have to count the number of objects
+      // You first have to count the number of objects
       arr = [];
       return async () => {
-        // you have to iterate for each object
+        // You have to iterate for each object
         // then for each value in length
         for (let i = 0; i < 1000; i++) {
           if (i === arr.length) {
-            // double the vector
+            // Double the vector
             arr.length = arr.length * 2 || 2;
           }
           arr[i] = { id: i, mark: false };
-          // arr.push({ id: i, mark: false});
+          // Arr.push({ id: i, mark: false});
         }
-        // this has to iterate the length of the array
+        // This has to iterate the length of the array
         // but stop as soon as it reaches the end
         // it gets complicate, but for 5x improvement
         // it could be interesting
@@ -74,7 +74,7 @@ async function main () {
         for (let i = 0; i < 1000; i++) {
           set.delete(i);
         }
-        for (const i of set) {
+        for (const _i of set) {
           // NOOP
         }
       };

diff --git a/src/acl/ACL.ts b/src/acl/ACL.ts
@@ -133,7 +133,7 @@ class ACL {
       return this.withTransactionF(async (tran) => this.getNodePerms(tran));
     }
     const permIds: Record<PermissionIdString, Record<NodeId, Permission>> = {};
-    for await (const [keyPath, value] of tran.iterator(undefined, [
+    for await (const [keyPath, value] of tran.iterator([
       ...this.aclNodesDbPath,
     ])) {
       const key = keyPath[0] as Buffer;
@@ -175,8 +175,8 @@ class ACL {
     }
     const vaultPerms: Record<VaultId, Record<NodeId, Permission>> = {};
     for await (const [keyPath, nodeIds] of tran.iterator<Record<NodeId, null>>(
-      { valueAsBuffer: false },
       [...this.aclVaultsDbPath],
+      { valueAsBuffer: false },
     )) {
       const key = keyPath[0] as Buffer;
       const vaultId = IdInternal.fromBuffer<VaultId>(key);

diff --git a/src/discovery/Discovery.ts b/src/discovery/Discovery.ts
@@ -130,10 +130,11 @@ class Discovery {
     }
     // Getting latest ID and creating ID generator
     let latestId: DiscoveryQueueId | undefined;
-    const keyIterator = this.db.iterator(
-      { limit: 1, reverse: true, values: false },
-      this.discoveryQueueDbPath,
-    );
+    const keyIterator = this.db.iterator(this.discoveryQueueDbPath, {
+      limit: 1,
+      reverse: true,
+      values: false,
+    });
     for await (const [keyPath] of keyIterator) {
       const key = keyPath[0] as Buffer;
       latestId = IdInternal.fromBuffer<DiscoveryQueueId>(key);
@@ -204,8 +205,8 @@ class Discovery {
       // Processing queue
       this.logger.debug('DiscoveryQueue is processing');
       for await (const [keyPath, vertex] of this.db.iterator<GestaltKey>(
-        { valueAsBuffer: false },
         this.discoveryQueueDbPath,
+        { valueAsBuffer: false },
       )) {
         const key = keyPath[0] as Buffer;
         const vertexId = IdInternal.fromBuffer<DiscoveryQueueId>(key);
@@ -425,10 +426,10 @@ class Discovery {
   protected async queueIsEmpty(): Promise<boolean> {
     return await this.lock.withF(async () => {
       let nextDiscoveryQueueId: DiscoveryQueueId | undefined;
-      const keyIterator = this.db.iterator(
-        { limit: 1, values: false },
-        this.discoveryQueueDbPath,
-      );
+      const keyIterator = this.db.iterator(this.discoveryQueueDbPath, {
+        limit: 1,
+        values: false,
+      });
       for await (const [keyPath] of keyIterator) {
         const key = keyPath[0] as Buffer;
         nextDiscoveryQueueId = IdInternal.fromBuffer<DiscoveryQueueId>(key);
@@ -449,8 +450,8 @@ class Discovery {
       [this.db.transaction(), this.lock.lock()],
       async ([tran]) => {
         const valueIterator = tran.iterator<GestaltKey>(
-          { valueAsBuffer: false },
           this.discoveryQueueDbPath,
+          { valueAsBuffer: false },
         );
         for await (const [, value] of valueIterator) {
           if (value === gestaltKey) {

diff --git a/src/gestalts/GestaltGraph.ts b/src/gestalts/GestaltGraph.ts
@@ -104,8 +104,8 @@ class GestaltGraph {
     }
     const unvisited: Map<GestaltKey, GestaltKeySet> = new Map();
     for await (const [k, gKs] of tran.iterator<GestaltKeySet>(
-      { valueAsBuffer: false },
       [...this.gestaltGraphMatrixDbPath],
+      { valueAsBuffer: false },
     )) {
       const gK = k.toString() as GestaltKey;
       unvisited.set(gK, gKs);

diff --git a/src/nodes/NodeGraph.ts b/src/nodes/NodeGraph.ts
@@ -192,11 +192,11 @@ class NodeGraph {
     }
 
     for await (const [keyPath, nodeData] of tran.iterator<NodeData>(
+      this.nodeGraphBucketsDbPath,
       {
         reverse: order !== 'asc',
         valueAsBuffer: false,
       },
-      this.nodeGraphBucketsDbPath,
     )) {
       const { nodeId } = nodesUtils.parseBucketsDbKey(keyPath);
       yield [nodeId, nodeData];
@@ -273,10 +273,10 @@ class NodeGraph {
     const bucketKey = nodesUtils.bucketKey(bucketIndex);
     // Remove the oldest entry in the bucket
     const oldestNodeIds: Array<NodeId> = [];
-    for await (const [keyPath] of tran.iterator({ limit }, [
-      ...this.nodeGraphLastUpdatedDbPath,
-      bucketKey,
-    ])) {
+    for await (const [keyPath] of tran.iterator(
+      [...this.nodeGraphLastUpdatedDbPath, bucketKey],
+      { limit },
+    )) {
       const { nodeId } = nodesUtils.parseLastUpdatedBucketDbKey(keyPath);
       oldestNodeIds.push(nodeId);
     }
@@ -338,11 +338,11 @@ class NodeGraph {
     const bucket: NodeBucket = [];
     if (sort === 'nodeId' || sort === 'distance') {
       for await (const [key, nodeData] of tran.iterator<NodeData>(
+        [...this.nodeGraphBucketsDbPath, bucketKey],
         {
           reverse: order !== 'asc',
           valueAsBuffer: false,
         },
-        [...this.nodeGraphBucketsDbPath, bucketKey],
       )) {
         const nodeId = nodesUtils.parseBucketDbKey(key[0] as Buffer);
         bucket.push([nodeId, nodeData]);
@@ -356,15 +356,15 @@ class NodeGraph {
       }
     } else if (sort === 'lastUpdated') {
       const bucketDbIterator = tran.iterator<NodeData>(
-        { valueAsBuffer: false },
         [...this.nodeGraphBucketsDbPath, bucketKey],
+        { valueAsBuffer: false },
       );
       try {
         for await (const [, nodeIdBuffer] of tran.iterator(
+          [...this.nodeGraphLastUpdatedDbPath, bucketKey],
           {
             reverse: order !== 'asc',
           },
-          [...this.nodeGraphLastUpdatedDbPath, bucketKey],
         )) {
           const nodeId = IdInternal.fromBuffer<NodeId>(nodeIdBuffer);
           bucketDbIterator.seek(nodeIdBuffer);
@@ -375,7 +375,7 @@ class NodeGraph {
           bucket.push([nodeId, nodeData]);
         }
       } finally {
-        await bucketDbIterator.end();
+        await bucketDbIterator.destroy(); // FIXME: should this be `.destroy` now?
       }
     }
     return bucket;
@@ -410,11 +410,11 @@ class NodeGraph {
     let bucket: NodeBucket = [];
     if (sort === 'nodeId' || sort === 'distance') {
       for await (const [key, nodeData] of tran.iterator<NodeData>(
+        this.nodeGraphBucketsDbPath,
         {
           reverse: order !== 'asc',
           valueAsBuffer: false,
         },
-        this.nodeGraphBucketsDbPath,
       )) {
         const { bucketIndex: bucketIndex_, nodeId } =
           nodesUtils.parseBucketsDbKey(key);
@@ -452,15 +452,15 @@ class NodeGraph {
       }
     } else if (sort === 'lastUpdated') {
       const bucketsDbIterator = tran.iterator<NodeData>(
-        { valueAsBuffer: false },
         this.nodeGraphBucketsDbPath,
+        { valueAsBuffer: false },
       );
       try {
         for await (const [key] of tran.iterator(
+          this.nodeGraphLastUpdatedDbPath,
           {
             reverse: order !== 'asc',
           },
-          this.nodeGraphLastUpdatedDbPath,
         )) {
           const { bucketIndex: bucketIndex_, nodeId } =
             nodesUtils.parseLastUpdatedBucketsDbKey(key);
@@ -488,7 +488,7 @@ class NodeGraph {
           yield [bucketIndex, bucket];
         }
       } finally {
-        await bucketsDbIterator.end();
+        await bucketsDbIterator.destroy(); // FIXME: destroy?
       }
     }
   }
@@ -524,8 +524,8 @@ class NodeGraph {
     // Iterating over all entries across all buckets
 
     for await (const [key, nodeData] of tran.iterator<NodeData>(
-      { valueAsBuffer: false },
       this.nodeGraphBucketsDbPath,
+      { valueAsBuffer: false },
     )) {
       // The key is a combined bucket key and node ID
       const { bucketIndex: bucketIndexOld, nodeId } =
@@ -551,12 +551,9 @@ class NodeGraph {
       } else {
         let oldestIndexKey: KeyPath | undefined = undefined;
         let oldestNodeId: NodeId | undefined = undefined;
-        for await (const [key] of tran.iterator(
-          {
-            limit: 1,
-          },
-          indexPathNew,
-        )) {
+        for await (const [key] of tran.iterator(indexPathNew, {
+          limit: 1,
+        })) {
           oldestIndexKey = key;
           ({ nodeId: oldestNodeId } =
             nodesUtils.parseLastUpdatedBucketDbKey(key));
@@ -716,12 +713,12 @@ class NodeGraph {
       const remainingLimit = limit - nodeIds.length;
       // Iterate over lower buckets
       for await (const [key, nodeData] of tran.iterator<NodeData>(
+        this.nodeGraphBucketsDbPath,
         {
           lt: [bucketIdKey, ''],
           limit: remainingLimit,
           valueAsBuffer: false,
         },
-        this.nodeGraphBucketsDbPath,
       )) {
         const info = nodesUtils.parseBucketsDbKey(key);
         nodeIds.push([info.nodeId, nodeData]);
@@ -732,20 +729,17 @@ class NodeGraph {
       const bucketId = Buffer.from(nodesUtils.bucketKey(startingBucket + 1));
       const remainingLimit = limit - nodeIds.length;
       // Iterate over ids further away
-      tran.iterator(
-        {
-          gt: [bucketId, ''],
-          limit: remainingLimit,
-        },
-        this.nodeGraphBucketsDbPath,
-      );
+      tran.iterator(this.nodeGraphBucketsDbPath, {
+        gt: [bucketId, ''],
+        limit: remainingLimit,
+      });
       for await (const [key, nodeData] of tran.iterator<NodeData>(
+        this.nodeGraphBucketsDbPath,
         {
           gt: [bucketId, ''],
           limit: remainingLimit,
           valueAsBuffer: false,
         },
-        this.nodeGraphBucketsDbPath,
       )) {
         const info = nodesUtils.parseBucketsDbKey(key);
         nodeIds.push([info.nodeId, nodeData]);

diff --git a/src/notifications/NotificationsManager.ts b/src/notifications/NotificationsManager.ts
@@ -139,10 +139,11 @@ class NotificationsManager {
 
         // Getting latest ID and creating ID generator
         let latestId: NotificationId | undefined;
-        const keyIterator = tran.iterator(
-          { limit: 1, reverse: true, values: false },
-          this.notificationsMessagesDbPath,
-        );
+        const keyIterator = tran.iterator(this.notificationsMessagesDbPath, {
+          limit: 1,
+          reverse: true,
+          values: false,
+        });
         for await (const [keyPath] of keyIterator) {
           const key = keyPath[0] as Buffer;
           latestId = IdInternal.fromBuffer<NotificationId>(key);
@@ -368,8 +369,8 @@ class NotificationsManager {
   ): Promise<Array<NotificationId>> {
     const notificationIds: Array<NotificationId> = [];
     const messageIterator = tran.iterator<Notification>(
-      { valueAsBuffer: false },
       this.notificationsMessagesDbPath,
+      { valueAsBuffer: false },
     );
     for await (const [keyPath, notification] of messageIterator) {
       const key = keyPath[0] as Buffer;
@@ -391,8 +392,8 @@ class NotificationsManager {
   ): Promise<Array<Notification>> {
     const notifications: Array<Notification> = [];
     for await (const [, notification] of tran.iterator<Notification>(
-      { valueAsBuffer: false },
       this.notificationsMessagesDbPath,
+      { valueAsBuffer: false },
     )) {
       if (type === 'all') {
         notifications.push(notification);