feat(indexedDB): clean up searchIndex, lunr -> minisearch, friends in…

… indexeddb

.gitignore

@@ -91,3 +91,7 @@ sw.*
 
 # Vim swap files
 *.swp
+
+# Scratchpad files for testing stuff
+scratchpad.ts
+scratchpad.js

.vscode/settings.json

@@ -3,5 +3,6 @@
   "editor.formatOnSave": true,
   "[vue]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
-  }
+  },
+  "cSpell.words": ["Dexie"]
 }

cli.tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "module": "commonjs",
+    "sourceMap": true,
+    "declaration": true,
+    "esModuleInterop": true,
+    "paths": {
+      "~/*": ["./*"],
+      "@/*": ["./*"]
+    },
+    "types": ["@types/node", "types/modules", "jest"]
+  },
+  "exclude": ["node_modules", ".nuxt", "dist"]
+}

components/interactables/Search/Input/Input.html

@@ -1,5 +1,5 @@
 <div class="search-container" v-click-outside="lostFocus">
-  <div :class="{'search-box': true, focus: isFocus}" class="disabled">
+  <div :class="{'search-box': true, focus: isFocus}">
     <div :class="{'search-placeholder': true, hide: !isEmpty}">
       {{placeholder}}
     </div>
@@ -12,13 +12,12 @@
     <div
       ref="searchInput"
       class="search-input"
-      @mousedown="/* setFocus disable temporarily until search rework */"
+      @mousedown="setFocus"
       @keydown="keydown"
       @keyup="keyup"
       @input="input"
     ></div>
   </div>
-  <!-- temporarily disabled due to conflicts with solr search filters (from:, has:, etc...)
   <div :class="{'search-options': true}" v-if="isOption">
     <ul class="search-option-group">
       <li class="search-option-heading">
@@ -60,7 +59,6 @@
       </li>
     </ul>
   </div>
-  -->
   <div
     refs="datePicker"
     :class="{'date-selection': true}"

libraries/SatelliteDB/SatelliteDB.test.ts

@@ -0,0 +1,70 @@
+import 'fake-indexeddb/auto'
+import { SatelliteDB } from './SatelliteDB'
+import SearchIndex from './SearchIndex'
+
+describe('SatelliteDB', () => {
+  let db: SatelliteDB
+  beforeEach(async () => {
+    db = new SatelliteDB()
+  })
+  test('tables', () => {
+    expect(Object.keys(db.tables)).toMatchSnapshot()
+  })
+
+  test('storing and retrieving data', async () => {
+    const data = [
+      {
+        key: '1',
+        lastInbound: 1,
+      },
+      {
+        key: '2',
+        lastInbound: 2,
+      },
+    ]
+    await db.conversations.bulkPut(data)
+    expect(await db.conversations.toArray()).toEqual(data)
+  })
+
+  test('creating search indexes', async () => {
+    await db.initializeSearchIndexes()
+    expect(Object.keys(db.search)).toEqual(['friends', 'conversationMessages'])
+    expect(db.search.conversationMessages).toBeInstanceOf(SearchIndex)
+  })
+
+  test('restoring search indexes', async () => {
+    const where = { address: 'foo' }
+    const data = { name: 'bar', textilePubkey: 'baz' }
+    await db.initializeSearchIndexes()
+    await db.upsert('friends', where, data)
+    await db.saveSearchIndexes()
+    await db.close()
+
+    db = new SatelliteDB()
+    await db.initializeSearchIndexes()
+    expect(await db.search.friends.search('bar')?.[0]?.address).toEqual(
+      where.address,
+    )
+  })
+
+  test('upserting records', async () => {
+    const where = { address: '1' }
+    const original = {
+      textilePubkey: 'foobarbaz',
+      name: 'foo',
+      photoHash: 'bar',
+      lastUpdate: 1,
+    }
+    await db.upsert('friends', where, original)
+    expect(await db.friends.get('1')).toEqual({ ...where, ...original })
+    const update = {
+      name: 'foo bar',
+    }
+    await db.upsert('friends', where, update)
+    expect(await db.friends.get('1')).toEqual({
+      ...where,
+      ...original,
+      ...update,
+    })
+  })
+})

libraries/SatelliteDB/SatelliteDB.ts

@@ -0,0 +1,160 @@
+import { Dexie, IndexableType } from 'dexie'
+import SearchIndex from './SearchIndex'
+import { Message } from '~/types/textile/mailbox'
+
+export type DexieConversation = {
+  key: string
+  lastInbound: number
+}
+export type DexieMessage = Message & {
+  conversation: string
+}
+
+export type DexieFriend = {
+  address: string
+  name: string
+  photoHash: string | undefined
+  textilePubkey: string
+  lastUpdate: number
+}
+
+export type KeyValue = {
+  key: string
+  value: string
+  namespace: string
+}
+
+export class SatelliteDB extends Dexie {
+  public isInitialized: boolean
+  public conversations: Dexie.Table<DexieConversation, string>
+  public conversationMessages: Dexie.Table<DexieMessage, string>
+  public friends: Dexie.Table<DexieFriend, string>
+  public keyValue: Dexie.Table<KeyValue, string>
+
+  public search: {
+    [key: string]: SearchIndex
+  } = {}
+
+  public constructor() {
+    super('SatelliteDB')
+    this.initializeSchema()
+
+    this.conversations = this.table('conversations')
+    this.conversationMessages = this.table('conversationMessages')
+    this.friends = this.table('friends')
+    this.keyValue = this.table('keyValue')
+  }
+
+  /**
+   * Initialize the schema for the database.
+   * @returns {void}
+   */
+  initializeSchema() {
+    if (this.isInitialized) return
+    this.version(1).stores({
+      conversations: 'key, lastInbound',
+      conversationMessages:
+        '&id, conversation, from, to, at, readAt, type, payload',
+    })
+
+    this.version(2).stores({
+      friends: '&address, textilePubkey, name, photoHash, lastUpdate',
+    })
+
+    this.version(3).stores({
+      keyValue: '&key, value, namespace',
+    })
+  }
+
+  /**
+   * Initialize search indexes for tables that have them.
+   * @returns {Promise<void>
+   */
+  async initializeSearchIndexes() {
+    if (this.isInitialized) return
+    const searchIndexes: KeyValue[] = await this.keyValue
+      .where('namespace')
+      .equals('searchIndex')
+      .toArray()
+    for (const index of searchIndexes) {
+      const { key, value } = index
+      if (typeof key === 'string' && value) {
+        try {
+          this.search[key] = SearchIndex.deserialize(value)
+        } catch (_) {}
+      }
+    }
+
+    if (!this.search.friends) {
+      this.search.friends = new SearchIndex({
+        schema: {
+          fields: ['address', 'name', 'photoHash', 'textilePubkey'],
+          storeFields: ['address', 'name', 'photoHash', 'textilePubkey'],
+          idField: 'address',
+        },
+      })
+    }
+
+    if (!this.search.conversationMessages) {
+      this.search.conversationMessages = new SearchIndex({
+        schema: {
+          fields: [
+            'id',
+            'conversation',
+            'from',
+            'to',
+            'at',
+            'readAt',
+            'type',
+            'payload',
+          ],
+          storeFields: [
+            'id',
+            'conversation',
+            'from',
+            'to',
+            'at',
+            'readAt',
+            'type',
+            'payload',
+          ],
+        },
+      })
+    }
+  }
+
+  /**
+   *
+   * @param table the name of the table
+   * @param where the where clause
+   * @param data the data to insert or update. if inserting, will merge with where clause,
+   * @returns {Promise<IndexableType>}
+   */
+  async upsert(
+    table: string,
+    where: { [key: string]: any },
+    data: { [key: string]: any },
+  ) {
+    if (this.search[table]) {
+      this.search[table].add({ ...where, ...data })
+    }
+    const exists = await this.table(table).get(where)
+    if (exists) {
+      return this.table(table).update(where, data)
+    }
+    return this.table(table).add({ ...where, ...data })
+  }
+
+  async saveSearchIndexes() {
+    for (const [key, index] of Object.entries(this.search)) {
+      await this.keyValue.put({
+        key,
+        value: index.serialize(),
+        namespace: 'searchIndex',
+      })
+    }
+  }
+}
+
+export const db = new SatelliteDB()
+export default db

libraries/SatelliteDB/SearchIndex.test.ts

@@ -0,0 +1,87 @@
+import SearchIndex from './SearchIndex'
+
+describe('SatelliteDB/SearchIndex', () => {
+  let idx: SearchIndex
+  beforeAll(() => {
+    idx = new SearchIndex({
+      schema: {
+        fields: ['id', 'text'],
+        storeFields: ['id', 'text'],
+      },
+    })
+  })
+
+  beforeEach(() => {
+    idx.update([])
+  })
+
+  test('constructor without a schema', () => {
+    const idxb = new SearchIndex()
+  })
+
+  test('searchIndex.update()', async () => {
+    const data = [
+      { id: '1', text: 'first match' },
+      { id: '2', text: 'second match' },
+      { id: '3', text: 'third match' },
+    ]
+    idx.update(data)
+    expect(idx.search('first')?.map((r) => r.id)).toEqual(['1'])
+    expect(idx.search('second')?.map((r) => r.id)).toEqual(['2'])
+    expect(idx.search('third')?.map((r) => r.id)).toEqual(['3'])
+    expect(idx.search('match')?.map((r) => r.id)).toEqual(['3', '2', '1'])
+  })
+
+  test('searchIndex.update() without an id', async () => {
+    const data = [{ text: 'foo bar' }]
+    expect(() => idx.update(data)).toThrow()
+  })
+
+  test('searchIndex.add()', async () => {
+    const data = { id: '1', text: 'foo bar' }
+    idx.add(data)
+    expect(idx.search('foo')?.map((r) => r.id)).toEqual(['1'])
+  })
+
+  test('searchIndex.add() without an id', async () => {
+    const data = { text: 'foo bar' }
+    expect(() => idx.add(data)).toThrow()
+  })
+
+  test('searchIndex.remove()', async () => {
+    const data = [
+      { id: '1', text: 'first match' },
+      { id: '2', text: 'second match' },
+    ]
+    idx.update(data)
+    expect(idx.search('first')?.map((r) => r.id)).toEqual(['1'])
+    expect(idx.search('second')?.map((r) => r.id)).toEqual(['2'])
+    expect(idx.search('match')?.map((r) => r.id)).toEqual(['2', '1'])
+    idx.remove(data[0])
+    expect(idx.search('first')?.map((r) => r.id)).toEqual([])
+    expect(idx.search('match')?.map((r) => r.id)).toEqual(['2'])
+  })
+
+  test('searchIndex.addAll()', async () => {
+    const data = [
+      { id: '1', text: 'first match' },
+      { id: '2', text: 'second match' },
+    ]
+    idx.addAll(data)
+    expect(idx.search('first')?.map((r) => r.id)).toEqual(['1'])
+    expect(idx.search('second')?.map((r) => r.id)).toEqual(['2'])
+    expect(idx.search('match')?.map((r) => r.id)).toEqual(['2', '1'])
+  })
+
+  test('searchIndex.removeAll()', async () => {
+    const data = [
+      { id: '1', text: 'first match' },
+      { id: '2', text: 'second match' },
+    ]
+    idx.update(data)
+    idx.removeAll()
+    expect(idx.search('first')?.map((r) => r.id)).toEqual([])
+    expect(idx.search('second')?.map((r) => r.id)).toEqual([])
+    expect(idx.search('match')?.map((r) => r.id)).toEqual([])
+  })
+})