feat: rebuild SQLite when migrations occur

See [#436].

[#436]: #436

package.json

@@ -182,7 +182,7 @@
     "magic-bytes.js": "^1.10.0",
     "map-obj": "^5.0.2",
     "mime": "^4.0.3",
-    "multi-core-indexer": "^1.0.0-alpha.10",
+    "multi-core-indexer": "^1.0.0",
     "p-defer": "^4.0.0",
     "p-event": "^6.0.1",
     "p-timeout": "^6.1.2",

src/constants.js

@@ -32,3 +32,6 @@ export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
 })
 
 export const SUPPORTED_CONFIG_VERSION = 1
+
+// WARNING: This value is persisted. Be careful when changing it.
+export const DRIZZLE_MIGRATIONS_TABLE = '__drizzle_migrations'

src/datastore/README.md

@@ -19,6 +19,7 @@ const datastore = new DataStore({
     // Process entries here using an indexer...
   },
   namespace: 'data',
+  reindex: false,
 })
 
 /** @type {MapeoDoc} */

src/datastore/index.js

@@ -51,8 +51,9 @@ export class DataStore extends TypedEmitter {
    * @param {TNamespace} opts.namespace
    * @param {(entries: MultiCoreIndexer.Entry<'binary'>[]) => Promise<import('../index-writer/index.js').IndexedDocIds>} opts.batch
    * @param {MultiCoreIndexer.StorageParam} opts.storage
+   * @param {boolean} opts.reindex
    */
-  constructor({ coreManager, namespace, batch, storage }) {
+  constructor({ coreManager, namespace, batch, storage, reindex }) {
     super()
     this.#coreManager = coreManager
     this.#namespace = namespace
@@ -66,6 +67,7 @@ export class DataStore extends TypedEmitter {
     this.#coreIndexer = new MultiCoreIndexer(cores, {
       storage,
       batch: (entries) => this.#handleEntries(entries),
+      reindex,
     })
     coreManager.on('add-core', (coreRecord) => {
       if (coreRecord.namespace !== namespace) return

src/lib/drizzle-helpers.js

@@ -0,0 +1,48 @@
+import { sql } from 'drizzle-orm'
+import { assert } from '../utils.js'
+/** @import { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' */
+
+/**
+ * @param {unknown} queryResult
+ * @returns {number}
+ */
+const getNumberResult = (queryResult) => {
+  assert(
+    queryResult &&
+      typeof queryResult === 'object' &&
+      'result' in queryResult &&
+      typeof queryResult.result === 'number',
+    'expected query to return proper result'
+  )
+  return queryResult.result
+}
+
+/**
+ * Get the number of rows in a table using `SELECT COUNT(*)`.
+ * Returns 0 if the table doesn't exist.
+ *
+ * @param {BetterSQLite3Database} db
+ * @param {string} tableName
+ * @returns {number}
+ */
+export const tableCountIfExists = (db, tableName) =>
+  db.transaction((tx) => {
+    const existsQuery = sql`
+      SELECT EXISTS (
+        SELECT 1
+        FROM sqlite_master
+        WHERE type IS 'table'
+        AND name IS ${tableName}
+      ) AS result
+    `
+    const existsResult = tx.get(existsQuery)
+    const exists = getNumberResult(existsResult)
+    if (!exists) return 0
+
+    const countQuery = sql`
+      SELECT COUNT(*) AS result
+      FROM ${sql.identifier(tableName)}
+    `
+    const countResult = tx.get(countQuery)
+    return getNumberResult(countResult)
+  })

src/mapeo-project.js

@@ -6,7 +6,11 @@ import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
 import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 
-import { NAMESPACES, NAMESPACE_SCHEMAS } from './constants.js'
+import {
+  NAMESPACES,
+  NAMESPACE_SCHEMAS,
+  DRIZZLE_MIGRATIONS_TABLE,
+} from './constants.js'
 import { CoreManager } from './core-manager/index.js'
 import { DataStore } from './datastore/index.js'
 import { DataType, kCreateWithDocId } from './datatype/index.js'
@@ -44,6 +48,7 @@ import {
   projectKeyToPublicId,
   valueOf,
 } from './utils.js'
+import { tableCountIfExists } from './lib/drizzle-helpers.js'
 import { omit } from './lib/omit.js'
 import { MemberApi } from './member-api.js'
 import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js'
@@ -139,11 +144,37 @@ export class MapeoProject extends TypedEmitter {
     this.#isArchiveDevice = isArchiveDevice
 
     ///////// 1. Setup database
+
     this.#sqlite = new Database(dbPath)
     const db = drizzle(this.#sqlite)
-    migrate(db, { migrationsFolder: projectMigrationsFolder })
+    const migrationsBefore = tableCountIfExists(db, DRIZZLE_MIGRATIONS_TABLE)
+    migrate(db, {
+      migrationsFolder: projectMigrationsFolder,
+      migrationsTable: DRIZZLE_MIGRATIONS_TABLE,
+    })
+    const migrationsAfter = tableCountIfExists(db, DRIZZLE_MIGRATIONS_TABLE)
+    const reindex = migrationsBefore > 0 && migrationsAfter !== migrationsBefore
+
+    const indexedTables = [
+      observationTable,
+      trackTable,
+      presetTable,
+      fieldTable,
+      coreOwnershipTable,
+      roleTable,
+      deviceInfoTable,
+      iconTable,
+      translationTable,
+      remoteDetectionAlertTable,
+    ]
+
+    ///////// 2. Wipe data if we need to re-index
+
+    if (reindex) {
+      for (const table of indexedTables) db.delete(table).run()
+    }
 
-    ///////// 2. Setup random-access-storage functions
+    ///////// 3. Setup random-access-storage functions
 
     /** @type {ConstructorParameters<typeof CoreManager>[0]['storage']} */
     const coreManagerStorage = (name) =>
@@ -153,7 +184,7 @@ export class MapeoProject extends TypedEmitter {
     const indexerStorage = (name) =>
       coreStorage(path.join(INDEXER_STORAGE_FOLDER_NAME, name))
 
-    ///////// 3. Create instances
+    ///////// 4. Create instances
 
     this.#coreManager = new CoreManager({
       projectSecretKey,
@@ -166,18 +197,7 @@ export class MapeoProject extends TypedEmitter {
     })
 
     this.#indexWriter = new IndexWriter({
-      tables: [
-        observationTable,
-        trackTable,
-        presetTable,
-        fieldTable,
-        coreOwnershipTable,
-        roleTable,
-        deviceInfoTable,
-        iconTable,
-        translationTable,
-        remoteDetectionAlertTable,
-      ],
+      tables: indexedTables,
       sqlite: this.#sqlite,
       getWinner,
       mapDoc: (doc, version) => {
@@ -199,6 +219,7 @@ export class MapeoProject extends TypedEmitter {
         namespace: 'auth',
         batch: (entries) => this.#indexWriter.batch(entries),
         storage: indexerStorage,
+        reindex,
       }),
       config: new DataStore({
         coreManager: this.#coreManager,
@@ -209,12 +230,14 @@ export class MapeoProject extends TypedEmitter {
             sharedIndexWriter,
           }),
         storage: indexerStorage,
+        reindex,
       }),
       data: new DataStore({
         coreManager: this.#coreManager,
         namespace: 'data',
         batch: (entries) => this.#indexWriter.batch(entries),
         storage: indexerStorage,
+        reindex,
       }),
     }
 
@@ -363,7 +386,7 @@ export class MapeoProject extends TypedEmitter {
       dataType: this.#dataTypes.translation,
     })
 
-    ///////// 4. Replicate local peers automatically
+    ///////// 5. Replicate local peers automatically
 
     // Replicate already connected local peers
     for (const peer of localPeers.peers) {

test-e2e/migration.js

@@ -1,18 +1,111 @@
-import test from 'node:test'
 import { KeyManager } from '@mapeo/crypto'
-import RAM from 'random-access-memory'
-import { MapeoManager } from '../src/mapeo-manager.js'
 import Fastify from 'fastify'
 import assert from 'node:assert/strict'
 import fsPromises from 'node:fs/promises'
+import test from 'node:test'
+import RAM from 'random-access-memory'
 import { temporaryDirectory } from 'tempy'
-import { createOldManagerOnVersion2_0_1 } from './utils.js'
+import { MapeoManager } from '../src/mapeo-manager.js'
+import {
+  connectPeers,
+  createManager,
+  createOldManagerOnVersion2_0_1,
+  invite,
+} from './utils.js'
 
 const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url)
   .pathname
 const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url)
   .pathname
 
+test('migrations pick up values that were not previously understood', async (t) => {
+  // Create Manager 1, which has new data.
+
+  const manager1 = createManager('a', t)
+  await manager1.setDeviceInfo({
+    name: 'a',
+    deviceType: 'selfHostedServer',
+    // Old versions shouldn't be able to recognize this.
+    selfHostedServerDetails: { baseUrl: 'https://comapeo-test.example/' },
+  })
+
+  const projectId = await manager1.createProject({ name: 'test project' })
+  const manager1Project = await manager1.getProject(projectId)
+
+  {
+    const manager1Members = await manager1Project.$member.getMany()
+    assert(
+      manager1Members.some(
+        (member) =>
+          member.selfHostedServerDetails?.baseUrl ===
+          'https://comapeo-test.example/'
+      ),
+      'test setup: new manager has new data'
+    )
+  }
+
+  // Create Manager 2, which is not yet up to date.
+
+  const manager2DbFolder = temporaryDirectory()
+  const manager2CoreStorage = temporaryDirectory()
+  t.after(() => fsPromises.rm(manager2DbFolder, { recursive: true }))
+  t.after(() => fsPromises.rm(manager2CoreStorage, { recursive: true }))
+
+  const manager2BeforeMigration = await createOldManagerOnVersion2_0_1('b', {
+    dbFolder: manager2DbFolder,
+    coreStorage: manager2CoreStorage,
+  })
+  await manager2BeforeMigration.setDeviceInfo({
+    name: 'b',
+    deviceType: 'mobile',
+  })
+
+  // Connect them and ensure that Manager 2 doesn't yet know about the new data.
+
+  const disconnect = connectPeers([manager1, manager2BeforeMigration])
+
+  await invite({
+    projectId,
+    invitor: manager1,
+    invitees: [manager2BeforeMigration],
+  })
+
+  {
+    const manager2Project = await manager2BeforeMigration.getProject(projectId)
+    await manager2Project.$sync.waitForSync('initial')
+    const manager2Members = await manager2Project.$member.getMany()
+    assert(
+      !manager2Members.some((member) => 'selfHostedServerDetails' in member),
+      "test setup: old manager doesn't understand new data (yet)"
+    )
+
+    await manager2Project.close()
+  }
+
+  await disconnect()
+
+  // Migrate Manager 2 and see that it now knows about the data.
+
+  const manager2AfterMigration = createManager('b', t, {
+    dbFolder: manager2DbFolder,
+    coreStorage: manager2CoreStorage,
+  })
+
+  {
+    const manager2Project = await manager2AfterMigration.getProject(projectId)
+    const manager2Members = await manager2Project.$member.getMany()
+    const serverMember = manager2Members.find(
+      (member) => member.deviceType === 'selfHostedServer'
+    )
+    assert(serverMember, 'we still have the server member')
+    assert.equal(
+      serverMember.selfHostedServerDetails?.baseUrl,
+      'https://comapeo-test.example/',
+      'migrated manager has new data'
+    )
+  }
+})
+
 test('migration of localDeviceInfo table', async (t) => {
   const dbFolder = temporaryDirectory()
   const rootKey = KeyManager.generateRootKey()

test/data-type.js

@@ -60,6 +60,7 @@ test('private createWithDocId() method', async () => {
       return indexWriter.batch(entries)
     },
     storage: () => new RAM(),
+    reindex: false,
   })
   const dataType = new DataType({
     dataStore,
@@ -95,6 +96,7 @@ test('private createWithDocId() method throws when doc exists', async () => {
       return indexWriter.batch(entries)
     },
     storage: () => new RAM(),
+    reindex: false,
   })
   const dataType = new DataType({
     dataStore,
@@ -316,6 +318,7 @@ async function testenv(opts = {}) {
     namespace: 'data',
     batch: async (entries) => indexWriter.batch(entries),
     storage: () => new RAM(),
+    reindex: false,
   })
 
   const configDataStore = new DataStore({
@@ -347,6 +350,7 @@ async function testenv(opts = {}) {
       return indexed
     },
     storage: () => new RAM(),
+    reindex: false,
   })
 
   const translationDataType = new DataType({