feat: rebuild SQLite when migrations occur

See [#436].

[#436]: #436

src/constants.js

@@ -32,3 +32,6 @@ export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
 })
 
 export const SUPPORTED_CONFIG_VERSION = 1
+
+// WARNING: This value is persisted. Be careful when changing it.
+export const DRIZZLE_MIGRATIONS_TABLE = '__drizzle_migrations'

src/lib/count-drizzle-migrations.js

@@ -0,0 +1,45 @@
+import { sql } from 'drizzle-orm'
+import { assert } from '../utils.js'
+/** @import { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' */
+
+/**
+ * @param {unknown} queryResult
+ * @returns {number}
+ */
+const getNumberResult = (queryResult) => {
+  assert(
+    queryResult &&
+      typeof queryResult === 'object' &&
+      'result' in queryResult &&
+      typeof queryResult.result === 'number',
+    'expected query to return proper result'
+  )
+  return queryResult.result
+}
+
+/**
+ * @param {BetterSQLite3Database} db
+ * @param {string} migrationsTableName
+ * @returns {number}
+ */
+export const countDrizzleMigrations = (db, migrationsTableName) =>
+  db.transaction((tx) => {
+    const existsQuery = sql`
+      SELECT EXISTS (
+        SELECT 1
+        FROM sqlite_master
+        WHERE type IS 'table'
+        AND name IS ${migrationsTableName}
+      ) AS result
+    `
+    const existsResult = tx.get(existsQuery)
+    const exists = getNumberResult(existsResult)
+    if (!exists) return 0
+
+    const countQuery = sql`
+      SELECT COUNT(*) AS result
+      FROM ${sql.identifier(migrationsTableName)}
+    `
+    const countResult = tx.get(countQuery)
+    return getNumberResult(countResult)
+  })

src/mapeo-manager.js

@@ -10,6 +10,7 @@ import { TypedEmitter } from 'tiny-typed-emitter'
 import pTimeout from 'p-timeout'
 import { createRequire } from 'module'
 
+import { DRIZZLE_MIGRATIONS_TABLE } from './constants.js'
 import { IndexWriter } from './index-writer/index.js'
 import {
   MapeoProject,
@@ -153,7 +154,12 @@ export class MapeoManager extends TypedEmitter {
         : path.join(dbFolder, CLIENT_SQLITE_FILE_NAME)
     )
     this.#db = drizzle(sqlite)
-    migrate(this.#db, { migrationsFolder: clientMigrationsFolder })
+    migrate(this.#db, {
+      migrationsFolder: clientMigrationsFolder,
+      migrationsTable: DRIZZLE_MIGRATIONS_TABLE,
+    })
+    // TODO(evanhahn) Do we need to handle migration changes and clear storage
+    // here? Doesn't actually seem relevant yet but could be...
 
     this.#localPeers = new LocalPeers({ logger })
     this.#localPeers.on('peers', (peers) => {

src/mapeo-project.js

@@ -1,12 +1,17 @@
 import path from 'path'
+import * as fs from 'node:fs'
 import Database from 'better-sqlite3'
 import { decodeBlockPrefix, decode, parseVersionId } from '@comapeo/schema'
 import { drizzle } from 'drizzle-orm/better-sqlite3'
 import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
 import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 
-import { NAMESPACES, NAMESPACE_SCHEMAS } from './constants.js'
+import {
+  NAMESPACES,
+  NAMESPACE_SCHEMAS,
+  DRIZZLE_MIGRATIONS_TABLE,
+} from './constants.js'
 import { CoreManager } from './core-manager/index.js'
 import { DataStore } from './datastore/index.js'
 import { DataType, kCreateWithDocId } from './datatype/index.js'
@@ -43,6 +48,7 @@ import {
   projectKeyToPublicId,
   valueOf,
 } from './utils.js'
+import { countDrizzleMigrations } from './lib/count-drizzle-migrations.js'
 import { MemberApi } from './member-api.js'
 import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js'
 import { Logger } from './logger.js'
@@ -131,11 +137,44 @@ export class MapeoProject extends TypedEmitter {
     this.#loadingConfig = false
 
     ///////// 1. Setup database
+
     this.#sqlite = new Database(dbPath)
     const db = drizzle(this.#sqlite)
-    migrate(db, { migrationsFolder: projectMigrationsFolder })
+    const migrationsBefore = countDrizzleMigrations(
+      db,
+      DRIZZLE_MIGRATIONS_TABLE
+    )
+    migrate(db, {
+      migrationsFolder: projectMigrationsFolder,
+      migrationsTable: DRIZZLE_MIGRATIONS_TABLE,
+    })
+    const migrationsAfter = countDrizzleMigrations(db, DRIZZLE_MIGRATIONS_TABLE)
+    const didMigrateDatabase = migrationsAfter !== migrationsBefore
+
+    const indexedTables = [
+      observationTable,
+      trackTable,
+      presetTable,
+      fieldTable,
+      coreOwnershipTable,
+      roleTable,
+      deviceInfoTable,
+      iconTable,
+      translationTable,
+    ]
+
+    ///////// 2. Wipe data if we need to re-index
+
+    if (didMigrateDatabase) {
+      fs.rmSync(INDEXER_STORAGE_FOLDER_NAME, {
+        force: true,
+        recursive: true,
+        maxRetries: 10,
+      })
+      for (const table of indexedTables) db.delete(table).run()
+    }
 
-    ///////// 2. Setup random-access-storage functions
+    ///////// 3. Setup random-access-storage functions
 
     /** @type {ConstructorParameters<typeof CoreManager>[0]['storage']} */
     const coreManagerStorage = (name) =>
@@ -145,7 +184,7 @@ export class MapeoProject extends TypedEmitter {
     const indexerStorage = (name) =>
       coreStorage(path.join(INDEXER_STORAGE_FOLDER_NAME, name))
 
-    ///////// 3. Create instances
+    ///////// 4. Create instances
 
     this.#coreManager = new CoreManager({
       projectSecretKey,
@@ -158,17 +197,7 @@ export class MapeoProject extends TypedEmitter {
     })
 
     this.#indexWriter = new IndexWriter({
-      tables: [
-        observationTable,
-        trackTable,
-        presetTable,
-        fieldTable,
-        coreOwnershipTable,
-        roleTable,
-        deviceInfoTable,
-        iconTable,
-        translationTable,
-      ],
+      tables: indexedTables,
       sqlite: this.#sqlite,
       getWinner,
       mapDoc: (doc, version) => {
@@ -347,7 +376,7 @@ export class MapeoProject extends TypedEmitter {
       dataType: this.#dataTypes.translation,
     })
 
-    ///////// 4. Replicate local peers automatically
+    ///////// 5. Replicate local peers automatically
 
     // Replicate already connected local peers
     for (const peer of localPeers.peers) {

test-e2e/version-upgrade.js

@@ -0,0 +1,10 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+
+test('migrations pick up values that were previously not understood', async () => {
+  // TODO(evanhahn) Write this test
+  // Receive an observation with a new field, `foo`
+  // Get the bytes, add it to the core, see that it's in SQLite without a `foo` column
+  // Do a migration where `foo` is added
+  // Reload the project and see that `foo` is now there
+})

test/fixtures/schemas/one_table/0000_burly_rockslide.sql

@@ -0,0 +1,3 @@
+CREATE TABLE `foo` (
+	`bar` text
+);

test/fixtures/schemas/one_table/meta/0000_snapshot.json

@@ -0,0 +1,30 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "edd7be22-bd12-4abd-877b-2bf62df76631",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "foo": {
+      "name": "foo",
+      "columns": {
+        "bar": {
+          "name": "bar",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

test/fixtures/schemas/one_table/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1716500546941,
+      "tag": "0000_burly_rockslide",
+      "breakpoints": true
+    }
+  ]
+}

test/fixtures/schemas/one_table/one_table.js

@@ -0,0 +1,3 @@
+import { text, sqliteTable } from 'drizzle-orm/sqlite-core'
+
+export const foo = sqliteTable('foo', { bar: text('bar') })

test/fixtures/schemas/two_tables/0000_burly_rockslide.sql

@@ -0,0 +1,3 @@
+CREATE TABLE `foo` (
+	`bar` text
+);

test/fixtures/schemas/two_tables/0001_furry_king_cobra.sql

@@ -0,0 +1,3 @@
+CREATE TABLE `baz` (
+	`qux` text
+);

test/fixtures/schemas/two_tables/meta/0000_snapshot.json

@@ -0,0 +1,30 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "edd7be22-bd12-4abd-877b-2bf62df76631",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "foo": {
+      "name": "foo",
+      "columns": {
+        "bar": {
+          "name": "bar",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

test/fixtures/schemas/two_tables/meta/0001_snapshot.json

@@ -0,0 +1,46 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "9730c082-6ea7-4355-99b6-dd9eb12f9b82",
+  "prevId": "edd7be22-bd12-4abd-877b-2bf62df76631",
+  "tables": {
+    "baz": {
+      "name": "baz",
+      "columns": {
+        "qux": {
+          "name": "qux",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "foo": {
+      "name": "foo",
+      "columns": {
+        "bar": {
+          "name": "bar",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

test/fixtures/schemas/two_tables/meta/_journal.json

@@ -0,0 +1,20 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1716500546941,
+      "tag": "0000_burly_rockslide",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "5",
+      "when": 1716501178989,
+      "tag": "0001_furry_king_cobra",
+      "breakpoints": true
+    }
+  ]
+}

test/fixtures/schemas/two_tables/two_tables.js

@@ -0,0 +1,5 @@
+import { text, sqliteTable } from 'drizzle-orm/sqlite-core'
+
+export const foo = sqliteTable('foo', { bar: text('bar') })
+
+export const baz = sqliteTable('baz', { qux: text('qux') })