feat: migration 10 to allow upgrading level in the browser

We use the [level](https://www.npmjs.com/package/level) module to supply
either [leveldown](http://npmjs.com/package/leveldown) or
[level-js](https://www.npmjs.com/package/level-js)
to [datastore-level](https://www.npmjs.com/package/datastore-level) depending
on if we're running under node or in the browser.

`[email protected]` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`
which includes the changes from
[Level/level-js#179](Level/level-js#179)
so `>5.x.x` requires all database keys/values to be Uint8Arrays and they
can no longer be strings.

We already store values as Uint8Arrays but our keys are strings, so here
we add a migration to converts all datastore keys to Uint8Arrays.

N.b. `leveldown` already does this conversion for us so this migration
only needs to run in the browser.

README.md

@@ -42,6 +42,11 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com
   - [Tests](#tests)
   - [Empty migrations](#empty-migrations)
   - [Migrations matrix](#migrations-matrix)
+  - [Migrations](#migrations)
+    - [7](#7)
+    - [8](#8)
+    - [9](#9)
+    - [10](#10)
 - [Developer](#developer)
   - [Module versioning notes](#module-versioning-notes)
 - [Contribute](#contribute)
@@ -268,6 +273,24 @@ This will create an empty migration with the next version.
 |                  8 | v0.48.0          |
 |                  9 | v0.49.0          |
 
+### Migrations
+
+#### 7
+
+This is the initial version of the datastore, inherited from go-IPFS in an attempt to maintain cross-compatibility between the two implementations.
+
+#### 8
+
+Blockstore keys are transformed into base32 representations of the multihash from the CID of the block.
+
+#### 9
+
+Pins were migrated from a DAG to a Datastore - see [ipfs/js-ipfs#2771](https://github.com/ipfs/js-ipfs/pull/2771)
+
+#### 10
+
+`[email protected]` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`.  This update requires a database migration to convert all string keys/values into buffers. Only runs in the browser, node is unaffected. See [Level/level-js#179](https://github.com/Level/level-js/pull/179)
+
 ## Developer
 
 ### Module versioning notes

migrations/index.js

@@ -16,5 +16,6 @@ module.exports = [
   Object.assign({version: 6}, emptyMigration),
   Object.assign({version: 7}, emptyMigration),
   require('./migration-8'),
-  require('./migration-9')
+  require('./migration-9'),
+  require('./migration-10')
 ]

migrations/migration-10/index.js

@@ -0,0 +1,157 @@
+'use strict'
+
+const {
+  createStore,
+  findLevelJs
+} = require('../../src/utils')
+const { Key } = require('interface-datastore')
+const fromString = require('uint8arrays/from-string')
+const toString = require('uint8arrays/to-string')
+
+async function keysToBinary (name, store, onProgress = () => {}) {
+  let db = findLevelJs(store)
+
+  // only interested in level-js
+  if (!db) {
+    onProgress(`${name} did not need an upgrade`)
+
+    return
+  }
+
+  onProgress(`Upgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: fromString(key), value: value }
+    ]
+  })
+}
+
+async function keysToStrings (name, store, onProgress = () => {}) {
+  let db = findLevelJs(store)
+
+  // only interested in level-js
+  if (!db) {
+    onProgress(`${name} did not need a downgrade`)
+
+    return
+  }
+
+  onProgress(`Downgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: toString(key), value: value }
+    ]
+  })
+}
+
+async function process (repoPath, repoOptions, onProgress, fn) {
+  const datastores = Object.keys(repoOptions.storageBackends)
+    .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore')
+    .map(name => ({
+      name,
+      store: createStore(repoPath, name, repoOptions)
+    }))
+
+  onProgress(0, `Migrating ${datastores.length} dbs`)
+  let migrated = 0
+
+  for (const { name, store } of datastores) {
+    await store.open()
+
+    try {
+      await fn(name, store, (message) => {
+        onProgress(parseInt((migrated / datastores.length) * 100), message)
+      })
+    } finally {
+      migrated++
+      store.close()
+    }
+  }
+
+  onProgress(100, `Migrated ${datastores.length} dbs`)
+}
+
+module.exports = {
+  version: 10,
+  description: 'Migrates datastore-level keys to binary',
+  migrate: (repoPath, repoOptions, onProgress = () => {}) => {
+    return process(repoPath, repoOptions, onProgress, keysToBinary)
+  },
+  revert: (repoPath, repoOptions, onProgress = () => {}) => {
+    return process(repoPath, repoOptions, onProgress, keysToStrings)
+  }
+}
+
+/**
+ * @typedef {Uint8Array|string} Key
+ * @typedef {Uint8Array} Value
+ * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation
+ *
+ * Uses the upgrade strategy from [email protected] - note we can't call the `.upgrade` command
+ * directly because it will be removed in [email protected] and we can't guarantee users will
+ * have migrated by then - e.g. they may jump from [email protected] straight to [email protected]
+ * so we have to duplicate the code here.
+ *
+ * @param {import('interface-datastore').Datastore} db
+ * @param {function (Key, Value): Operation[]} fn
+ */
+function withEach (db, fn) {
+  function batch (operations, next) {
+    const store = db.store('readwrite')
+    const transaction = store.transaction
+    let index = 0
+    let error
+
+    transaction.onabort = () => next(error || transaction.error || new Error('aborted by user'))
+    transaction.oncomplete = () => next()
+
+    function loop () {
+      var op = operations[index++]
+      var key = op.key
+
+      try {
+        var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key)
+      } catch (err) {
+        error = err
+        transaction.abort()
+        return
+      }
+
+      if (index < operations.length) {
+        req.onsuccess = loop
+      }
+    }
+
+    loop()
+  }
+
+  return new Promise((resolve, reject) => {
+    const it = db.iterator()
+    // raw keys and values only
+    it._deserializeKey = it._deserializeValue = (data) => data
+    next()
+
+    function next () {
+      it.next((err, key, value) => {
+        if (err || key === undefined) {
+          it.end((err2) => {
+            if (err2) {
+              reject(err2)
+              return
+            }
+
+            resolve()
+          })
+
+          return
+        }
+
+        batch(fn(key, value), next)
+      })
+    }
+  })
+}

migrations/migration-8/index.js

@@ -75,10 +75,10 @@ async function process (repoPath, repoOptions, onProgress, keyFunction) {
 module.exports = {
   version: 8,
   description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32',
-  migrate: (repoPath, repoOptions, onProgress) => {
+  migrate: (repoPath, repoOptions, onProgress = () => {}) => {
     return process(repoPath, repoOptions, onProgress, keyToMultihash)
   },
-  revert: (repoPath, repoOptions, onProgress) => {
+  revert: (repoPath, repoOptions, onProgress = () => {}) => {
     return process(repoPath, repoOptions, onProgress, keyToCid)
   }
 }

migrations/migration-9/index.js

@@ -135,10 +135,10 @@ async function process (repoPath, repoOptions, onProgress, fn) {
 module.exports = {
   version: 9,
   description: 'Migrates pins to datastore',
-  migrate: (repoPath, repoOptions, onProgress) => {
+  migrate: (repoPath, repoOptions, onProgress = () => {}) => {
     return process(repoPath, repoOptions, onProgress, pinsToDatastore)
   },
-  revert: (repoPath, repoOptions, onProgress) => {
+  revert: (repoPath, repoOptions, onProgress = () => {}) => {
     return process(repoPath, repoOptions, onProgress, pinsToDAG)
   }
 }

package.json

@@ -64,6 +64,8 @@
     "datastore-level": "^3.0.0",
     "it-all": "^1.0.2",
     "just-safe-set": "^2.1.0",
+    "level-5": "npm:level@^5.0.0",
+    "level-6": "npm:level@^6.0.0",
     "ncp": "^2.0.0",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.2"

src/repo/version.js

@@ -4,6 +4,7 @@ const repoInit = require('./init')
 const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors')
 const { VERSION_KEY, createStore } = require('../utils')
 const uint8ArrayFromString = require('uint8arrays/from-string')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 exports.getVersion = getVersion
 
@@ -28,7 +29,14 @@ async function getVersion (path, repoOptions) {
   const store = createStore(path, 'root', repoOptions)
   await store.open()
 
-  const version = parseInt(await store.get(VERSION_KEY))
+  let version = await store.get(VERSION_KEY)
+
+  if (version instanceof Uint8Array) {
+    version = uint8ArrayToString(version)
+  }
+
+  version = parseInt(version)
+
   await store.close()
 
   return version