Optimize loading

.gitignore

@@ -2,12 +2,14 @@ node_modules/
 coverage/
 ipfs/
 ipfs-log/
+orbitdb/
 dist/*.js.map
 examples/browser/bundle.js
 test/keystore/
 test-keys/
 test/browser/bundle.js
 test/browser/bundle.js.map
+ipfs-log-benchmarks/
 examples/browser/bundle.js.map
 examples/browser/ipfs.min.js
 examples/browser/ipfslog.min.js

Makefile

@@ -15,6 +15,7 @@ build: test
 clean:
 	rm -rf ipfs/
 	rm -rf ipfs-log-benchmarks/
+	rm -rf orbitdb/
 	rm -rf node_modules/
 	rm -rf coverage/
 	rm -rf test/keystore/

benchmarks/benchmark-append.js

@@ -7,9 +7,6 @@ const Log = require('../src/log')
 const IdentityProvider = require('orbit-db-identity-provider')
 const Keystore = require('orbit-db-keystore')
 
-const leveldown = require('leveldown')
-const storage = require('orbit-db-storage-adapter')(leveldown)
-
 // State
 let ipfs
 let log
@@ -21,7 +18,7 @@ let queriesPerSecond = 0
 let lastTenSeconds = 0
 
 const queryLoop = async () => {
-  await log.append(totalQueries.toString())
+  await log.append(totalQueries.toString(), 1, false)
   totalQueries++
   lastTenSeconds++
   queriesPerSecond++
@@ -56,9 +53,7 @@ let run = (() => {
     // const memstore = new MemStore()
     // ipfs.dag.put = memstore.put.bind(memstore)
     // ipfs.dag.get = memstore.get.bind(memstore)
-    const signingKeysPath = './test/fixtures/keys'
-    const store = await storage.createStore(signingKeysPath)
-    const keystore = new Keystore(store)
+    const keystore = new Keystore('./ipfs-log-benchmarks/keys/')
     const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
 
     log = new Log(ipfs, identity, { logId: 'A' })

benchmarks/benchmark-from-entry-hash.js

@@ -5,7 +5,7 @@ const IPFSRepo = require('ipfs-repo')
 const DatastoreLevel = require('datastore-level')
 const Log = require('../src/log')
 const IdentityProvider = require('orbit-db-identity-provider')
-
+const Keystore = require('orbit-db-keystore')
 // State
 let ipfs
 let log
@@ -27,7 +27,7 @@ let run = (() => {
   }
 
   ipfs = new IPFS({
-    repo: new IPFSRepo('./ipfs-log-benchmarks/fromEntryHash/ipfs', repoConf),
+    repo: new IPFSRepo('./ipfs-log-benchmarks/ipfs', repoConf),
     start: false,
     EXPERIMENTAL: {
       pubsub: false,
@@ -42,8 +42,8 @@ let run = (() => {
 
   ipfs.on('ready', async () => {
     // Create a log
-    const signingKeysPath = './test/fixtures/keys'
-    const identity = await IdentityProvider.createIdentity({ id: 'userA', signingKeysPath })
+    const keystore = new Keystore('./ipfs-log-benchmarks/keys/')
+    const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
 
     log = new Log(ipfs, identity, { logId: 'A' })
 

benchmarks/benchmark-join.js

@@ -5,6 +5,7 @@ const IPFSRepo = require('ipfs-repo')
 const DatastoreLevel = require('datastore-level')
 const Log = require('../src/log')
 const IdentityProvider = require('orbit-db-identity-provider')
+const Keystore = require('orbit-db-keystore')
 
 // State
 let ipfs
@@ -62,10 +63,10 @@ let run = (() => {
     // const memstore = new MemStore()
     // ipfs.dag.put = memstore.put.bind(memstore)
     // ipfs.dag.get = memstore.get.bind(memstore)
+    const keystore = new Keystore('./ipfs-log-benchmarks/keys/')
 
-    const signingKeysPath = './test/fixtures/keys'
-    const identity = await IdentityProvider.createIdentity({ id: 'userA', signingKeysPath })
-    const identity2 = await IdentityProvider.createIdentity({ id: 'userB', signingKeysPath })
+    const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
+    const identity2 = await IdentityProvider.createIdentity({ id: 'userB', keystore })
 
     log1 = new Log(ipfs, identity, { logId: 'A' })
     log2 = new Log(ipfs, identity2, { logId: 'A' })

benchmarks/benchmark-join2.js

@@ -0,0 +1,96 @@
+'use strict'
+
+const IPFS = require('ipfs')
+const IPFSRepo = require('ipfs-repo')
+const DatastoreLevel = require('datastore-level')
+const Log = require('../src/log')
+const IdentityProvider = require('orbit-db-identity-provider')
+const Keystore = require('orbit-db-keystore')
+
+// State
+let ipfs
+let log1, log2
+
+// Metrics
+let totalQueries = 0
+const queryLoop = async () => {
+  try {
+    await Promise.all([
+      log1.append('a' + totalQueries),
+      log2.append('b' + totalQueries)
+    ])
+
+    await log1.join(log2)
+    await log2.join(log1)
+    totalQueries++
+    setImmediate(queryLoop)
+  } catch (e) {
+    console.error(e)
+    process.exit(0)
+  }
+}
+
+let run = (() => {
+  console.log('Starting benchmark...')
+
+  const repoConf = {
+    storageBackends: {
+      blocks: DatastoreLevel
+    }
+  }
+
+  ipfs = new IPFS({
+    repo: new IPFSRepo('./ipfs-log-benchmarks/ipfs', repoConf),
+    start: false,
+    EXPERIMENTAL: {
+      pubsub: true
+    }
+  })
+
+  ipfs.on('error', (err) => {
+    console.error(err)
+    process.exit(1)
+  })
+
+  ipfs.on('ready', async () => {
+    // Use memory store to test without disk IO
+    // const memstore = new MemStore()
+    // ipfs.dag.put = memstore.put.bind(memstore)
+    // ipfs.dag.get = memstore.get.bind(memstore)
+    const keystore = new Keystore('./benchmarks/ipfs-log-benchmarks/keys')
+    const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
+    const identity2 = await IdentityProvider.createIdentity({ id: 'userB', keystore })
+
+    log1 = new Log(ipfs, identity, { logId: 'A' })
+    log2 = new Log(ipfs, identity2, { logId: 'A' })
+
+    const amount = 10000
+    console.log('log length:', amount)
+
+    console.log('Writing log...')
+    const st3 = new Date().getTime()
+    for (let i = 0; i < amount; i++) {
+      await log1.append('a' + i, 64)
+    }
+    const et3 = new Date().getTime()
+    console.log('write took', (et3 - st3), 'ms')
+
+    console.log('Joining logs...')
+    const st = new Date().getTime()
+    await log2.join(log1)
+    const et = new Date().getTime()
+    console.log('join took', (et - st), 'ms')
+
+    console.log('Loading log...')
+    const st2 = new Date().getTime()
+    const l2 = await Log.fromEntryHash(ipfs, identity, log1.heads[0].hash, { logId: 'A' })
+    const et2 = new Date().getTime()
+    console.log('load took', (et2 - st2), 'ms')
+    console.log('Entry size:', Buffer.from(JSON.stringify(l2.heads)).length, 'bytes')
+    // console.log(log2.heads)
+    console.log('log length:', log2.values.length)
+    // console.log(log2.values.map(e => e.payload))
+  })
+})()
+
+module.exports = run

benchmarks/browser/benchmark-append-signed.html

@@ -19,7 +19,6 @@ <h2>Results</h2>
     <pre id="output"></pre>
 
     <script type="text/javascript" src="../../dist/ipfslog.min.js" charset="utf-8"></script>
-    <script type="text/javascript" src="../../node_modules/orbit-db-keystore/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/orbit-db-identity-provider/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/ipfs/dist/index.min.js" charset="utf-8"></script>
 
@@ -32,20 +31,21 @@ <h2>Results</h2>
       let queriesPerSecond = 0
       let lastTenSeconds = 0
 
-      const queryLoop = () => {
-        return log.append(totalQueries.toString())
-          .then((res) => {
-            totalQueries ++
-            lastTenSeconds ++
-            queriesPerSecond ++
-            setImmediate(queryLoop)
-          })
-          .catch((e) => console.error(e))
+      const queryLoop = async () => {
+        try {
+          await log.append(totalQueries.toString())
+          totalQueries ++
+          lastTenSeconds ++
+          queriesPerSecond ++
+          setImmediate(queryLoop)
+        } catch (e) {
+          console.error(e)
+        }
       }
 
       let run = (() => {
         ipfs = new Ipfs({
-          repo: './ipfs-log/examples/browser/benchmark-append/0.34.4',
+          repo: './ipfs-log/examples/browser/benchmark-append-signed',
           start: false,
           EXPERIMENTAL: {
             pubsub: false,
@@ -72,9 +72,9 @@ <h2>Results</h2>
             queriesPerSecond = 0
           }, 1000)
 
-          const signingKeysPath = '../test/fixtures/keys'
-          const identity = await Identities.createIdentity({ id: 'userA', signingKeysPath })
+          const identity = await Identities.createIdentity({ id: 'userA' })
           log = new Log(ipfs, identity, { logId: 'browser-benchmark-append-signed' })
+
           queryLoop()
         })
 

benchmarks/browser/benchmark-join-signed.html

@@ -28,7 +28,6 @@ <h2>Results</h2>
     <pre id="output"></pre>
 
     <script type="text/javascript" src="../../dist/ipfslog.min.js" charset="utf-8"></script>
-    <script type="text/javascript" src="../../node_modules/orbit-db-keystore/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/orbit-db-identity-provider/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/ipfs/dist/index.min.js" charset="utf-8"></script>
 
@@ -42,26 +41,25 @@ <h2>Results</h2>
       let lastTenSeconds = 0
       let log1, log2
 
-      const { AccessController } = Log
-      const queryLoop = () => {
-        const add1 = log1.append("a" + totalQueries)
-        const add2 = log2.append("b" + totalQueries)
-
-        Promise.all([add1, add2])
-          .then((res) => {
-            log1.join(log2)
-            log2.join(log1)
-            totalQueries ++
-            lastTenSeconds ++
-            queriesPerSecond ++
-            setImmediate(queryLoop)
-          })
-          .catch((e) => console.error(e))
+      const queryLoop = async () => {
+        try {
+          const add1 = log1.append("a" + totalQueries)
+          const add2 = log2.append("b" + totalQueries)
+          await Promise.all([add1, add2])
+          log1.join(log2)
+          log2.join(log1)
+          totalQueries ++
+          lastTenSeconds ++
+          queriesPerSecond ++
+          setImmediate(queryLoop)
+        } catch(e) {
+          console.error(e)
+        }
       }
 
       let run = (() => {
         ipfs = new Ipfs({
-          repo: './ipfs-log/examples/browser/benchmark-join/new/0.33.1',
+          repo: './ipfs-log/examples/browser/benchmark-join-signed',
           start: false,
           EXPERIMENTAL: {
             pubsub: false,
@@ -88,9 +86,7 @@ <h2>Results</h2>
             queriesPerSecond = 0
           }, 1000)
 
-          const testKeyPath = '../test/fixtures/keys'
-          const keystore = Keystore.create(testKeyPath)
-          const identity = await Identities.createIdentity({ id: 'userA', keystore })
+          const identity = await Identities.createIdentity({ id: 'userA' })
 
           log1 = new Log(ipfs, identity, { logId: 'browser-benchmark-join-signed' })
           log2 = new Log(ipfs, identity, { logId: 'browser-benchmark-join-signed' })

examples/browser/browser.html

@@ -7,14 +7,13 @@ <h1>ipfs-log example</h1>
     <pre id="output"></pre>
 
     <script type="text/javascript" src="../../dist/ipfslog.min.js" charset="utf-8"></script>
-    <script type="text/javascript" src="../../node_modules/orbit-db-keystore/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/orbit-db-identity-provider/dist/index-browser.min.js" charset="utf-8"></script>
     <script type="text/javascript" src="../../node_modules/ipfs/dist/index.min.js" charset="utf-8"></script>
 
     <script type="text/javascript">
-      const dataPath = './ipfs-log/examples/browser/ipfs-0.34.4'
+      const dataPath = './ipfs-log/examples/browser/ipfs/browser.html'
       const ipfs = new Ipfs({
-        repo: dataPath + '/index.js',
+        repo: dataPath,
         start: false,
         EXPERIMENTAL: {
           pubsub: true

examples/browser/index.js

@@ -4,9 +4,9 @@ const IPFS = require('ipfs')
 const IdentityProvider = require('orbit-db-identity-provider')
 const Log = require('../../src/log')
 
-const dataPath = './ipfs-log/examples/browser/ipfs-0.34.4'
+const dataPath = './ipfs-log/examples/browser/ipfs/index.html'
 const ipfs = new IPFS({
-  repo: dataPath + '/index.js',
+  repo: dataPath,
   start: false,
   EXPERIMENTAL: {
     pubsub: true