Merge pull request #277 from orbitdb/feat/load-opts-fixes3

Improve entry fetching algorithm
orbitdb-archive · Nov 18, 2019 · 536f747 · 536f747
2 parents a742011 + 5076bef
commit 536f747
Show file tree

Hide file tree

Showing 15 changed files with 697 additions and 714 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@ node_modules/
 coverage/
 ipfs/
 ipfs-log/
+orbitdb/
 dist/*.js.map
 examples/browser/bundle.js
 test/keystore/

diff --git a/Makefile b/Makefile
@@ -15,6 +15,7 @@ build: test
 clean:
 	rm -rf ipfs/
 	rm -rf ipfs-log-benchmarks/
+	rm -rf orbitdb/
 	rm -rf node_modules/
 	rm -rf coverage/
 	rm -rf test/keystore/

diff --git a/benchmarks/benchmark-append.js b/benchmarks/benchmark-append.js
@@ -56,7 +56,7 @@ let run = (() => {
     // const memstore = new MemStore()
     // ipfs.dag.put = memstore.put.bind(memstore)
     // ipfs.dag.get = memstore.get.bind(memstore)
-    const signingKeysPath = './benchmarks/ipfs-log-benchmarks/keys1'
+    const signingKeysPath = './test/fixtures/keys/signing-keys'
     const store = await storage.createStore(signingKeysPath)
     const keystore = new Keystore(store)
     const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })

diff --git a/benchmarks/benchmark-join2.js b/benchmarks/benchmark-join2.js
@@ -9,6 +9,7 @@ const Keystore = require('orbit-db-keystore')
 
 const leveldown = require('leveldown')
 const storage = require('orbit-db-storage-adapter')(leveldown)
+
 // State
 let ipfs
 let log1, log2
@@ -60,14 +61,11 @@ let run = (() => {
     // ipfs.dag.put = memstore.put.bind(memstore)
     // ipfs.dag.get = memstore.get.bind(memstore)
 
-    const signingKeysPath1 = './benchmarks/ipfs-log-benchmarks/keys1'
-    const signingKeysPath2 = './benchmarks/ipfs-log-benchmarks/keys2'
-    const store1 = await storage.createStore(signingKeysPath1)
-    const store2 = await storage.createStore(signingKeysPath2)
-    const keystore1 = new Keystore(store1)
-    const keystore2 = new Keystore(store2)
-    const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore: keystore1 })
-    const identity2 = await IdentityProvider.createIdentity({ id: 'userB', keystore: keystore2 })
+    const signingKeysPath = './test/fixtures/keys/signing-keys'
+    const store = await storage.createStore(signingKeysPath)
+    const keystore = new Keystore(store)
+    const identity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
+    const identity2 = await IdentityProvider.createIdentity({ id: 'userB', keystore })
 
     log1 = new Log(ipfs, identity, { logId: 'A' })
     log2 = new Log(ipfs, identity2, { logId: 'A' })
@@ -97,7 +95,7 @@ let run = (() => {
     console.log('Entry size:', Buffer.from(JSON.stringify(l2.heads)).length, 'bytes')
     // console.log(log2.heads)
     console.log('log length:', log2.values.length)
-    console.log(log2.values.map(e => e.payload))
+    // console.log(log2.values.map(e => e.payload))
   })
 })()
 

diff --git a/dist/ipfslog.min.js b/dist/ipfslog.min.js
diff --git a/examples/browser/loader-visualization.html b/examples/browser/loader-visualization.html
@@ -67,19 +67,19 @@
       <button id="startButton">Load!</button>
     </div>
     <h1>ipfs-log</h1>
-    <h2>loading simulation</h2>
+    <h2>loading visualization</h2>
     <a href="https://github.com/orbitdb/ipfs-log">https://github.com/orbitdb/ipfs-log</a>
     <div id="log">
       <svg id="log-svg" width=10000 height=4 shape-rendering="optimizeSpeed">
         <rect id="totalPercentage"></rect>
+        <g id="loadProgressPercentage"></g>
         <rect id="createProgressPercentage"></rect>
-        <rect id="loadProgressPercentage"></rect>
       </svg>
     </div>
     <pre id="output-create"></pre>
     <pre id="output-load"></pre>
     <script type="text/javascript" src="bundle.js" charset="utf-8"></script>
-    <script src="https://d3js.org/d3.v4.min.js"></script>
+    <script src="https://d3js.org/d3.v5.min.js"></script>
 
     <script>
       const outputLoadingElm = document.getElementById('output-load')
@@ -167,36 +167,33 @@ <h2>loading simulation</h2>
           const hash = await log.append('hello' + i, refCount)
           if (i % 10 === 0) {
             await onLogCreateUpdated(log, i, hash)
-            // if (delay > 0)
-            //   await sleep(1)
           }
         }
 
         const lt2 = new Date().getTime()
         setTimeout(() => {
-          // outputElm.innerHTML += "Creating took: " + (lt2 - st) + " ms"
           outputElm.innerHTML = `Creating a log... ${count} / ${count} (${(lt2 - st)} ms)`
         }, 100)
-        // outputLoadingElm.innerHTML += "Creating took: " + (lt2 - st) + " ms"
 
-        // outputLoadingElm.innerHTML = "Loading a log..."
         window.onLogCreated()
+
         const lt1 = new Date().getTime()
         const result = await EntryIO.fetchAll(
-          ipfs, 
+          ipfs,
           log.heads.map(e => e.hash),
           {
             length: -1,
             exclude: [],
             timeout: 20000,
             onProgressCallback: window.onDataUpdated,
-            concurrency: null
+            onStartProgressCallback: null,
+            concurrency,
+            delay
           }
         )
         const et = new Date().getTime()
         setTimeout(() => {
           outputLoadingElm.innerHTML = "Loading a log...  " + result.length + " / " + log.length + " (" + (et - lt1) + " ms)<br>"
-          // outputLoadingElm.innerHTML += "<br>Loading took: " + (et - lt1) + " ms<br>"
           outputLoadingElm.innerHTML += "Duration: " + (et - st) + " ms<br>"
           startButtonElm.addEventListener('click', run)
           startButtonElm.disabled = false
@@ -210,9 +207,13 @@ <h2>loading simulation</h2>
       const margin = 0
 
       let svgContainer
+      let data = Array(count).fill(0)
 
       const resetRendering = async () => {
+        logLength = 0
+        data = []
         const w = logElm.clientWidth
+
         svgContainer = d3.select("#log-svg")
           .attr("width", w)
           .attr("height", sizeY)
@@ -222,73 +223,73 @@ <h2>loading simulation</h2>
           .attr("width", w)
           .style("fill", "rgb(228, 228, 228)")
 
-        svgContainer.select("#createProgressPercentage")
-          .attr("width", 0)
         svgContainer.select("#loadProgressPercentage")
-          .attr("width", 0)
+          .selectAll("rect")
+          .remove()
 
-        return new Promise((resolve) => {
-          setTimeout(() => resolve(), 0)
-        })
+        return new Promise((resolve) => setTimeout(resolve, 0))
       }
 
       window.onresize = (event) => {
         sizeX = logElm.clientWidth / count
       }
 
-      let _renderIndex = {}
-      let circleElements = []
-      let d = new Array(count)
-
-      let loadProgressRenderInterval
       let createProgressRenderInterval
 
       let logLength = 0
       window.onLogCreateUpdated = (log, progress, entry) => {
         logLength = progress
-        return new Promise((resolve) => {
-          setTimeout(() => resolve(), 0)
-        })
-      }
+          outputElm.innerHTML = "Creating a log... " + logLength + " / " + count + "<br>"
 
-      window.onLogCreationStarted = () => {
-        createProgressRenderInterval = setInterval(() => {
-            outputElm.innerHTML = "Creating a log... " + logLength + " / " + count + "<br>"
+          svgContainer.select("#createProgressPercentage")
+            .attr("height", sizeY)
+            .attr("width", (logLength / (count / logElm.clientWidth)))
+            .style("fill", "rgba(164, 164, 164)")
 
-            svgContainer.select("#createProgressPercentage")
-              .attr("height", sizeY)
-              .attr("width", (logLength / (count / logElm.clientWidth)))
-              .style("fill", "rgba(112, 164, 112)")
+        return new Promise((resolve) => setTimeout(resolve, 0))
+      }
 
-            if (logLength === count) {
-              clearInterval(createProgressRenderInterval)
-              return
-            }
-        }, 100)
+      window.onLogCreationStarted = () => {
+        logLength = 0
       }
 
       let loadLength = 0
       window.onDataUpdated = (hash, entry, resultLength, result, queue) => {
         loadLength = resultLength
-        return new Promise((resolve) => {
-          setTimeout(() => resolve(), 0)
-        })
+        data[data.length - entry.clock.time] = 1
+
+        const w = logElm.clientWidth
+        count = parseInt(countElm.value)
+
+        svgContainer.select("#loadProgressPercentage")
+          .selectAll("rect")
+          .data(data)
+          .join(
+            enter => enter.append("rect")
+              .attr("x", (d, i) => i * (w / count))
+              .attr("width", (w / count))
+              .attr("height", sizeY)
+              .style("fill", "rgb(164, 164, 164)"),
+            update => update
+              .attr("x", (d, i) => i * (w / count))
+              .attr("width", (w / count))
+              .style("fill", d => d === 0 ? "rgb(164, 164, 164)" : "rgb(112, 228, 112)"),
+            exit => exit
+              .remove(),
+          )
+
+        outputLoadingElm.innerHTML = "Loading a log...  " + loadLength + " / " + count
+
+        return new Promise((resolve) => setTimeout(resolve, 0))
       }
 
       window.onLogCreated = () => {
-        loadProgressRenderInterval = setInterval(() => {
-          outputLoadingElm.innerHTML = "Loading a log...  " + loadLength + " / " + count
-
-          svgContainer.select("#loadProgressPercentage")
-            .attr("height", sizeY)
-            .attr("width", (loadLength / (count / logElm.clientWidth)))
-            .style("fill", "rgb(112, 228, 112)")
+        svgContainer.select("#createProgressPercentage")
+          .attr("width", 0)
+          .attr("height", 0)
 
-          if (loadLength === count) {
-            clearInterval(loadProgressRenderInterval)
-            return
-          }
-        }, 100)
+        count = parseInt(countElm.value)
+        data = Array(count).fill(0)
       }
     </script>
   </body>