Show progress bar while geometry cache is being loaded

CMakeLists.txt

@@ -2,10 +2,24 @@ cmake_minimum_required (VERSION 2.8)
 
 project (qlever-mapui)
 
+include(CMakePrintHelpers)
+
+cmake_print_variables(CMAKE_BUILD_TYPE)
+
+# only change first char
 if (CMAKE_BUILD_TYPE)
-	string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+	string(SUBSTRING ${CMAKE_BUILD_TYPE} 0 1 FIRST_CHAR)
+	string(TOUPPER ${FIRST_CHAR} FIRST_CHAR)
+	string(REGEX REPLACE "^.(.*)" "${FIRST_CHAR}\\1" CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}")
 endif()
 
+# old solution
+#if (CMAKE_BUILD_TYPE)
+#	string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+#endif()
+
+cmake_print_variables(CMAKE_BUILD_TYPE)
+
 enable_testing()
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")

src/qlever-petrimaps/GeomCache.cpp

@@ -10,6 +10,7 @@
 #include <iostream>
 #include <parallel/algorithm>
 #include <sstream>
+#include <atomic>
 
 #include "qlever-petrimaps/GeomCache.h"
 #include "qlever-petrimaps/Misc.h"
@@ -116,6 +117,8 @@ size_t GeomCache::writeCbCount(void* contents, size_t size, size_t nmemb,
 
 // _____________________________________________________________________________
 void GeomCache::parse(const char* c, size_t size) {
+  _loadStatusStage = _LoadStatusStages::Parse;
+
   const char* start = c;
   while (c < start + size) {
     if (_raw.size() < 10000) _raw.push_back(*c);
@@ -312,8 +315,7 @@ void GeomCache::parse(const char* c, size_t size) {
               LOG(INFO) << "[GEOMCACHE] "
                         << "@ row " << _curRow << " (" << std::fixed
                         << std::setprecision(2)
-                        << (static_cast<double>(_curRow) /
-                            static_cast<double>(_totalSize) * 100)
+                        << getLoadStatusPercent()
                         << "%, " << _pointsFSize << " points, " << _linesFSize
                         << " (open) polygons)";
             }
@@ -339,8 +341,40 @@ void GeomCache::parse(const char* c, size_t size) {
   }
 }
 
+double GeomCache::getLoadStatusPercent() {
+  /*
+  There are 2 loading stages: Parse, afterwards ParseIds.
+  Because ParseIds is usually pretty short, we merge the progress of both stages to one total progress.
+  Progress is calculated by _curRow / _totalSize, which are handled by each stage individually.
+  */
+  if (_totalSize == 0) {
+    return 0.0;
+  }
+
+  float parsePercent = 95.0f;
+  float parseIdsPercent = 5.0f;
+  float totalPercent = 0.0f;
+  switch(_loadStatusStage) {
+    case _LoadStatusStages::Parse:
+      totalPercent = std::atomic<size_t>(_curRow) / static_cast<double>(_totalSize) * parsePercent;
+      break;
+    case _LoadStatusStages::ParseIds:
+      totalPercent = parsePercent;
+      totalPercent += std::atomic<size_t>(_curRow) / static_cast<double>(_totalSize) * parseIdsPercent;
+      break;
+  }
+
+  return totalPercent;
+}
+
+int GeomCache::getLoadStatusStage() {
+  return _loadStatusStage;
+}
+
 // _____________________________________________________________________________
 void GeomCache::parseIds(const char* c, size_t size) {
+  _loadStatusStage = _LoadStatusStages::ParseIds;
+
   size_t lastQid = -1;
   for (size_t i = 0; i < size; i++) {
     if (_raw.size() < 10000) _raw.push_back(c[i]);

src/qlever-petrimaps/GeomCache.h

@@ -89,6 +89,9 @@ class GeomCache {
     return id + 1 < _lines.size() ? _lines[id + 1] : _linePoints.size();
   }
 
+  double getLoadStatusPercent();
+  int getLoadStatusStage();
+
  private:
   std::string _backendUrl;
   CURL* _curl;
@@ -97,6 +100,9 @@ class GeomCache {
   ID _curId;
   QLEVER_ID_TYPE _maxQid;
   size_t _curRow, _curUniqueGeom;
+
+  enum _LoadStatusStages {Parse = 1, ParseIds};
+  _LoadStatusStages _loadStatusStage;
 
   static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp);
   static size_t writeCbIds(void* contents, size_t size, size_t nmemb,

src/qlever-petrimaps/server/Server.cpp

@@ -81,6 +81,8 @@ util::http::Answer Server::handle(const util::http::Req& req, int con) const {
       a = handlePosReq(params);
     } else if (cmd == "/export") {
       a = handleExportReq(params, con);
+    } else if (cmd == "/loadstatus") {
+      a = handleLoadStatusReq(params);
     } else if (cmd == "/build.js") {
       a = util::http::Answer(
           "200 OK", std::string(build_js, build_js + sizeof build_js /
@@ -745,6 +747,7 @@ util::http::Answer Server::handleLoadReq(const Params& pars) const {
 
   LOG(INFO) << "[SERVER] Queried backend is " << backend;
 
+  createCache(backend);
   loadCache(backend);
 
   auto answ = util::http::Answer("200 OK", "{}");
@@ -764,6 +767,7 @@ util::http::Answer Server::handleQueryReq(const Params& pars) const {
   LOG(INFO) << "[SERVER] Queried backend is " << backend;
   LOG(INFO) << "[SERVER] Query is:\n" << query;
 
+  createCache(backend);
   loadCache(backend);
 
   std::string queryId = backend + "$" + query;
@@ -1108,6 +1112,22 @@ util::http::Answer Server::handleExportReq(const Params& pars, int sock) const {
   return aw;
 }
 
+util::http::Answer Server::handleLoadStatusReq(const Params& pars) const {
+  if (pars.count("backend") == 0 || pars.find("backend")->second.empty())
+    throw std::invalid_argument("No backend (?backend=) specified.");
+  auto backend = pars.find("backend")->second;
+  createCache(backend);
+  std::shared_ptr<GeomCache> cache = _caches[backend];
+  double loadStatusPercent = cache->getLoadStatusPercent();
+  int loadStatusStage = cache->getLoadStatusStage();
+
+  std::stringstream json;
+  json << "{\"percent\": " << loadStatusPercent << ", \"stage\": " << loadStatusStage << "}";
+  util::http::Answer ans = util::http::Answer("200 OK", json.str());
+
+  return ans;
+}
+
 // _____________________________________________________________________________
 void Server::drawPoint(std::vector<uint32_t>& points,
                        std::vector<double>& points2, int px, int py, int w,
@@ -1140,9 +1160,7 @@ std::string Server::getSessionId() const {
   return std::to_string(d(rng));
 }
 
-// _____________________________________________________________________________
-void Server::loadCache(const std::string& backend) const {
-  std::shared_ptr<Requestor> reqor;
+void Server::createCache(const std::string& backend) const {
   std::shared_ptr<GeomCache> cache;
 
   {
@@ -1154,6 +1172,12 @@ void Server::loadCache(const std::string& backend) const {
       _caches[backend] = cache;
     }
   }
+}
+
+// _____________________________________________________________________________
+void Server::loadCache(const std::string& backend) const {
+  //std::shared_ptr<Requestor> reqor;
+  std::shared_ptr<GeomCache> cache = _caches[backend];
 
   try {
     cache->load(_cacheDir);

src/qlever-petrimaps/server/Server.h

@@ -40,7 +40,9 @@ class Server : public util::http::Handler {
   util::http::Answer handleLoadReq(const Params& pars) const;
 
   util::http::Answer handleExportReq(const Params& pars, int sock) const;
+  util::http::Answer handleLoadStatusReq(const Params& pars) const;
 
+  void createCache(const std::string& backend) const;
   void loadCache(const std::string& backend) const;
 
   void clearSession(const std::string& id) const;

web/index.html

@@ -16,19 +16,28 @@
    <body>
       <main>
          <div id='m'></div>
-		 <div id='ex'>
-			 <button id='ex-csv'>Export as CSV</button>
-			 <button id='ex-tsv'>Export as TSV</button>
-			 <button id='ex-geojson'>Export as GeoJSON</button>
-		 </div>
-		 <div id='stats'>
-		 </div>
+		   <div id='ex'>
+			   <button id='ex-csv'>Export as CSV</button>
+			   <button id='ex-tsv'>Export as TSV</button>
+			   <button id='ex-geojson'>Export as GeoJSON</button>
+		   </div>
+		   <div id='stats'></div>
       </main>
-	  <div id='msg'>
-		  <div id='msg-inner'>Loading results from QLever...</div>
-		  <div id='msg-inner-desc'></div>
-        <div id="loader"></div>
-	  </div>
+	   <div id='msg'>
+		   <div id='msg-inner'>Loading results from QLever</div>
+		   <div id='msg-inner-desc'></div>
+
+         <div id="load">
+            <div id="load-stage">Parsing geometry... (1/2)</div>
+            <div id="load-status">
+               <div id="load-bar"></div>
+               <div id="load-percent">0.00%</div>
+               <div id="load-spinner"></div>
+            </div>
+         </div>
+      </div>
+
+	   </div>
       <script src='build.js' defer></script>
    </body>
 </html>

web/script.js

@@ -1,26 +1,29 @@
-var sessionId;
-var curGeojson;
-var curGeojsonId = -1;
+let sessionId;
+let curGeojson;
+let curGeojsonId = -1;
 
-var urlParams = new URLSearchParams(window.location.search);
-var qleverBackend = urlParams.get("backend");
-var query = urlParams.get("query");
+let urlParams = new URLSearchParams(window.location.search);
+let qleverBackend = urlParams.get("backend");
+let query = urlParams.get("query");
 
-var map = L.map('m', {
+// id of SetInterval to stop loadStatus requests on error or load finish
+let loadStatusIntervalId = -1;
+
+let map = L.map('m', {
     renderer: L.canvas(),
     preferCanvas: true
 }).setView([47.9965, 7.8469], 13);
 map.attributionControl.setPrefix('University of Freiburg');
 
-var layerControl = L.control.layers([], [], {collapsed:true, position: 'topleft'}).addTo(map);
+let layerControl = L.control.layers([], [], {collapsed:true, position: 'topleft'}).addTo(map);
 
-var osmLayer = L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
+let osmLayer = L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
     attribution: '&copy; <a rel="noreferrer" target="_blank" href="#">OpenStreetMap</a>',
     maxZoom: 19,
     opacity:0.9
 }).addTo(map);
 
-var genError = "<p>Session has been removed from cache.</p> <p> <a href='javascript:location.reload();'>Resend request</a></p>";
+let genError = "<p>Session has been removed from cache.</p> <p> <a href='javascript:location.reload();'>Resend request</a></p>";
 
 function openPopup(data) {
     if (data.length > 0) {
@@ -71,7 +74,7 @@ function openPopup(data) {
           // the result value as another. Reformat a bit, so that it looks nice in
           // an HTML table.
           let key = variable.substring(1);
-          if (row[i] == null) { row[i] = "---"; }
+          if (row[i] == null) { row[i] = "---" }
           let value = row[i].replace(/\\([()])/g, "$1")
                         .replace(/<((.*)\/(.*))>/,
                          "<a class=\"link\" href=\"$1\" target=\"_blank\">$3</a>")
@@ -116,14 +119,13 @@ function getGeoJsonLayer(geom) {
                     opacity: 1,
                     fillOpacity: 0.2
                 });}
-            })
+            });
 }
 
 function showError(error) {
     error = error.toString();
-    console.log(error);
     document.getElementById("msg").style.display = "block";
-    document.getElementById("loader").style.display = "none";
+    document.getElementById("load").style.display = "none";
     document.getElementById("msg-inner").style.color = "red";
     document.getElementById("msg-inner").style.fontSize = "20px";
     document.getElementById("msg-inner").innerHTML = error.split("\n")[0];
@@ -188,7 +190,7 @@ function loadMap(id, bounds, numObjects) {
 	layerControl.addBaseLayer(autoLayer, "Auto");
 
     map.on('click', function(e) {
-        const ll= e.latlng;
+        const ll = e.latlng;
         const pos = L.Projection.SphericalMercator.project(ll);
 
         const w = map.getPixelBounds().max.x - map.getPixelBounds().min.x;
@@ -227,15 +229,65 @@ function loadMap(id, bounds, numObjects) {
     });
 }
 
-console.log("Fetching results...")
-fetch('query' + window.location.search)
-  .then(response => {
-      if (!response.ok) return response.text().then(text => {throw new Error(text)});
-      return response;
+function updateLoad(stage, percent) {
+    const stageElem = document.getElementById("load-stage");
+    const barElem = document.getElementById("load-bar");
+    const percentElem = document.getElementById("load-percent");
+    switch (stage) {
+        case 1:
+            stageElem.innerHTML = "Parsing geometry... (1/2)";
+            break;
+        case 2:
+            stageElem.innerHTML = "Parsing geometry Ids... (2/2)";
+            break;
+    }
+    barElem.style.width = percent + "%";
+    percentElem.innerHTML = percent.toString() + "%";
+}
+
+function fetchResults() {
+    console.log("Fetching results...");
+
+    fetch('query' + window.location.search)
+    .then(response => {
+        if (!response.ok) return response.text().then(text => {throw new Error(text)});
+        return response;
+        })
+    .then(response => response.json())
+    .then(data => {
+        clearInterval(loadStatusIntervalId);
+        loadMap(data["qid"], data["bounds"], data["numobjects"]);
     })
-  .then(response => response.json())
-  .then(data => loadMap(data["qid"], data["bounds"], data["numobjects"]))
-  .catch(error => {showError(error);});
+    .catch(error => showError(error));
+}
+
+function fetchLoadStatusInterval(interval) {
+    fetchLoadStatus();
+    loadStatusIntervalId = setInterval(fetchLoadStatus, interval);
+}
+
+async function fetchLoadStatus() {
+    console.log("Fetching load status...");
+
+    fetch('loadstatus?backend=' + qleverBackend)
+    .then(response => {
+        if (!response.ok) return response.text().then(text => {throw new Error(text)});
+        return response;
+        })
+    .then(response => response.json())
+    .then(data => {
+        var stage = data["stage"];
+        var percent = parseFloat(data["percent"]).toFixed(2);
+        updateLoad(stage, percent);
+    })
+    .catch(error => {
+        showError(error);
+        clearInterval(loadStatusIntervalId);
+    });
+}
+
+fetchResults();
+fetchLoadStatusInterval(1000);
 
 document.getElementById("ex-geojson").onclick = function() {
     if (!sessionId) return;