Merge branch 'ad-freiburg:master' into master

.github/workflows/build.yml

@@ -83,8 +83,8 @@ jobs:
         run: mkdir build && cd build && cmake ..
       - name: make
         run: cd build && make
-  macos-12-build:
-    runs-on: macOS-12
+  macos-13-build:
+    runs-on: macOS-13
     steps:
       - name: Checkout repository code
         uses: actions/checkout@v2

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required (VERSION 3.5)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 
 project (qlever-petrimaps)
 

src/qlever-petrimaps/GeomCache.h

@@ -14,6 +14,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#include <chrono>
 
 #include "qlever-petrimaps/Misc.h"
 #include "util/geo/Geo.h"
@@ -55,6 +56,7 @@ class GeomCache {
   void requestPart(size_t offset);
 
   void requestIds();
+  void requestIdPart(size_t offset);
 
   void parse(const char*, size_t size);
   void parseIds(const char*, size_t size);
@@ -103,9 +105,10 @@ class GeomCache {
   QLEVER_ID_TYPE _maxQid;
   size_t _totalSize = 0;
   std::atomic<size_t> _curRow;
+  std::atomic<size_t> _curIdRow;
   size_t _curUniqueGeom;
 
-  enum _LoadStatusStages { Parse = 1, ParseIds, FromFile };
+  enum _LoadStatusStages { Parse = 1, ParseIds, FromFile, Finished };
   _LoadStatusStages _loadStatusStage = Parse;
 
   static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp);
@@ -124,15 +127,23 @@ class GeomCache {
 
   std::string queryUrl(std::string query, size_t offset, size_t limit) const;
 
-  util::geo::FPoint parsePoint(const std::string& a, size_t p) const;
+  util::geo::FPoint createPoint(const std::string& a, size_t p) const;
 
   static bool pointValid(const util::geo::FPoint& p);
   static bool pointValid(const util::geo::DPoint& p);
 
-  static util::geo::DLine parseLineString(const std::string& a, size_t p);
+  static util::geo::DLine createLineString(const std::string& a, size_t p);
+
+	size_t parsePolygon(const std::string& str, size_t p, size_t end, size_t* i);
+
+	size_t parseMultiPoint(const std::string &str, size_t p, size_t end, size_t* i);
+	size_t parseMultiLineString(const std::string &str, size_t p, size_t end, size_t* i);
+  size_t parseMultiPolygon(const std::string &str, size_t p, size_t end, size_t* i);
 
   void insertLine(const util::geo::DLine& l, bool isArea);
 
+	static std::vector<size_t> getGeomStarts(const std::string &str, size_t a);
+
   std::string indexHashFromDisk(const std::string& fname);
 
   std::vector<util::geo::FPoint> _points;
@@ -144,13 +155,18 @@ class GeomCache {
   size_t _linesFSize;
   size_t _qidToIdFSize;
 
+  size_t _lastBytesReceived;
+  std::chrono::time_point<std::chrono::high_resolution_clock> _lastReceivedTime;
+
   std::fstream _pointsF;
   std::fstream _linePointsF;
   std::fstream _linesF;
   std::fstream _qidToIdF;
 
   size_t _geometryDuplicates = 0;
 
+  size_t _lastQid = -1;
+
   IdMapping _lastQidToId;
 
   std::vector<IdMapping> _qidToId;

src/qlever-petrimaps/Misc.h

@@ -16,10 +16,10 @@
 #define PETRIMAPS_MISC_H_
 
 #define ID_TYPE uint32_t
-// #define QLEVER_ID_TYPE size_t
-#define QLEVER_ID_TYPE uint32_t
+#define QLEVER_ID_TYPE size_t
 
-const static ID_TYPE I_OFFSET = 500000000;
+// half of the ID space for points, half for the rest
+const static ID_TYPE I_OFFSET = 2147483648;
 const static size_t MAXROWS = 18446744073709551615u;
 
 // major coordinates will fit into 2^15, as coordinates go from
@@ -43,7 +43,6 @@ union ID {
 
 inline bool operator<(const IdMapping& lh, const IdMapping& rh) {
   if (lh.qid < rh.qid) return true;
-  // if (lh.qid == rh.qid && lh.id < rh.id) return true;
   return false;
 }
 

src/qlever-petrimaps/PetriMapsMain.cpp

@@ -24,7 +24,9 @@ void printHelp(int argc, char** argv) {
          "(default: 9090)"
       << "\n    -m <memory>  Max memory in GB (default: 90% of system RAM)"
       << "\n    -c <dir>     cache dir (default: none)"
-      << "\n    -t <minutes> request cache lifetime (default: 360)\n";
+      << "\n    -t <minutes> request cache lifetime (default: 360)"
+      << "\n    -a <numobjects> threshold for auto layer selection (default: "
+         "1000)\n";
 }
 
 // _____________________________________________________________________________
@@ -41,6 +43,7 @@ int main(int argc, char** argv) {
   // default port
   int port = 9090;
   int cacheLifetime = 6 * 60;
+  size_t autoThreshold = 1000;
   double maxMemoryGB =
       (sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGE_SIZE) * 0.9) / 1000000000;
   std::string cacheDir;
@@ -74,6 +77,12 @@ int main(int argc, char** argv) {
         exit(1);
       }
       cacheLifetime = atof(argv[i]);
+    } else if (cur == "-a") {
+      if (++i >= argc) {
+        LOG(ERROR) << "Missing argument for auto threshold (-a).";
+        exit(1);
+      }
+      autoThreshold = atoi(argv[i]);
     }
   }
 
@@ -85,7 +94,7 @@ int main(int argc, char** argv) {
 
   LOG(INFO) << "Starting server...";
   LOG(INFO) << "Max memory is " << maxMemoryGB << " GB...";
-  Server serv(maxMemoryGB * 1000000000, cacheDir, cacheLifetime);
+  Server serv(maxMemoryGB * 1000000000, cacheDir, cacheLifetime, autoThreshold);
 
   LOG(INFO) << "Listening on port " << port;
   util::http::HttpServer(port, &serv, std::thread::hardware_concurrency())