From 6085247f7506d28e146564b9c237c9458bdc0391 Mon Sep 17 00:00:00 2001 From: Grandro <48568096+Grandro@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:08:52 +0200 Subject: [PATCH] Fixes concerning Grandro-submit-menu (#12) * Fixed logic for parsing GeoJson files for polygons and multipart geometries. * Changed CMakeLists debug build flags to allow debugging without optimized out variable values. * Implemented better support for multi-geometries such that geometries of a multi-geometry belong together. * Reverted datastructure in SPARQLCache for _lines and _points because it handles multi-geometries differently than GeoJSONCache. * Fixed GeoJson export (for MultiGeometries) * Fixed inconsistencies. --- CMakeLists.txt | 4 +- src/qlever-petrimaps/GeoJSONCache.cpp | 71 +++++++++++++------ src/qlever-petrimaps/GeoJSONCache.h | 12 ++++ src/qlever-petrimaps/GeomCache.h | 21 +++--- src/qlever-petrimaps/SPARQLCache.h | 12 ++++ .../server/GeoJSONRequestor.cpp | 8 +++ src/qlever-petrimaps/server/Requestor.cpp | 34 ++++----- src/qlever-petrimaps/server/Requestor.h | 7 +- .../server/SPARQLRequestor.cpp | 8 +++ src/qlever-petrimaps/server/Server.cpp | 8 ++- 10 files changed, 126 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e79e9..d0e6095 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ if (PNG_FOUND) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -set(CMAKE_CXX_FLAGS_DEBUG "-Og -g -DLOGLEVEL=3") +set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -DLOGLEVEL=3")#-Og -g -DLOGLEVEL=3") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -DLOGLEVEL=2") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -DLOGLEVEL=2") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS} -g -DLOGLEVEL=3") @@ -63,4 +63,4 @@ add_subdirectory(src) install( FILES ${CMAKE_BINARY_DIR}/petrimaps DESTINATION bin PERMISSIONS OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE -) +) \ No newline at end of file diff --git a/src/qlever-petrimaps/GeoJSONCache.cpp b/src/qlever-petrimaps/GeoJSONCache.cpp index 36c2db5..0e44f99 100644 --- a/src/qlever-petrimaps/GeoJSONCache.cpp +++ b/src/qlever-petrimaps/GeoJSONCache.cpp @@ -14,13 +14,24 @@ std::vector> GeoJSONCache::getRelObjects() const { // Used for GeoJSON, returns all objects as vector> // geomID starts from 0 ascending, Row = geomID std::vector> objects; - objects.reserve(_points.size() + _linePoints.size()); + objects.reserve(_points.size() + _lines.size()); + size_t idx = 0; for (size_t i = 0; i < _points.size(); i++) { - objects.push_back({i, i}); + bool isFirst = std::get<1>(_points[i]); + if (isFirst && i > 0) { + idx++; + } + objects.push_back({i, idx}); } + + idx = 0; for (size_t i = 0; i < _lines.size(); i++) { - objects.push_back({i + I_OFFSET, i + I_OFFSET}); + bool isFirst = std::get<1>(_lines[i]); + if (isFirst && i > 0) { + idx++; + } + objects.push_back({i + I_OFFSET, idx + I_OFFSET}); } return objects; @@ -131,24 +142,26 @@ void GeoJSONCache::load() { auto coords = geom["coordinates"]; auto properties = feature["properties"]; - LOG(INFO) << geom["type"]; + LOG(INFO) << type; // PRIMITIVES + // Point if (type == "Point") { auto point = latLngToWebMerc(FPoint(coords[0], coords[1])); if (!pointValid(point)) { LOG(INFO) << "[GeomCache] Invalid point found. Skipping..."; continue; } - _points.push_back(point); + _points.push_back({point, true}); _curUniqueGeom++; _attr[numPoints] = properties; numPoints++; - + + // LineString } else if (type == "LineString") { util::geo::DLine line; - line.reserve(2); + line.reserve(coords.size()); for (std::vector coord : coords) { auto point = latLngToWebMerc(DPoint(coord[0], coord[1])); @@ -159,18 +172,20 @@ void GeoJSONCache::load() { line.push_back(point); } std::size_t idx = _linePoints.size(); - _lines.push_back(idx); + _lines.push_back({idx, true}); line = util::geo::densify(line, 200 * 3); insertLine(line, false); _curUniqueGeom++; _attr[numLines + I_OFFSET] = properties; numLines++; - + + // Polygon } else if (type == "Polygon") { - for (auto args : coords) { + for (size_t i = 0; i < coords.size(); i++) { + auto args = coords[i]; util::geo::DLine line; - line.reserve(2); + line.reserve(args.size()); for (std::vector coord : args) { auto point = latLngToWebMerc(DPoint(coord[0], coord[1])); @@ -180,8 +195,10 @@ void GeoJSONCache::load() { } line.push_back(point); } + std::size_t idx = _linePoints.size(); - _lines.push_back(idx); + bool isFirst = i == 0; + _lines.push_back({idx, isFirst}); line = util::geo::densify(line, 200 * 3); insertLine(line, true); } @@ -190,24 +207,30 @@ void GeoJSONCache::load() { _attr[numLines + I_OFFSET] = properties; numLines++; - // MULTIPART + // MULTIPART + // MultiPoint } else if (type == "MultiPoint") { - for (std::vector coord : coords) { + for (size_t i = 0; i < coords.size(); i++) { + std::vector coord = coords[i]; auto point = latLngToWebMerc(FPoint(coord[0], coord[1])); if (!pointValid(point)) { LOG(INFO) << "[GeomCache] Invalid point found. Skipping..."; continue; } - _points.push_back(point); + bool isFirst = i == 0; + _points.push_back({point, isFirst}); } _curUniqueGeom++; _attr[numPoints] = properties; numPoints++; + + // MultiLineString } else if (type == "MultiLineString") { - for (auto args : coords) { + for (size_t i = 0; i < coords.size(); i++) { + auto args = coords[i]; util::geo::DLine line; - line.reserve(2); + line.reserve(args.size()); for (std::vector coord : args) { auto point = latLngToWebMerc(DPoint(coord[0], coord[1])); @@ -218,7 +241,8 @@ void GeoJSONCache::load() { line.push_back(point); } std::size_t idx = _linePoints.size(); - _lines.push_back(idx); + bool isFirst = i == 0; + _lines.push_back({idx, isFirst}); line = util::geo::densify(line, 200 * 3); insertLine(line, false); } @@ -226,12 +250,14 @@ void GeoJSONCache::load() { _curUniqueGeom++; _attr[numLines + I_OFFSET] = properties; numLines++; - + + // MultiPolygon } else if (type == "MultiPolygon") { - for (auto args1 : coords) { + for (size_t i = 0; i < coords.size(); i++) { + auto args1 = coords[i]; for (auto args2 : args1) { util::geo::DLine line; - line.reserve(2); + line.reserve(args2.size()); for (std::vector coord : args2) { auto point = latLngToWebMerc(DPoint(coord[0], coord[1])); @@ -242,7 +268,8 @@ void GeoJSONCache::load() { line.push_back(point); } std::size_t idx = _linePoints.size(); - _lines.push_back(idx); + bool isFirst = i == 0; + _lines.push_back({idx, isFirst}); line = util::geo::densify(line, 200 * 3); insertLine(line, true); } diff --git a/src/qlever-petrimaps/GeoJSONCache.h b/src/qlever-petrimaps/GeoJSONCache.h index 8a3cb9c..85d8b60 100644 --- a/src/qlever-petrimaps/GeoJSONCache.h +++ b/src/qlever-petrimaps/GeoJSONCache.h @@ -22,6 +22,16 @@ class GeoJSONCache : public GeomCache { return *this; }; + const util::geo::FPoint& getPoint(ID_TYPE id) const { + return std::get<0>(_points[id]); + } + size_t getLine(ID_TYPE id) const { + return std::get<0>(_lines[id]); + } + size_t getLineEnd(ID_TYPE id) const { + return id + 1 < _lines.size() ? getLine(id + 1) : _linePoints.size(); + } + void load(); void setContent(const std::string& content); std::vector> getRelObjects() const; @@ -36,6 +46,8 @@ class GeoJSONCache : public GeomCache { // Map geomID to map std::map> _attr; // ------------------------ + std::vector> _points; + std::vector> _lines; protected: diff --git a/src/qlever-petrimaps/GeomCache.h b/src/qlever-petrimaps/GeomCache.h index 9e4836b..b563edc 100644 --- a/src/qlever-petrimaps/GeomCache.h +++ b/src/qlever-petrimaps/GeomCache.h @@ -40,23 +40,20 @@ class GeomCache { return ready; } - const std::vector& getPoints() const { return _points; } + const virtual util::geo::FPoint& getPoint(ID_TYPE id) const = 0; + virtual size_t getLine(ID_TYPE id) const = 0; + virtual size_t getLineEnd(ID_TYPE id) const = 0; + const std::vector>& getLinePoints() const { return _linePoints; } - const std::vector& getLines() const { return _lines; } - size_t getLine(ID_TYPE id) const { return _lines[id]; } - size_t getLineEnd(ID_TYPE id) const { - return id + 1 < _lines.size() ? _lines[id + 1] : _linePoints.size(); - } - - util::geo::FBox getPointBBox(size_t id) const { - return util::geo::getBoundingBox(_points[id]); - } + util::geo::DBox getLineBBox(size_t id) const; double getLoadStatusPercent(bool total); - double getLoadStatusPercent() { return getLoadStatusPercent(false); }; + double getLoadStatusPercent() { + return getLoadStatusPercent(false); + }; int getLoadStatusStage(); size_t getTotalProgress(); size_t getCurrentProgress(); @@ -72,9 +69,7 @@ class GeomCache { mutable std::mutex _m; bool _ready = false; - std::vector _points; std::vector> _linePoints; - std::vector _lines; static bool pointValid(const util::geo::FPoint& p); static bool pointValid(const util::geo::DPoint& p); diff --git a/src/qlever-petrimaps/SPARQLCache.h b/src/qlever-petrimaps/SPARQLCache.h index 154cbf0..18aefef 100644 --- a/src/qlever-petrimaps/SPARQLCache.h +++ b/src/qlever-petrimaps/SPARQLCache.h @@ -27,6 +27,16 @@ class SPARQLCache : public GeomCache { return *this; }; + const util::geo::FPoint& getPoint(ID_TYPE id) const { + return _points[id]; + } + size_t getLine(ID_TYPE id) const { + return _lines[id]; + } + size_t getLineEnd(ID_TYPE id) const { + return id + 1 < _lines.size() ? getLine(id + 1) : _linePoints.size(); + } + std::string load(const std::string& cacheFile); void request(); size_t requestSize(); @@ -83,6 +93,8 @@ class SPARQLCache : public GeomCache { IdMapping _lastQidToId; std::vector _qidToId; + std::vector _points; + std::vector _lines; std::string _dangling, _prev, _raw; ParseState _state; diff --git a/src/qlever-petrimaps/server/GeoJSONRequestor.cpp b/src/qlever-petrimaps/server/GeoJSONRequestor.cpp index c050518..24bd546 100644 --- a/src/qlever-petrimaps/server/GeoJSONRequestor.cpp +++ b/src/qlever-petrimaps/server/GeoJSONRequestor.cpp @@ -20,12 +20,20 @@ void GeoJSONRequestor::request() { _ready = false; _objects.clear(); _clusterObjects.clear(); + _rowIdToObjectId.clear(); _objects = _cache->getRelObjects(); _numObjects = _objects.size(); LOG(INFO) << "[REQUESTOR] ... done, got " << _objects.size() << " objects."; + // Create mapping rowId to objectId for multigeometries + for (size_t objectId = 0; objectId < _objects.size(); objectId++) { + std::pair object = _objects[objectId]; + ID_TYPE rowId = object.second; + _rowIdToObjectId[rowId] = objectId; + } + LOG(INFO) << "[REQUESTOR] Calculating bounding box of result..."; util::geo::FBox pointBbox; diff --git a/src/qlever-petrimaps/server/Requestor.cpp b/src/qlever-petrimaps/server/Requestor.cpp index 15538c6..a81388d 100644 --- a/src/qlever-petrimaps/server/Requestor.cpp +++ b/src/qlever-petrimaps/server/Requestor.cpp @@ -34,18 +34,14 @@ void Requestor::createBboxes(util::geo::FBox& pointBbox, #pragma omp parallel for num_threads(NUM_THREADS) schedule(static) for (size_t t = 0; t < NUM_THREADS; t++) { - for (size_t i = batch * t; i < batch * (t + 1) && i < _objects.size(); - i++) { + for (size_t i = batch * t; i < batch * (t + 1) && i < _objects.size(); i++) { auto geomId = _objects[i].first; - if (geomId < I_OFFSET) { auto pId = geomId; - pointBoxes[t] = - util::geo::extendBox(_cache->getPoints()[pId], pointBoxes[t]); + pointBoxes[t] = util::geo::extendBox(_cache->getPoint(pId), pointBoxes[t]); } else if (geomId < std::numeric_limits::max()) { auto lId = geomId - I_OFFSET; - lineBoxes[t] = - util::geo::extendBox(_cache->getLineBBox(lId), lineBoxes[t]); + lineBoxes[t] = util::geo::extendBox(_cache->getLineBBox(lId), lineBoxes[t]); numLines[t]++; } } @@ -126,12 +122,12 @@ void Requestor::createGrid(util::geo::FBox pointBbox, for (size_t m = 0; m < clusterI; m++) { const auto& p = _objects[i - m]; auto geomId = p.first; - _pgrid.add(_cache->getPoints()[geomId], j); + _pgrid.add(_cache->getPoint(geomId), j); _clusterObjects.push_back({i - m, {m, clusterI}}); j++; } } else { - _pgrid.add(_cache->getPoints()[geomId], i); + _pgrid.add(_cache->getPoint(geomId), i); } // every 100000 objects, check memory... @@ -293,7 +289,7 @@ const ResObj Requestor::getNearest(util::geo::DPoint rp, double rad, double res, size_t cid = i - _objects.size(); p = clusterGeom(cid, res); } else { - p = _cache->getPoints()[_objects[i].first]; + p = _cache->getPoint(_objects[i].first); } if (!util::geo::contains(p, fbox)) continue; @@ -458,23 +454,23 @@ const ResObj Requestor::getNearest(util::geo::DPoint rp, double rad, double res, } // _____________________________________________________________________________ -const ResObj Requestor::getGeom(size_t id, double rad) const { +const ResObj Requestor::getGeom(size_t objectId, double rad) const { if (!_cache->ready()) { throw std::runtime_error("Geom cache not ready"); } - auto obj = _objects[id]; + std::pair obj = _objects[objectId]; + if (obj.first >= I_OFFSET) { size_t lineId = obj.first - I_OFFSET; bool isArea = Requestor::isArea(lineId); - if (isArea) { - return {true, id, {{0, 0}}, {}, {}, geomPolyGeoms(id, rad / 10)}; + return {true, objectId, {{0, 0}}, {}, {}, geomPolyGeoms(objectId, rad / 10)}; } else { - return {true, id, {{0, 0}}, {}, geomLineGeoms(id, rad / 10), {}}; + return {true, objectId, {{0, 0}}, {}, geomLineGeoms(objectId, rad / 10), {}}; } } else { - return {true, id, geomPointGeoms(id), {}, {}, {}}; + return {true, objectId, geomPointGeoms(objectId), {}, {}, {}}; } } @@ -564,13 +560,13 @@ util::geo::MultiPoint Requestor::geomPointGeoms(size_t oid, for (size_t i = oid; i < _objects.size() && _objects[i].second == _objects[oid].second; i++) { if (_objects[oid].first >= I_OFFSET) continue; - points.push_back(_cache->getPoints()[_objects[i].first]); + points.push_back(_cache->getPoint(_objects[i].first)); } for (size_t i = oid - 1; i < _objects.size() && _objects[i].second == _objects[oid].second; i--) { if (_objects[oid].first >= I_OFFSET) continue; - points.push_back(_cache->getPoints()[_objects[i].first]); + points.push_back(_cache->getPoint(_objects[i].first)); } return points; @@ -602,7 +598,7 @@ util::geo::MultiPolygon Requestor::geomPolyGeoms(size_t oid, // _____________________________________________________________________________ util::geo::FPoint Requestor::clusterGeom(size_t cid, double res) const { size_t oid = _clusterObjects[cid].first; - const auto& pp = _cache->getPoints()[_objects[oid].first]; + const auto& pp = _cache->getPoint(_objects[oid].first); if (res < 0) return {pp}; diff --git a/src/qlever-petrimaps/server/Requestor.h b/src/qlever-petrimaps/server/Requestor.h index c04f2da..2ddf25b 100644 --- a/src/qlever-petrimaps/server/Requestor.h +++ b/src/qlever-petrimaps/server/Requestor.h @@ -63,8 +63,12 @@ class Requestor { return _clusterObjects; } + ID_TYPE getObjectIdFromRowId(ID_TYPE rowId) const { + return _rowIdToObjectId.at(rowId); + } + const util::geo::FPoint& getPoint(ID_TYPE id) const { - return _cache->getPoints()[id]; + return _cache->getPoint(id); } size_t getLine(ID_TYPE id) const { return _cache->getLine(id); } @@ -120,6 +124,7 @@ class Requestor { std::vector> _objects; std::vector>> _clusterObjects; size_t _numObjects = 0; + std::map _rowIdToObjectId; // Match rowId to objectId for multigeometries }; } // namespace petrimaps diff --git a/src/qlever-petrimaps/server/SPARQLRequestor.cpp b/src/qlever-petrimaps/server/SPARQLRequestor.cpp index a364ddc..b947305 100644 --- a/src/qlever-petrimaps/server/SPARQLRequestor.cpp +++ b/src/qlever-petrimaps/server/SPARQLRequestor.cpp @@ -24,6 +24,7 @@ void SPARQLRequestor::request(const std::string& query) { _ready = false; _objects.clear(); _clusterObjects.clear(); + _rowIdToObjectId.clear(); RequestReader reader(_cache->getBackendURL(), _maxMemory); @@ -48,6 +49,13 @@ void SPARQLRequestor::request(const std::string& query) { _numObjects = ret.second; LOG(INFO) << "[REQUESTOR] ... done, got " << _objects.size() << " objects."; + // Create mapping rowId to objectId for multigeometries + for (size_t objectId = 0; objectId < _objects.size(); objectId++) { + std::pair object = _objects[objectId]; + ID_TYPE rowId = object.second; + _rowIdToObjectId[rowId] = objectId; + } + LOG(INFO) << "[REQUESTOR] Calculating bounding box of result..."; util::geo::FBox pointBbox; diff --git a/src/qlever-petrimaps/server/Server.cpp b/src/qlever-petrimaps/server/Server.cpp index be26082..44c5586 100755 --- a/src/qlever-petrimaps/server/Server.cpp +++ b/src/qlever-petrimaps/server/Server.cpp @@ -1071,16 +1071,18 @@ util::http::Answer Server::handleExportReq(const Params& pars, int sock) const { } bool first = false; + size_t rowId = 0; reqor->requestRows( - [sock, &first, reqor, this]( + [sock, &first, &rowId, reqor, this]( std::vector>> rows) { std::stringstream ss; ss << std::setprecision(10); util::json::Val attrs; for (size_t i = 0; i < rows.size(); i++) { auto& row = rows[i]; - auto res = reqor->getGeom(i, 0); + ID_TYPE objectId = reqor->getObjectIdFromRowId(rowId); + auto res = reqor->getGeom(objectId, 0); for (size_t j = 0; j < row.size(); j++) { attrs.dict[row[j].first] = row[j].second; @@ -1091,6 +1093,8 @@ util::http::Answer Server::handleExportReq(const Params& pars, int sock) const { processGeoJsonOutput(geoJsonOut, res, attrs); first = true; ss << "\n"; + + rowId++; } std::string buff = ss.str();