Merge pull request #86 from GIScience/oshdb-0.6

upgrade OSHDB to version 0.6

CHANGELOG.md

@@ -1,9 +1,14 @@
 ## current master
 
+* update OSHDB to version 0.6.0 ([#86](https://github.com/GIScience/ohsome-api/pull/86))
+  * returned values of `length`, `area` and `perimeter` requests are more precise now: The previously used formulas could be off by a few of % in some cases, now all returned values are within half a percent from the real value. ([OSHDB#193](https://github.com/GIScience/oshdb/issues/193))
+  * adjusted unit tests accordingly, from now on allowing up to 0.5% discrepancy in all affected tests
+  * improved performance of `groupBy/boundary` requests ([OSHDB#272](https://github.com/GIScience/oshdb/issues/272)) and processing of complex multipolygon relations ([OSHDB#249](https://github.com/GIScience/oshdb/issues/249) and [OSHDB#287](https://github.com/GIScience/oshdb/issues/287))
 * data extraction: escape tags with `@` characters ([#40](https://github.com/GIScience/ohsome-api/issues/40))
 * extend info on response parameters ([#73](https://github.com/GIScience/ohsome-api/pull/73))
 * delombok code to enable working javadoc ([#90](https://github.com/GIScience/ohsome-api/pull/90))
 
+
 ## 1.2.3
 
 ### Bug Fixes

README.md

@@ -59,7 +59,7 @@ To run the tests locally, you need the following:
     * -Dport.get starts data-aggregation + metadata tests using GET requests
     * -Dport.post starts data-aggregation tests using POST requests
     * -Dport.data starts data-extraction tests using GET and POST requests
-2. [heidelberg.oshb](http://downloads.ohsome.org/v0.4/germany/baden-wuerrtemberg/heidelberg.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
+2. [heidelberg.oshdb](https://downloads.ohsome.org/v0.6/europe/germany/baden-wuerttemberg/heidelberg_68900_2020-07-23.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
 3. maven command: `mvn -Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=<path-to-your-heidelberg.oshdb-file>" test`
 
 *Note:* 
@@ -166,4 +166,3 @@ DOCS_DEPLOYMENT=release make clean html
 ## License
 
 [AGPL](LICENSE).
-

pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.heigit.ohsome</groupId>
     <artifactId>ohsome-parent</artifactId>
-    <version>2.5</version>
+    <version>2.7</version>
   </parent>
 
   <artifactId>ohsome-api</artifactId>
@@ -28,9 +28,8 @@
     <jsonpath.version>2.2.0</jsonpath.version>
     <jts2geojson.version>0.13.0</jts2geojson.version>
     <mavenjar.version>3.2.0</mavenjar.version>
-    <ohsome-filter.version>1.3.0</ohsome-filter.version>
     <opencsv.version>4.0</opencsv.version>
-    <oshdb.version>0.5.10</oshdb.version>
+    <oshdb.version>0.6.1</oshdb.version>
     <projectlombok.version>1.18.16</projectlombok.version>
     <springboot.version>2.0.3.RELEASE</springboot.version>
     <springfox.version>2.9.2</springfox.version>
@@ -58,7 +57,7 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.heigit.bigspatialdata</groupId>
+      <groupId>org.heigit.ohsome</groupId>
       <artifactId>oshdb-api</artifactId>
       <version>${oshdb.version}</version>
       <exclusions>
@@ -73,7 +72,7 @@
       </exclusions>
     </dependency>
     <dependency>
-      <groupId>org.heigit.bigspatialdata</groupId>
+      <groupId>org.heigit.ohsome</groupId>
       <artifactId>oshdb-util</artifactId>
       <version>${oshdb.version}</version>
       <exclusions>
@@ -87,12 +86,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- ohsome OSM entity filter parser + interpreter -->
-    <dependency>
-      <groupId>org.heigit.ohsome</groupId>
-      <artifactId>ohsome-filter</artifactId>
-      <version>${ohsome-filter.version}</version>
-    </dependency>
     <!-- needed for creating a custom error reporting class -->
     <dependency>
       <groupId>com.fasterxml.jackson.datatype</groupId>

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java

@@ -1,6 +1,7 @@
 package org.heigit.ohsome.ohsomeapi.executor;
 
 import static org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator.createGeoJsonFeatures;
+
 import com.opencsv.CSVWriter;
 import java.io.IOException;
 import java.text.DecimalFormat;
@@ -9,7 +10,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.SortedMap;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
@@ -25,7 +25,6 @@
 import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
 import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
 import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.Application;
 import org.heigit.ohsome.ohsomeapi.exception.BadRequestException;
 import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
@@ -352,10 +351,6 @@ private ElementsResult[] fillElementsResult(SortedMap<OSHDBTimestamp, ? extends
     if (processingData.isContainingSimpleFeatureTypes()) {
       mapAgg = inputProcessor.filterOnSimpleFeatures(mapAgg);
     }
-    Optional<FilterExpression> filter = processingData.getFilterExpression();
-    if (filter.isPresent() && ProcessingData.filterContainsGeometryTypeCheck(filter.get())) {
-      mapAgg = inputProcessor.filterOnGeometryType(mapAgg, filter.get());
-    }
     preResult = mapAgg.map(OSMEntitySnapshot::getGeometry);
     switch (requestResource) {
       case COUNT:

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/FlatMapExecutor.java → src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java

@@ -7,19 +7,19 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.function.Supplier;
 import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
 import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
 import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
 import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
 import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
-import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
 import org.locationtech.jts.geom.Geometry;
 import org.wololo.geojson.Feature;
 
-public class FlatMapExecutor implements Serializable {
+public class DataExtractionTransformer implements Serializable {
 
   private final boolean isContributionsLatestEndpoint;
   private final boolean isContributionsEndpoint;
@@ -28,22 +28,18 @@ public class FlatMapExecutor implements Serializable {
   private final String startTimestamp;
   private final InputProcessingUtils utils;
   private final Set<SimpleFeatureType> simpleFeatureTypes;
-  private final boolean requiresGeometryTypeCheck;
-  private final FilterExpression filterExpression;
   private final Set<Integer> keysInt;
   private final boolean includeTags;
   private final boolean includeOSMMetadata;
   private final ElementsGeometry elementsGeometry;
   private final String endTimestamp;
   private final boolean isContainingSimpleFeatureTypes;
 
-  public FlatMapExecutor(boolean isContributionsLatestEndpoint, boolean isContributionsEndpoint,
-      ExecutionUtils exeUtils, boolean clipGeometries, String startTimestamp,
-      InputProcessingUtils utils,
-      Set<SimpleFeatureType> simpleFeatureTypes, boolean requiresGeometryTypeCheck,
-      FilterExpression filterExpression, Set<Integer> keysInt, boolean includeTags,
-      boolean includeOSMMetadata,
-      ElementsGeometry elementsGeometry, String endTimestamp,
+  public DataExtractionTransformer(boolean isContributionsLatestEndpoint,
+      boolean isContributionsEndpoint, ExecutionUtils exeUtils,
+      boolean clipGeometries, String startTimestamp, InputProcessingUtils utils,
+      Set<SimpleFeatureType> simpleFeatureTypes, Set<Integer> keysInt, boolean includeTags,
+      boolean includeOSMMetadata, ElementsGeometry elementsGeometry, String endTimestamp,
       boolean isContainingSimpleFeatureTypes) {
     this.isContributionsLatestEndpoint = isContributionsLatestEndpoint;
     this.isContributionsEndpoint = isContributionsEndpoint;
@@ -52,8 +48,6 @@ public FlatMapExecutor(boolean isContributionsLatestEndpoint, boolean isContribu
     this.startTimestamp = startTimestamp;
     this.utils = utils;
     this.simpleFeatureTypes = simpleFeatureTypes;
-    this.requiresGeometryTypeCheck = requiresGeometryTypeCheck;
-    this.filterExpression = filterExpression;
     this.keysInt = keysInt;
     this.includeTags = includeTags;
     this.includeOSMMetadata = includeOSMMetadata;
@@ -96,9 +90,9 @@ public List<Feature> buildChangedFeatures(List<OSMContribution> contributions) {
         // set valid_to of previous row
         validTo = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
         if (!skipNext && currentGeom != null && !currentGeom.isEmpty()) {
+          final Geometry geomToCheck = currentGeom;
           boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils,
-              simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, currentGeom,
-              currentEntity);
+              simpleFeatureTypes, () -> geomToCheck);
           if (addToOutput) {
             properties = new TreeMap<>();
             if (!isContributionsEndpoint) {
@@ -140,9 +134,9 @@ public List<Feature> buildChangedFeatures(List<OSMContribution> contributions) {
             TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp()));
       }
       if (!currentGeom.isEmpty()) {
+        final Geometry geomToCheck = currentGeom;
         boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils,
-            simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, currentGeom,
-            currentEntity);
+            simpleFeatureTypes, () -> geomToCheck);
         if (addToOutput) {
           output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
               includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry,
@@ -168,35 +162,36 @@ public List<Feature> buildUnchangedFeatures(OSMEntitySnapshot snapshot) {
     if (includeOSMMetadata) {
       properties.put("@lastEdit", entity.getTimestamp().toString());
     }
-    Geometry geom = snapshot.getGeometry();
-    if (!clipGeometries) {
-      geom = snapshot.getGeometryUnclipped();
+    Supplier<Geometry> geom;
+    if (clipGeometries) {
+      geom = snapshot::getGeometry;
+    } else {
+      geom = snapshot::getGeometryUnclipped;
     }
     properties.put("@snapshotTimestamp",
         TimestampFormatter.getInstance().isoDateTime(snapshot.getTimestamp()));
     properties.put("@validFrom", startTimestamp);
     properties.put("@validTo", endTimestamp);
     boolean addToOutput = addEntityToOutput(isContainingSimpleFeatureTypes, utils,
-        simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, geom, entity);
+        simpleFeatureTypes, geom);
     if (addToOutput) {
       return Collections.singletonList(
-          exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags,
+          exeUtils.createOSMFeature(entity, geom.get(), properties, keysInt, includeTags,
               includeOSMMetadata, isContributionsEndpoint, elementsGeometry, null));
     } else {
       return Collections.emptyList();
     }
   }
 
   /** Checks whether the given entity should be added to the output (true) or not (false). */
-  public static boolean addEntityToOutput(boolean isContainingSimpleFeatureTypes,
+  public static boolean addEntityToOutput(
+      boolean isContainingSimpleFeatureTypes,
       InputProcessingUtils utils,
-      final Set<SimpleFeatureType> simpleFeatureTypes, final boolean requiresGeometryTypeCheck,
-      FilterExpression filterExpression, Geometry currentGeom, OSMEntity currentEntity) {
+      final Set<SimpleFeatureType> simpleFeatureTypes,
+      Supplier<Geometry> currentGeom) {
     boolean addToOutput;
     if (isContainingSimpleFeatureTypes) {
-      addToOutput = utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes);
-    } else if (requiresGeometryTypeCheck) {
-      addToOutput = filterExpression.applyOSMGeometry(currentEntity, currentGeom);
+      addToOutput = utils.checkGeometryOnSimpleFeatures(currentGeom.get(), simpleFeatureTypes);
     } else {
       addToOutput = true;
     }

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java

@@ -15,7 +15,7 @@
 import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
 import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
 import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
-import org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser;
+import org.heigit.bigspatialdata.oshdb.util.time.IsoDateTimeParser;
 import org.heigit.ohsome.filter.FilterExpression;
 import org.heigit.ohsome.ohsomeapi.Application;
 import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry;
@@ -52,12 +52,12 @@ public DataRequestExecutor(RequestResource requestResource, ElementsGeometry ele
    * 
    * @throws Exception thrown by
    *         {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor#processParameters()
-   *         processParameters},
-   *         {@link org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser#parseISODateTime(String)
-   *         parseISODateTime},
+   *         processParameters}, {@link
+   *         org.heigit.bigspatialdata.oshdb.util.time.IsoDateTimeParser#parseIsoDateTime(String)
+   *         parseIsoDateTime},
    *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#stream() stream}, or
-   *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#streamResponse(HttpServletResponse, DataResponse, Stream)
-   *         streamElementsResponse}
+   *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#streamResponse(
+   *         HttpServletResponse, DataResponse, Stream) streamElementsResponse}
    */
   public void extract() throws Exception {
     inputProcessor.getProcessingData().setFullHistory(true);
@@ -96,22 +96,23 @@ public void extract() throws Exception {
     final boolean isContributionsEndpoint =
         isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS);
     final Set<SimpleFeatureType> simpleFeatureTypes = processingData.getSimpleFeatureTypes();
-    Optional<FilterExpression> filter = processingData.getFilterExpression();
-    final boolean requiresGeometryTypeCheck =
-        filter.isPresent() && ProcessingData.filterContainsGeometryTypeCheck(filter.get());
-    FilterExpression filterExpression = processingData.getFilterExpression().orElse(null);
-    String startTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[0])
+    String startTimestamp = IsoDateTimeParser.parseIsoDateTime(requestParameters.getTime()[0])
         .format(DateTimeFormatter.ISO_DATE_TIME);
-    String endTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[1])
+    String endTimestamp = IsoDateTimeParser.parseIsoDateTime(requestParameters.getTime()[1])
         .format(DateTimeFormatter.ISO_DATE_TIME);
     MapReducer<List<OSMContribution>> mapRedContributions = mapRedContribution.groupByEntity();
+    Optional<FilterExpression> filter = processingData.getFilterExpression();
+    if (filter.isPresent()) {
+      mapRedSnapshot = mapRedSnapshot.filter(filter.get());
+      mapRedContributions = mapRedContributions.filter(filter.get());
+    }
     final boolean isContainingSimpleFeatureTypes = processingData.isContainingSimpleFeatureTypes();
-    FlatMapExecutor flatMapExecutor = new FlatMapExecutor(isContributionsLatestEndpoint,
-        isContributionsEndpoint, exeUtils, clipGeometries, startTimestamp, utils,
-        simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, keysInt, includeTags,
-        includeOSMMetadata, elementsGeometry, endTimestamp, isContainingSimpleFeatureTypes);
+    DataExtractionTransformer dataExtractionTransformer = new DataExtractionTransformer(
+        isContributionsLatestEndpoint, isContributionsEndpoint, exeUtils, clipGeometries,
+        startTimestamp, utils, simpleFeatureTypes, keysInt, includeTags, includeOSMMetadata,
+        elementsGeometry, endTimestamp, isContainingSimpleFeatureTypes);
     MapReducer<Feature> contributionPreResult = mapRedContributions
-        .flatMap(flatMapExecutor::buildChangedFeatures)
+        .flatMap(dataExtractionTransformer::buildChangedFeatures)
         .filter(Objects::nonNull);
     Metadata metadata = null;
     if (processingData.isShowMetadata()) {
@@ -130,7 +131,7 @@ public void extract() throws Exception {
               && snapshots.get(0).getEntity().getVersion() == snapshots.get(1).getEntity()
                   .getVersion())
           .map(snapshots -> snapshots.get(0))
-          .flatMap(flatMapExecutor::buildUnchangedFeatures)
+          .flatMap(dataExtractionTransformer::buildUnchangedFeatures)
           .filter(Objects::nonNull);
     }
     try (

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

@@ -9,7 +9,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -192,10 +191,6 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
     if (processingData.isContainingSimpleFeatureTypes()) {
       mapAgg = inputProcessor.filterOnSimpleFeatures(mapAgg);
     }
-    Optional<FilterExpression> filter = processingData.getFilterExpression();
-    if (filter.isPresent() && ProcessingData.filterContainsGeometryTypeCheck(filter.get())) {
-      mapAgg = inputProcessor.filterOnGeometryType(mapAgg, filter.get());
-    }
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
     var preResult = mapAgg.map(f -> exeUtils.mapSnapshotToTags(keysInt, valuesInt, f))
         .aggregateBy(Pair::getKey, zeroFill).map(Pair::getValue)