oshdb-1.0.0-beta-1

CHANGELOG.md

@@ -3,6 +3,10 @@ Changelog
 
 ## 1.7.0-SNAPSHOT (current master)
 
+### Breaking Changes
+* update OSHDB to version 1.0.0-beta-1 ([#278])
+  * OSM type strings change to `node`, `way`, `relation`
+
 ### New Features
 
 * add `contributions/count[/density]/groupBy/boundary` endpoints to fetch contribution counts/densities grouped by boundary ([#217])
@@ -21,6 +25,7 @@ Changelog
 [#228]: https://github.com/GIScience/ohsome-api/pull/228
 [#230]: https://github.com/GIScience/ohsome-api/pull/230
 [#274]: https://github.com/GIScience/ohsome-api/pull/274
+[#278]: https://github.com/GIScience/ohsome-api/pull/278
 
 
 ## 1.6.3

Jenkinsfile

@@ -12,7 +12,7 @@ pipeline {
 
     MAVEN_GENERAL_OPTIONS = '--batch-mode --update-snapshots'
     // START CUSTOM ohsome API
-    MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=/opt/data/heidelberg-v0.7.oshdb"'
+    MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=/opt/data/heidelberg-v1.0-beta.oshdb"'
     // END CUSTOM ohsome API
     SNAPSHOT_BRANCH_REGEX = /(^master$)/
     RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)(-(RC|beta-|alpha-)[0-9]+)?$/

README.md

@@ -143,7 +143,7 @@ To run the tests locally, you need the following:
     * -Dport.get starts data-aggregation + metadata tests using GET requests
     * -Dport.post starts data-aggregation tests using POST requests
     * -Dport.data starts data-extraction tests using GET and POST requests
-2. [heidelberg.oshdb](https://downloads.ohsome.org/OSHDB/v0.7/europe/germany/baden-wuerttemberg/heidelberg.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
+2. [heidelberg.oshdb](https://downloads.ohsome.org/OSHDB/v1.0/europe/germany/baden-wuerttemberg/heidelberg.oshdb.mv.db) file (or any other, which includes the data from Heidelberg)
 3. maven command: `mvn -Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty="--database.db=<path-to-your-heidelberg.oshdb-file>" test`
 
 *Note:* 

docs/endpoints.rst

@@ -2043,7 +2043,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/323191854",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 8,
@@ -2077,7 +2077,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/5400804545",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 2,
@@ -2120,7 +2120,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/323191854",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 8,
@@ -2154,7 +2154,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/5400804545",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 2,
@@ -2197,7 +2197,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/323191854",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 8,
@@ -2231,7 +2231,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/5400804545",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 2,
@@ -2274,7 +2274,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/323191854",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 8,
@@ -2308,7 +2308,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg
             "@changesetId" : 83099383,
             "@contributionChangesetId" : 83099383,
             "@osmId" : "node/5400804545",
-            "@osmType" : "NODE",
+            "@osmType" : "node",
             "@tagChange" : true,
             "@timestamp" : "2020-04-05T13:32:50Z",
             "@version" : 2,
@@ -2398,7 +2398,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 85604249,
             "@geometryChange" : true,
             "@osmId" : "way/795435536",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2020-05-22T10:22:53Z",
             "@version" : 3,
             "landuse" : "construction"
@@ -2411,7 +2411,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 51902131,
             "@deletion" : true,
             "@osmId" : "way/135635599",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2017-09-10T09:22:03Z",
             "@version" : 9
           }
@@ -2447,7 +2447,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 85604249,
             "@geometryChange" : true,
             "@osmId" : "way/795435536",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2020-05-22T10:22:53Z",
             "@version" : 3,
             "landuse" : "construction"
@@ -2460,7 +2460,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 51902131,
             "@deletion" : true,
             "@osmId" : "way/135635599",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2017-09-10T09:22:03Z",
             "@version" : 9
           }
@@ -2496,7 +2496,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 85604249,
             "@geometryChange" : true,
             "@osmId" : "way/795435536",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2020-05-22T10:22:53Z",
             "@version" : 3,
             "landuse" : "construction"
@@ -2509,7 +2509,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 51902131,
             "@deletion" : true,
             "@osmId" : "way/135635599",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2017-09-10T09:22:03Z",
             "@version" : 9
           }
@@ -2545,7 +2545,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 85604249,
             "@geometryChange" : true,
             "@osmId" : "way/795435536",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2020-05-22T10:22:53Z",
             "@version" : 3,
             "landuse" : "construction"
@@ -2558,7 +2558,7 @@ Get the latest change of constructions in a certain area of the Bahnstadt in Hei
             "@contributionChangesetId" : 51902131,
             "@deletion" : true,
             "@osmId" : "way/135635599",
-            "@osmType" : "WAY",
+            "@osmType" : "way",
             "@timestamp" : "2017-09-10T09:22:03Z",
             "@version" : 9
           }

docs/response-parameters.rst

@@ -42,7 +42,7 @@ Description of the custom response parameters that are marked with a leading ``@
 * ``@osmId`` - id of the OSM element, including its type (e.g. node/1)
 * ``@version`` - version number of the OSM element
 * ``@changesetId`` - id of the OSM changeset which last increased the version of this OSM element
-* ``@osmType`` - type of the OSM element (NODE, WAY or RELATION)
+* ``@osmType`` - type of the OSM element (node, way or relation)
 
 **specific for /elements**
 

pom.xml

@@ -23,22 +23,22 @@
   </scm>
 
   <properties>
-    <apachecommons-csv.version>1.6</apachecommons-csv.version>
-    <apachecommons.version>3.7</apachecommons.version>
-    <geojson-jackson.version>1.8</geojson-jackson.version>
-    <geotools.version>20.2</geotools.version>
+    <apachecommons-csv.version>1.9.0</apachecommons-csv.version>
+    <apachecommons.version>3.12.0</apachecommons.version>
+    <geojson-jackson.version>1.14</geojson-jackson.version>
+    <geotools.version>27.0</geotools.version>
     <gitcommitplugin.version>2.2.6</gitcommitplugin.version>
-    <hikaricp.version>3.4.2</hikaricp.version>
-    <jacksondatatype.version>2.9.8</jacksondatatype.version>
+    <hikaricp.version>5.0.1</hikaricp.version>
+    <jacksondatatype.version>2.13.3</jacksondatatype.version>
     <javaxjson.version>1.1.4</javaxjson.version>
-    <jsonpath.version>2.2.0</jsonpath.version>
-    <jts2geojson.version>0.13.0</jts2geojson.version>
+    <jsonpath.version>2.7.0</jsonpath.version>
+    <jts2geojson.version>0.17.0</jts2geojson.version>
     <mavenjar.version>3.2.0</mavenjar.version>
-    <opencsv.version>4.0</opencsv.version>
-    <oshdb.version>0.7.2</oshdb.version>
+    <opencsv.version>5.6</opencsv.version>
+    <oshdb.version>1.0.0-beta-1</oshdb.version>
     <projectlombok.version>1.18.16</projectlombok.version>
     <sonar.sources>src/main/lombok</sonar.sources>
-    <springboot.version>2.5.12</springboot.version>
+    <springboot.version>2.5.14</springboot.version>
     <spring.version>5.3.18</spring.version>
     <springfox.version>2.9.2</springfox.version>
   </properties>
@@ -155,10 +155,14 @@
       <artifactId>javax.json</artifactId>
       <version>${javaxjson.version}</version>
     </dependency>
-    <!-- https://mvnrepository.com/artifact/org.geotools/gt-geometry -->
     <dependency>
       <groupId>org.geotools</groupId>
-      <artifactId>gt-geometry</artifactId>
+      <artifactId>gt-opengis</artifactId>
+      <version>${geotools.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.geotools</groupId>
+      <artifactId>gt-main</artifactId>
       <version>${geotools.version}</version>
     </dependency>
     <!-- https://mvnrepository.com/artifact/org.geotools/gt-epsg-hsql -->

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java

@@ -36,6 +36,7 @@
 import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResult;
 import org.heigit.ohsome.ohsomeapi.output.ratio.RatioResult;
 import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
+import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter;
 import org.heigit.ohsome.oshdb.OSHDBTimestamp;
 import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex;
 import org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator;
@@ -44,7 +45,6 @@
 import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
 import org.heigit.ohsome.oshdb.util.geometry.Geo;
 import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
-import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Polygonal;
 

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java

@@ -11,12 +11,12 @@
 import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils;
 import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
+import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter;
 import org.heigit.ohsome.oshdb.filter.FilterExpression;
 import org.heigit.ohsome.oshdb.osm.OSMEntity;
 import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
 import org.heigit.ohsome.oshdb.util.mappable.OSMContribution;
 import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
-import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
 import org.wololo.geojson.Feature;
 

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

@@ -40,7 +40,9 @@
 import org.heigit.ohsome.ohsomeapi.output.elements.ElementsResult;
 import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResponse;
 import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResult;
+import org.heigit.ohsome.ohsomeapi.utils.FilterUtil;
 import org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator;
+import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter;
 import org.heigit.ohsome.oshdb.OSHDBTag;
 import org.heigit.ohsome.oshdb.OSHDBTimestamp;
 import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
@@ -57,7 +59,6 @@
 import org.heigit.ohsome.oshdb.util.geometry.Geo;
 import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
 import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
-import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Polygonal;
 import org.wololo.geojson.Feature;
@@ -485,6 +486,8 @@ public static Response aggregateGroupByKey(RequestResource requestResource,
         resultSet);
   }
 
+
+
   /**
    * Performs a count|length|perimeter|area|ratio calculation.
    *
@@ -576,7 +579,7 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc
     } else {
       mapRed = inputProcessor.processParameters();
     }
-    mapRed = mapRed.osmType(osmTypes);
+    mapRed = mapRed.filter(FilterUtil.filter(osmTypes));
     mapRed = ExecutionUtils.snapshotFilter(mapRed, osmTypes1, osmTypes2, simpleFeatureTypes1,
         simpleFeatureTypes2, keysInt1, keysInt2, valuesInt1, valuesInt2);
     var preResult = mapRed.aggregateByTimestamp().aggregateBy(snapshot -> {
@@ -825,7 +828,7 @@ public static <P extends Geometry & Polygonal> Response aggregateBasicFiltersRat
     } else {
       mapRed = inputProcessor.processParameters();
     }
-    mapRed = mapRed.osmType(osmTypes);
+    mapRed = mapRed.filter(FilterUtil.filter(osmTypes));
     ArrayList<Geometry> arrGeoms = new ArrayList<>(processingData.getBoundaryList());
     // intentionally as check for P on Polygonal is already performed
     @SuppressWarnings({"unchecked"})

src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java

@@ -68,6 +68,7 @@
 import org.heigit.ohsome.ohsomeapi.output.ratio.RatioResult;
 import org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator;
 import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
+import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter;
 import org.heigit.ohsome.oshdb.OSHDBBoundingBox;
 import org.heigit.ohsome.oshdb.OSHDBTag;
 import org.heigit.ohsome.oshdb.OSHDBTimestamp;
@@ -87,7 +88,6 @@
 import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot;
 import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag;
 import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
-import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Lineal;
@@ -135,13 +135,14 @@ public static boolean snapshotMatches(OSMEntitySnapshot snapshot, Set<OSMType> o
       Set<SimpleFeatureType> simpleFeatureTypes, Integer[] keysInt, Integer[] valuesInt) {
     boolean matchesTags = true;
     OSMEntity entity = snapshot.getEntity();
+
     if (osmTypes.contains(entity.getType())) {
       for (int i = 0; i < keysInt.length; i++) {
         boolean matchesTag;
         if (i < valuesInt.length) {
-          matchesTag = entity.hasTagValue(keysInt[i], valuesInt[i]);
+          matchesTag = entity.getTags().hasTagValue(keysInt[i], valuesInt[i]);
         } else {
-          matchesTag = entity.hasTagKey(keysInt[i]);
+          matchesTag = entity.getTags().hasTagKey(keysInt[i]);
         }
         if (!matchesTag) {
           matchesTags = false;
@@ -404,7 +405,7 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge
     }
     if (includeOSMMetadata) {
       properties.put("@version", entity.getVersion());
-      properties.put("@osmType", entity.getType());
+      properties.put("@osmType", entity.getType().toString());
       properties.put("@changesetId", entity.getChangesetId());
       if (isContributionsEndpoint) {
         properties = addContributionTypes(properties, contributionTypes);

src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java

@@ -13,16 +13,17 @@
 import org.heigit.ohsome.ohsomeapi.exception.NotFoundException;
 import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.ohsome.ohsomeapi.oshdb.ExtractMetadata;
+import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter;
 import org.heigit.ohsome.oshdb.OSHDBTag;
 import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer;
+import org.heigit.ohsome.oshdb.filter.Filter;
 import org.heigit.ohsome.oshdb.filter.FilterExpression;
 import org.heigit.ohsome.oshdb.filter.FilterParser;
 import org.heigit.ohsome.oshdb.osm.OSMType;
 import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible;
 import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
 import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser;
 import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps;
-import org.heigit.ohsome.oshdb.util.time.TimestampFormatter;
 import org.jparsec.error.ParserException;
 import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.Lineal;
@@ -360,9 +361,9 @@ public <T extends OSHDBMapReducible> MapReducer<T> filterOnPlanarRelations(MapRe
     TagTranslator tt = DbConnData.tagTranslator;
     OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon");
     OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary");
-    return mapRed.osmEntityFilter(entity -> !entity.getType().equals(OSMType.RELATION)
-        || entity.hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue())
-        || entity.hasTagValue(typeBoundary.getKey(), typeBoundary.getValue()));
+    return mapRed.filter(Filter.byOSMEntity(entity -> !entity.getType().equals(OSMType.RELATION)
+        || entity.getTags().hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue())
+        || entity.getTags().hasTagValue(typeBoundary.getKey(), typeBoundary.getValue())));
   }
 
   /**