Skip to content

Commit

Permalink
Don't bound low precision geohex aggregations on vector tiles (elasti…
Browse files Browse the repository at this point in the history 
…c#93195)

This commit adds a mechanism so geohex aggregations with precision <= 3 are created with unbounded bounds.
  • Loading branch information
@iverase
iverase authored Jan 24, 2023
1 parent a1bf143 commit 1f5546e
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 9 deletions.
60 changes: 56 additions & 4 deletions ...ector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class VectorTileRestIT extends ESRestTestCase {
private static final String INDEX_POINTS = "index-points";
private static final String INDEX_POLYGON = "index-polygon";
private static final String INDEX_COLLECTION = "index-collection";
private static final String INDEX_BIG_POLYGON = "index-big-polygon";
private static final String INDEX_POINTS_SHAPES = INDEX_POINTS + "," + INDEX_POLYGON;
private static final String INDEX_ALL = "index*";
private static final String META_LAYER = "meta";
Expand All @@ -62,7 +63,8 @@ public void indexDocuments() throws IOException {
x = randomIntBetween(0, (1 << z) - 1);
y = randomIntBetween(0, (1 << z) - 1);
indexPoints();
indexShapes();
indexPolygon();
indexBigPolygon();
indexCollection();
oneTimeSetup = true;
}
Expand Down Expand Up @@ -106,11 +108,16 @@ private void indexPoints() throws IOException {
assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK));
}

private void indexShapes() throws IOException {
private void indexPolygon() throws IOException {
final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z);
createIndexAndPutGeometry(INDEX_POLYGON, toPolygon(r), "polygon");
}

private void indexBigPolygon() throws IOException {
final Rectangle r = new Rectangle(-180, 180, 90, -90);
createIndexAndPutGeometry(INDEX_BIG_POLYGON, toPolygon(r), "polygon");
}

private void createIndexAndPutGeometry(String indexName, Geometry geometry, String id) throws IOException {
final Request createRequest = new Request(HttpPut.METHOD_NAME, indexName);
Response response = client().performRequest(createRequest);
Expand Down Expand Up @@ -234,8 +241,8 @@ public void testIndexAllGet() throws Exception {
mvtRequest.setJsonEntity("{\"size\" : 100}");
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
// 33 points, 1 polygon and two from geometry collection
assertLayer(tile, HITS_LAYER, 4096, 36, 2);
// 33 points, 1 big polygon, 1 polygon and two from geometry collection
assertLayer(tile, HITS_LAYER, 4096, 37, 2);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
Expand Down Expand Up @@ -325,6 +332,51 @@ public void testGridPrecision() throws Exception {
}
}

public void testGridPrecisionGeoTile() throws Exception {
final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM - 10);
final int x = randomIntBetween(0, (1 << z) - 1);
final int y = randomIntBetween(0, (1 << z) - 1);
for (int i = 1; i <= 8; i++) {
final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/" + z + "/" + x + "/" + y);
mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geotile\", \"grid_precision\" : " + i + " }");
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, (1 << i) * (1 << i), 2);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
}

public void testGridPrecisionGeoHex() throws Exception {
// the number of hex depends on the position of the tile, therefore we just check some of them.
final int[] expected_zoom_0 = new int[] { 122, 122, 842, 842, 5872, 5872, 41058, 41058 };
for (int i = 1; i <= 8; i++) {
final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/0/0/0");
mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_0[i - 1], 2);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
final int[] expected_zoom_1 = new int[] { 45, 241, 241, 1559, 1559, 10531, 10531, 10531 };
for (int i = 1; i <= 8; i++) {
final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/1/0/0");
mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_1[i - 1], 2);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
final int[] expected_zoom_5 = new int[] { 12, 55, 55, 55, 292, 292, 1873, 12673 };
for (int i = 1; i <= 8; i++) {
final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/5/16/8");
mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_5[i - 1], 2);
assertLayer(tile, META_LAYER, 4096, 1, 13);
}
}

public void testWithLabels() throws Exception {
final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
mvtRequest.setJsonEntity("{\"size\" : 100, \"with_labels\": true}");
Expand Down
22 changes: 22 additions & 0 deletions ...in/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) throws IOE
public Rectangle toRectangle(String bucketKey) {
return GeoTileUtils.toBoundingBox(bucketKey);
}

@Override
public boolean needsBounding(int z, int gridPrecision) {
// we always bound it as it is pretty efficient anyway.
return true;
}
},
GEOHEX {

Expand Down Expand Up @@ -174,6 +180,17 @@ public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) {
public Rectangle toRectangle(String bucketKey) {
return H3CartesianUtil.toBoundingBox(H3.stringToH3(bucketKey));
}

@Override
public boolean needsBounding(int z, int gridPrecision) {
/*
Bounded geohex aggregation can be expensive, in particular where there is lots of data outside the bounding
box. Because we are buffering our queries, this is magnified for low precision tiles. Because the total number
of buckets up to precision 3 is lower than the default max buckets, we better not bound those aggregations
which results in much better performance.
*/
return gridPrecisionToAggPrecision(z, gridPrecision) > 3;
}
};

/**
Expand Down Expand Up @@ -203,6 +220,11 @@ public Rectangle toRectangle(String bucketKey) {
*/
public abstract Rectangle toRectangle(String bucketKey);

/**
* If false, the aggregation at the given zoom and grid precision is not bound.
*/
public abstract boolean needsBounding(int z, int gridPrecision);

public static GridAggregation fromString(String type) {
return switch (type.toLowerCase(Locale.ROOT)) {
case "geotile" -> GEOTILE;
Expand Down
16 changes: 11 additions & 5 deletions ...ctor-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,11 +215,17 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie
}
searchRequestBuilder.setQuery(qBuilder);
if (request.getGridPrecision() > 0) {
final Rectangle rectangle = request.getBoundingBox();
final GeoBoundingBox boundingBox = new GeoBoundingBox(
new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()),
new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())
);
final GeoBoundingBox boundingBox;
if (request.getGridAgg().needsBounding(request.getZ(), request.getGridPrecision())) {
final Rectangle rectangle = request.getBoundingBox();
boundingBox = new GeoBoundingBox(
new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()),
new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())
);
} else {
// unbounded
boundingBox = new GeoBoundingBox(new GeoPoint(Double.NaN, Double.NaN), new GeoPoint(Double.NaN, Double.NaN));
}
final GeoGridAggregationBuilder tileAggBuilder = request.getGridAgg()
.newAgg(GRID_FIELD)
.field(request.getField())
Expand Down

0 comments on commit 1f5546e

Please sign in to comment.