From d7656cf5f572f812ee9fe591286277bd722c4e42 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Wed, 8 Feb 2023 17:48:55 -0800 Subject: [PATCH] Add limit to geojson upload API Will limit upto 10_000 features to upload. Signed-off-by: Vijayan Balasubramanian --- .../geojson/UploadGeoJSONRequestContent.java | 28 +++++++++++++++++ .../UploadGeoJSONRequestContentTests.java | 31 ++++++++++++++----- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java index 306cc60b..f5a7e7ef 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java @@ -17,6 +17,7 @@ import org.opensearch.common.ParseField; import org.opensearch.common.Strings; +import org.opensearch.geospatial.GeospatialParser; /** * UploadGeoJSONRequestContent is the Data model for UploadGeoJSONRequest's body @@ -29,6 +30,10 @@ public final class UploadGeoJSONRequestContent { public static final ParseField FIELD_GEOSPATIAL = new ParseField("field"); public static final ParseField FIELD_GEOSPATIAL_TYPE = new ParseField("type"); public static final ParseField FIELD_DATA = new ParseField("data"); + + // Custom Vector Map can support fetching up to 10K Features. Hence, we chose same value as limit + // for upload as well. + public static final int MAX_SUPPORTED_GEOJSON_FEATURE_COUNT = 10_000; private final String indexName; private final String fieldName; private final String fieldType; @@ -61,9 +66,32 @@ public static UploadGeoJSONRequestContent create(Map input) { geoJSONData + " is not an instance of List, but of type [ " + geoJSONData.getClass().getName() + " ]" ); } + validateFeatureCount(geoJSONData); return new UploadGeoJSONRequestContent(index, fieldName, fieldType, (List) geoJSONData); } + private static void validateFeatureCount(Object geoJSONData) { + final long featureCount = getFeatureCount(geoJSONData); + if (featureCount > MAX_SUPPORTED_GEOJSON_FEATURE_COUNT) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Received %d features, but, cannot upload more than %d features", + featureCount, + MAX_SUPPORTED_GEOJSON_FEATURE_COUNT + ) + ); + } + } + + private static long getFeatureCount(Object geoJSONData) { + return ((List) geoJSONData).stream() + .map(GeospatialParser::toStringObjectMap) + .map(GeospatialParser::getFeatures) + .flatMap(List::stream) + .count(); + } + private static String validateIndexName(Map input) { String index = extractValueAsString(input, FIELD_INDEX.getPreferredName()); if (Strings.hasText(index)) { diff --git a/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContentTests.java b/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContentTests.java index 64a212e0..7486cc68 100644 --- a/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContentTests.java +++ b/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContentTests.java @@ -10,6 +10,7 @@ import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.FIELD_DATA; import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.GEOSPATIAL_DEFAULT_FIELD_NAME; +import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.MAX_SUPPORTED_GEOJSON_FEATURE_COUNT; import java.util.Collections; import java.util.Map; @@ -19,6 +20,7 @@ import org.opensearch.test.OpenSearchTestCase; public class UploadGeoJSONRequestContentTests extends OpenSearchTestCase { + private static int MIN_FEATURE_COUNT = 3; private String indexName; private String fieldName; @@ -29,21 +31,21 @@ public void setUp() throws Exception { fieldName = randomLowerCaseString(); } - private Map buildRequestContent(String indexName, String fieldName) { + private Map buildRequestContent(String indexName, String fieldName, int count) { final var contents = new JSONObject(); contents.put(UploadGeoJSONRequestContent.FIELD_INDEX.getPreferredName(), indexName); contents.put(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL.getPreferredName(), fieldName); contents.put(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL_TYPE.getPreferredName(), "geo_shape"); JSONArray values = new JSONArray(); - values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap()))); - values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap()))); - values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap()))); + for (int i = 0; i < count; i++) { + values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap()))); + } contents.put(FIELD_DATA.getPreferredName(), values); return contents.toMap(); } public void testCreate() { - Map contents = buildRequestContent(indexName, fieldName); + Map contents = buildRequestContent(indexName, fieldName, MIN_FEATURE_COUNT); final var content = UploadGeoJSONRequestContent.create(contents); assertNotNull(content); assertEquals(fieldName, content.getFieldName()); @@ -54,19 +56,32 @@ public void testCreate() { public void testCreateEmptyIndexName() { IllegalArgumentException invalidIndexName = assertThrows( IllegalArgumentException.class, - () -> UploadGeoJSONRequestContent.create(buildRequestContent("", "location")) + () -> UploadGeoJSONRequestContent.create(buildRequestContent("", "location", MIN_FEATURE_COUNT)) ); assertTrue(invalidIndexName.getMessage().contains("[ index ] cannot be empty")); } + public void testCreateWithOneMoreThanMaxSupportedFeatureCount() { + int featureCount = MAX_SUPPORTED_GEOJSON_FEATURE_COUNT + 1; + IllegalArgumentException reachedMaxFeatureCount = assertThrows( + IllegalArgumentException.class, + () -> UploadGeoJSONRequestContent.create(buildRequestContent(indexName, fieldName, featureCount)) + ); + assertEquals( + "wrong error returned", + reachedMaxFeatureCount.getMessage(), + "Received 10001 features, but, cannot upload more than 10000 features" + ); + } + public void testCreateEmptyGeospatialFieldName() { - final var content = UploadGeoJSONRequestContent.create(buildRequestContent(randomLowerCaseString(), "")); + final var content = UploadGeoJSONRequestContent.create(buildRequestContent(randomLowerCaseString(), "", MIN_FEATURE_COUNT)); assertNotNull(content); assertEquals("wrong field name", GEOSPATIAL_DEFAULT_FIELD_NAME, content.getFieldName()); } public void testCreateEmptyGeospatialFieldType() { - Map contents = buildRequestContent(indexName, fieldName); + Map contents = buildRequestContent(indexName, fieldName, MIN_FEATURE_COUNT); contents.remove(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL_TYPE.getPreferredName()); IllegalArgumentException invalidIndexName = assertThrows( IllegalArgumentException.class,