Skip to content

Commit

Permalink
Add limit to geojson upload API
Browse files Browse the repository at this point in the history 
Will limit upto 10_000 features to upload.

Signed-off-by: Vijayan Balasubramanian <[email protected]>
  • Loading branch information
@VijayanB
VijayanB committed Feb 15, 2023
1 parent a06d39b commit 0d5763f
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 8 deletions.
28 changes: 28 additions & 0 deletions ...ain/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import org.opensearch.common.ParseField;
import org.opensearch.common.Strings;
import org.opensearch.geospatial.GeospatialParser;

/**
* UploadGeoJSONRequestContent is the Data model for UploadGeoJSONRequest's body
Expand All @@ -29,6 +30,10 @@ public final class UploadGeoJSONRequestContent {
public static final ParseField FIELD_GEOSPATIAL = new ParseField("field");
public static final ParseField FIELD_GEOSPATIAL_TYPE = new ParseField("type");
public static final ParseField FIELD_DATA = new ParseField("data");

// Custom Vector Map can support fetching up to 10K Features. Hence, we chose same value as limit
// for upload as well.
public static final int MAX_SUPPORTED_GEOJSON_FEATURE_COUNT = 10_000;
private final String indexName;
private final String fieldName;
private final String fieldType;
Expand Down Expand Up @@ -61,9 +66,32 @@ public static UploadGeoJSONRequestContent create(Map<String, Object> input) {
geoJSONData + " is not an instance of List, but of type [ " + geoJSONData.getClass().getName() + " ]"
);
}
validateFeatureCount(geoJSONData);
return new UploadGeoJSONRequestContent(index, fieldName, fieldType, (List<Object>) geoJSONData);
}

private static void validateFeatureCount(Object geoJSONData) {
final long featureCount = getFeatureCount(geoJSONData);
if (featureCount > MAX_SUPPORTED_GEOJSON_FEATURE_COUNT) {
throw new IllegalArgumentException(
String.format(
Locale.ROOT,
"Received %d features, but, cannot upload more than %d features",
featureCount,
MAX_SUPPORTED_GEOJSON_FEATURE_COUNT
)
);
}
}

private static long getFeatureCount(Object geoJSONData) {
return ((List<Object>) geoJSONData).stream()
.map(GeospatialParser::toStringObjectMap)
.map(GeospatialParser::getFeatures)
.flatMap(List::stream)
.count();
}

private static String validateIndexName(Map<String, Object> input) {
String index = extractValueAsString(input, FIELD_INDEX.getPreferredName());
if (Strings.hasText(index)) {
Expand Down
31 changes: 23 additions & 8 deletions ...ava/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContentTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString;
import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.FIELD_DATA;
import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.GEOSPATIAL_DEFAULT_FIELD_NAME;
import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.MAX_SUPPORTED_GEOJSON_FEATURE_COUNT;

import java.util.Collections;
import java.util.Map;
Expand All @@ -19,6 +20,7 @@
import org.opensearch.test.OpenSearchTestCase;

public class UploadGeoJSONRequestContentTests extends OpenSearchTestCase {
private static int MIN_FEATURE_COUNT = 3;
private String indexName;
private String fieldName;

Expand All @@ -29,21 +31,21 @@ public void setUp() throws Exception {
fieldName = randomLowerCaseString();
}

private Map<String, Object> buildRequestContent(String indexName, String fieldName) {
private Map<String, Object> buildRequestContent(String indexName, String fieldName, int count) {
final var contents = new JSONObject();
contents.put(UploadGeoJSONRequestContent.FIELD_INDEX.getPreferredName(), indexName);
contents.put(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL.getPreferredName(), fieldName);
contents.put(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL_TYPE.getPreferredName(), "geo_shape");
JSONArray values = new JSONArray();
values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap())));
values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap())));
values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap())));
for (int i = 0; i < count; i++) {
values.put(randomGeoJSONFeature(buildProperties(Collections.emptyMap())));
}
contents.put(FIELD_DATA.getPreferredName(), values);
return contents.toMap();
}

public void testCreate() {
Map<String, Object> contents = buildRequestContent(indexName, fieldName);
Map<String, Object> contents = buildRequestContent(indexName, fieldName, MIN_FEATURE_COUNT);
final var content = UploadGeoJSONRequestContent.create(contents);
assertNotNull(content);
assertEquals(fieldName, content.getFieldName());
Expand All @@ -54,19 +56,32 @@ public void testCreate() {
public void testCreateEmptyIndexName() {
IllegalArgumentException invalidIndexName = assertThrows(
IllegalArgumentException.class,
() -> UploadGeoJSONRequestContent.create(buildRequestContent("", "location"))
() -> UploadGeoJSONRequestContent.create(buildRequestContent("", "location", MIN_FEATURE_COUNT))
);
assertTrue(invalidIndexName.getMessage().contains("[ index ] cannot be empty"));
}

public void testCreateWithOneMoreThanMaxSupportedFeatureCount() {
int featureCount = MAX_SUPPORTED_GEOJSON_FEATURE_COUNT + 1;
IllegalArgumentException reachedMaxFeatureCount = assertThrows(
IllegalArgumentException.class,
() -> UploadGeoJSONRequestContent.create(buildRequestContent(indexName, fieldName, featureCount))
);
assertEquals(
"wrong error returned",
reachedMaxFeatureCount.getMessage(),
"Received 10001 features, but, cannot upload more than 10000 features"
);
}

public void testCreateEmptyGeospatialFieldName() {
final var content = UploadGeoJSONRequestContent.create(buildRequestContent(randomLowerCaseString(), ""));
final var content = UploadGeoJSONRequestContent.create(buildRequestContent(randomLowerCaseString(), "", MIN_FEATURE_COUNT));
assertNotNull(content);
assertEquals("wrong field name", GEOSPATIAL_DEFAULT_FIELD_NAME, content.getFieldName());
}

public void testCreateEmptyGeospatialFieldType() {
Map<String, Object> contents = buildRequestContent(indexName, fieldName);
Map<String, Object> contents = buildRequestContent(indexName, fieldName, MIN_FEATURE_COUNT);
contents.remove(UploadGeoJSONRequestContent.FIELD_GEOSPATIAL_TYPE.getPreferredName());
IllegalArgumentException invalidIndexName = assertThrows(
IllegalArgumentException.class,
Expand Down

0 comments on commit 0d5763f

Please sign in to comment.