From 49565b2aec2a6b216cdebc50a9ae5d73ccfce8d3 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Tue, 25 Apr 2023 18:18:19 -0700 Subject: [PATCH 01/61] Update gradle version to 7.6 (#265) Signed-off-by: Vijayan Balasubramanian --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661e..070cb702 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From ba16116ccf78578cebeeead503aa6064afc8193d Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 26 Apr 2023 14:43:24 -0700 Subject: [PATCH 02/61] Exclude lombok generated code from jacoco coverage report (#268) Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + lombok.config | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 lombok.config diff --git a/CHANGELOG.md b/CHANGELOG.md index 27dacfa9..421c7a65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance ### Refactoring diff --git a/lombok.config b/lombok.config new file mode 100644 index 00000000..9745d1ed --- /dev/null +++ b/lombok.config @@ -0,0 +1,5 @@ +# tell lombok this is your root directory +config.stopBubbling = true +# add @lombok.Generated annotations to all generated nodes where possible +# to skip code coverage for auto generated code +lombok.addLombokGeneratedAnnotation = true From 35246835d1edce548be0f39993b5d309e00296a8 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 26 Apr 2023 14:44:13 -0700 Subject: [PATCH 03/61] Make jacoco report to be generated faster in local (#267) Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 421c7a65..0868a2b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Make jacoco report to be generated faster in local ([#267](https://github.com/opensearch-project/geospatial/pull/267)) * Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance diff --git a/build.gradle b/build.gradle index 1ae2da18..bcc2fc33 100644 --- a/build.gradle +++ b/build.gradle @@ -261,7 +261,7 @@ spotless { } jacocoTestReport { - dependsOn integTest, test, yamlRestTest + dependsOn test reports { xml.enabled = true html.enabled = true From d9884b36ba271b2a4cc00a13ccc31d4bdc153cc4 Mon Sep 17 00:00:00 2001 From: "mend-for-github.aaakk.us.kg[bot]" <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 13:46:39 -0700 Subject: [PATCH 04/61] Update dependency org.json:json to v20230227 (#273) Co-authored-by: mend-for-github.aaakk.us.kg[bot] <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index bcc2fc33..92f1b13e 100644 --- a/build.gradle +++ b/build.gradle @@ -150,7 +150,7 @@ dependencies { api project(":libs:h3") yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" - testImplementation 'org.json:json:20211205' + testImplementation 'org.json:json:20230227' implementation "org.apache.commons:commons-lang3:3.12.0" implementation "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}" implementation "org.locationtech.jts:jts-core:${versions.jts}" From b9db1c7480cb092fff4fe8a8cc272d5e950de607 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Tue, 2 May 2023 13:50:55 -0700 Subject: [PATCH 05/61] Baseline owners and maintainers (#275) Signed-off-by: Vijayan Balasubramanian --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 855a0949..f1b13fd6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @opensearch-project/geospatial \ No newline at end of file +* @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski diff --git a/MAINTAINERS.md b/MAINTAINERS.md index f319b09a..f030c04e 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -4,8 +4,13 @@ This document contains a list of maintainers in this repo. See [opensearch-proje ## Current Maintainers -| Maintainer | GitHub ID | Affiliation | -| ----------------------- | ------------------------------------------- | ----------- | -| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | -| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | -| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | +| Maintainer | GitHub ID | Affiliation | +|-------------------------|-------------------------------------------------------|-------------| +| Heemin Kim | [heemin32](https://github.com/heemin32) | Amazon | +| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | +| Junqiu Lei | [junqiu-lei](https://github.com/junqiu-lei) | Amazon | +| Martin Gaievski | [martin-gaievski](https://github.com/martin-gaievski) | Amazon | +| Naveen Tatikonda | [naveentatikonda](https://github.com/naveentatikonda) | Amazon | +| Navneet Verma | [navneet1v](https://github.com/navneet1v) | Amazon | +| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | +| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | From 7ee2f133879ce234408e2467d061924d07cf5129 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Fri, 5 May 2023 19:16:37 -0500 Subject: [PATCH 06/61] Add Auto Release Workflow (#288) Signed-off-by: Naveen Tatikonda --- .github/workflows/auto-release.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 00000000..b0bd22ed --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,28 @@ +name: Releases + +on: + push: + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Get tag + id: tag + uses: dawidd6/action-get-tag@v1 + - uses: actions/checkout@v2 + - uses: ncipollo/release-action@v1 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + bodyFile: release-notes/opensearch-geospatial.release-notes-${{steps.tag.outputs.tag}}.md From 55e77981281cd33fc0f297e981fa86f7caa47b43 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 19 May 2023 09:55:41 -0700 Subject: [PATCH 07/61] Change package for Strings.hasText (#314) Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + .../geospatial/action/upload/geojson/ContentBuilder.java | 2 +- .../upload/geojson/UploadGeoJSONRequestContent.java | 2 +- .../opensearch/geospatial/stats/upload/UploadMetric.java | 2 +- .../org/opensearch/geospatial/GeospatialRestTestCase.java | 8 ++++---- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0868a2b3..8befb4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,4 +21,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance +* Change package for Strings.hasText ([#314](https://github.com/opensearch-project/geospatial/pull/314)) ### Refactoring diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/ContentBuilder.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/ContentBuilder.java index 551b1c06..66e708ed 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/ContentBuilder.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/ContentBuilder.java @@ -14,7 +14,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.geospatial.GeospatialParser; /** diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java index 0bcde2ab..eae60881 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestContent.java @@ -15,8 +15,8 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; -import org.opensearch.common.Strings; import org.opensearch.core.ParseField; +import org.opensearch.core.common.Strings; import org.opensearch.geospatial.GeospatialParser; /** diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java index 0b744590..2dacef52 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java @@ -11,10 +11,10 @@ import lombok.Getter; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 4ca9cfec..e787fc7e 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -33,7 +33,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; import org.opensearch.common.geo.GeoJson; import org.opensearch.common.geo.ShapeRelation; @@ -41,6 +40,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Geometry; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; @@ -87,7 +87,7 @@ protected static void createPipeline(String name, Optional description, builder.endObject(); Request request = new Request("PUT", buildPipelinePath(name)); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(org.opensearch.common.Strings.toString(builder)); client().performRequest(request); } @@ -102,7 +102,7 @@ protected static void createIndex(String name, Settings settings, Map XContentBuilder builder = JsonXContent.contentBuilder().startObject(); build.accept(builder); builder.endObject(); - return Strings.toString(builder); + return org.opensearch.common.Strings.toString(builder); } public String buildSearchAggregationsBodyAsString(CheckedConsumer aggregationsBuilder) From 78e400574e334eb02aef4068ec2842ec916bb306 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Fri, 26 May 2023 08:35:59 -0700 Subject: [PATCH 08/61] Adding release notes for 2.8 (#323) Signed-off-by: Martin Gaievski --- CHANGELOG.md | 5 +---- .../opensearch-geospatial.release-notes-2.8.0.0.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 release-notes/opensearch-geospatial.release-notes-2.8.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8befb4d9..18ac4b48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,14 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.7...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.8...2.x) ### Features ### Enhancements ### Bug Fixes ### Infrastructure -* Make jacoco report to be generated faster in local ([#267](https://github.com/opensearch-project/geospatial/pull/267)) -* Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance -* Change package for Strings.hasText ([#314](https://github.com/opensearch-project/geospatial/pull/314)) ### Refactoring diff --git a/release-notes/opensearch-geospatial.release-notes-2.8.0.0.md b/release-notes/opensearch-geospatial.release-notes-2.8.0.0.md new file mode 100644 index 00000000..00281437 --- /dev/null +++ b/release-notes/opensearch-geospatial.release-notes-2.8.0.0.md @@ -0,0 +1,10 @@ +## Version 2.8.0.0 Release Notes + +Compatible with OpenSearch 2.8.0 + +### Infrastructure +* Make jacoco report to be generated faster in local ([#267](https://github.com/opensearch-project/geospatial/pull/267)) +* Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) + +### Maintenance +* Change package for Strings.hasText ([#314](https://github.com/opensearch-project/geospatial/pull/314)) \ No newline at end of file From ee06307be8e3155732f3db22e50437dca9b77de6 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Tue, 11 Jul 2023 11:42:39 -0500 Subject: [PATCH 09/61] Add 2.9.0 release notes (#350) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 2 +- .../opensearch-geospatial.release-notes-2.9.0.0.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 release-notes/opensearch-geospatial.release-notes-2.9.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ac4b48..5a19cc0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.8...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.9...2.x) ### Features ### Enhancements ### Bug Fixes diff --git a/release-notes/opensearch-geospatial.release-notes-2.9.0.0.md b/release-notes/opensearch-geospatial.release-notes-2.9.0.0.md new file mode 100644 index 00000000..baa64e3c --- /dev/null +++ b/release-notes/opensearch-geospatial.release-notes-2.9.0.0.md @@ -0,0 +1,9 @@ +## Version 2.9.0.0 Release Notes + +Compatible with OpenSearch 2.9.0 + +### Maintenance +Increment version to 2.9.0-SNAPSHOT ([#329](https://github.com/opensearch-project/geospatial/pull/329)) + +### Refactoring +Change package for Strings.hasText ([#314](https://github.com/opensearch-project/geospatial/pull/314)) From 806755fc1759543bbb0d997679d391afcf30ae14 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 13 Jul 2023 13:58:12 -0700 Subject: [PATCH 10/61] Update packages according to a change in OpenSearch core (#353) Signed-off-by: Heemin Kim --- .../java/org/opensearch/geospatial/GeospatialParser.java | 2 +- .../geospatial/action/upload/geojson/PipelineManager.java | 2 +- .../action/upload/geojson/UploadGeoJSONRequest.java | 6 +++--- .../action/upload/geojson/UploadGeoJSONResponse.java | 4 ++-- .../geospatial/index/query/xyshape/XYShapeQueryBuilder.java | 2 +- .../org/opensearch/geospatial/plugin/GeospatialPlugin.java | 2 +- .../rest/action/upload/geojson/RestUploadGeoJSONAction.java | 2 +- .../search/aggregations/bucket/geogrid/GeoHexGrid.java | 2 +- .../bucket/geogrid/GeoHexGridAggregationBuilder.java | 2 +- .../aggregations/bucket/geogrid/GeoHexGridBucket.java | 2 +- .../opensearch/geospatial/stats/upload/UploadMetric.java | 6 +++--- .../org/opensearch/geospatial/stats/upload/UploadStats.java | 6 +++--- .../geospatial/stats/upload/UploadStatsNodeRequest.java | 4 ++-- .../geospatial/stats/upload/UploadStatsNodeResponse.java | 4 ++-- .../geospatial/stats/upload/UploadStatsRequest.java | 2 +- .../geospatial/stats/upload/UploadStatsResponse.java | 6 +++--- .../geospatial/stats/upload/UploadStatsTransportAction.java | 2 +- .../org/opensearch/geospatial/GeospatialParserTests.java | 2 +- .../org/opensearch/geospatial/GeospatialRestTestCase.java | 2 +- .../org/opensearch/geospatial/GeospatialTestHelper.java | 2 +- .../action/upload/geojson/UploadGeoJSONRequestTests.java | 4 ++-- .../geospatial/index/mapper/xypoint/XYPointParserTests.java | 2 +- .../index/query/xyshape/XYShapeQueryBuilderTests.java | 2 +- .../opensearch/geospatial/plugin/GeospatialPluginIT.java | 2 +- .../opensearch/geospatial/processor/FeatureProcessorIT.java | 2 +- .../action/upload/geojson/RestUploadGeoJSONActionIT.java | 2 +- .../geospatial/stats/upload/RestUploadStatsActionIT.java | 2 +- .../geospatial/stats/upload/UploadMetricTests.java | 2 +- .../stats/upload/UploadStatsNodeResponseTests.java | 2 +- .../geospatial/stats/upload/UploadStatsResponseTests.java | 2 +- .../geospatial/stats/upload/UploadStatsTests.java | 2 +- 31 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/GeospatialParser.java b/src/main/java/org/opensearch/geospatial/GeospatialParser.java index d0d791d0..04c81efd 100644 --- a/src/main/java/org/opensearch/geospatial/GeospatialParser.java +++ b/src/main/java/org/opensearch/geospatial/GeospatialParser.java @@ -13,9 +13,9 @@ import java.util.Map; import java.util.Objects; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.geospatial.geojson.Feature; import org.opensearch.geospatial.geojson.FeatureCollection; diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/PipelineManager.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/PipelineManager.java index 09136850..4a485983 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/PipelineManager.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/PipelineManager.java @@ -18,9 +18,9 @@ import org.opensearch.action.ingest.PutPipelineRequest; import org.opensearch.client.ClusterAdminClient; import org.opensearch.common.UUIDs; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.ingest.Pipeline; diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequest.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequest.java index 8bc8df5c..fc05a2ae 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequest.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequest.java @@ -20,9 +20,9 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.rest.RestRequest; @AllArgsConstructor diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONResponse.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONResponse.java index 68707531..f1cd8616 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONResponse.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONResponse.java @@ -13,8 +13,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.action.bulk.BulkItemResponse; import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilder.java b/src/main/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilder.java index 433b361d..f1221a4a 100644 --- a/src/main/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilder.java +++ b/src/main/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilder.java @@ -13,7 +13,7 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.opensearch.common.geo.parsers.ShapeParser; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geometry.Geometry; diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 1d2ab65c..085bbf92 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -17,11 +17,11 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; diff --git a/src/main/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONAction.java b/src/main/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONAction.java index 5f211e31..4e4034d0 100644 --- a/src/main/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONAction.java +++ b/src/main/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONAction.java @@ -13,9 +13,9 @@ import java.util.List; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequest; import org.opensearch.rest.BaseRestHandler; diff --git a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGrid.java b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGrid.java index 9b336f86..6199d016 100644 --- a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGrid.java +++ b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGrid.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Map; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geo.search.aggregations.bucket.geogrid.BaseGeoGrid; import org.opensearch.geo.search.aggregations.bucket.geogrid.BaseGeoGridBucket; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java index 0a640cf6..018be48e 100644 --- a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java +++ b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java @@ -12,8 +12,8 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; diff --git a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridBucket.java b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridBucket.java index 29c5b80a..40c7bdc9 100644 --- a/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridBucket.java +++ b/src/main/java/org/opensearch/geospatial/search/aggregations/bucket/geogrid/GeoHexGridBucket.java @@ -9,7 +9,7 @@ import java.io.IOException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geo.search.aggregations.bucket.geogrid.BaseGeoGridBucket; import org.opensearch.geospatial.h3.H3; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java index 2dacef52..9f8b8e58 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadMetric.java @@ -11,10 +11,10 @@ import lombok.Getter; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java index cd4224e9..c2afd22b 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java @@ -12,10 +12,10 @@ import java.util.Objects; import java.util.Set; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java index 47abf9ef..e18f6195 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java @@ -7,8 +7,8 @@ import java.io.IOException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.transport.TransportRequest; public class UploadStatsNodeRequest extends TransportRequest { diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java index 8b6907d7..3baa29a1 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java @@ -10,8 +10,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java index 9bcacc99..5c3efa9d 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java @@ -8,7 +8,7 @@ import java.io.IOException; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; public class UploadStatsRequest extends BaseNodesRequest { diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java index a8b9b674..c8f0fe76 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java @@ -14,9 +14,9 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java index 5caa3ac0..869eecb4 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java @@ -13,7 +13,7 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/test/java/org/opensearch/geospatial/GeospatialParserTests.java b/src/test/java/org/opensearch/geospatial/GeospatialParserTests.java index 71c86382..d7de97b9 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialParserTests.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialParserTests.java @@ -12,7 +12,7 @@ import org.json.JSONArray; import org.json.JSONObject; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.geospatial.geojson.Feature; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index e787fc7e..1e5c1439 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -41,6 +41,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.Geometry; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; @@ -49,7 +50,6 @@ import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.ingest.Pipeline; -import org.opensearch.rest.RestStatus; public abstract class GeospatialRestTestCase extends OpenSearchSecureRestTestCase { diff --git a/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java b/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java index bc3d28a6..9f455a1e 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java @@ -39,11 +39,11 @@ import org.opensearch.common.Randomness; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.geospatial.action.upload.geojson.ContentBuilder; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; import org.opensearch.geospatial.h3.H3; import org.opensearch.geospatial.stats.upload.UploadMetric; -import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.RandomObjects; diff --git a/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestTests.java b/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestTests.java index d973192a..0a591f2c 100644 --- a/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONRequestTests.java @@ -13,9 +13,9 @@ import java.nio.charset.StandardCharsets; import org.json.JSONObject; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointParserTests.java b/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointParserTests.java index 6f322e9f..b7d8c97e 100644 --- a/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointParserTests.java +++ b/src/test/java/org/opensearch/geospatial/index/mapper/xypoint/XYPointParserTests.java @@ -10,8 +10,8 @@ import java.io.IOException; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilderTests.java b/src/test/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilderTests.java index e128f797..60103866 100644 --- a/src/test/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilderTests.java +++ b/src/test/java/org/opensearch/geospatial/index/query/xyshape/XYShapeQueryBuilderTests.java @@ -28,12 +28,12 @@ import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.common.Strings; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.geo.GeoJson; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geometry.Geometry; diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java index daf9b913..e28372dc 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginIT.java @@ -8,8 +8,8 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; -import org.opensearch.rest.RestStatus; public class GeospatialPluginIT extends GeospatialRestTestCase { diff --git a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java index 8e256aae..585f8628 100644 --- a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java @@ -21,8 +21,8 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; -import org.opensearch.rest.RestStatus; public class FeatureProcessorIT extends GeospatialRestTestCase { diff --git a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java index 53d12c84..167536e8 100644 --- a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java +++ b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java @@ -21,8 +21,8 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; -import org.opensearch.rest.RestStatus; public class RestUploadGeoJSONActionIT extends GeospatialRestTestCase { diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java index 303e7ccf..86d4168e 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java @@ -12,8 +12,8 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; -import org.opensearch.rest.RestStatus; public class RestUploadStatsActionIT extends GeospatialRestTestCase { diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadMetricTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadMetricTests.java index 4a6599f7..c69ca8e3 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/UploadMetricTests.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadMetricTests.java @@ -13,8 +13,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java index e453f03b..76b9587e 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java @@ -8,7 +8,7 @@ import java.io.IOException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java index ab22bc5d..e6fec0aa 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java @@ -20,8 +20,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsTests.java index baa4ff77..4177d5e4 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsTests.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsTests.java @@ -15,8 +15,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.GeospatialTestHelper; From 3c6dc552933c2acf2a59af6b4802c17908d849ad Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 25 Apr 2023 21:37:55 -0700 Subject: [PATCH 11/61] Implement creation of ip2geo feature (#257) * Update gradle version to 7.6 (#265) Signed-off-by: Vijayan Balasubramanian * Implement creation of ip2geo feature * Implementation of ip2geo datasource creation * Implementation of ip2geo processor creation Signed-off-by: Heemin Kim --------- Signed-off-by: Vijayan Balasubramanian Signed-off-by: Heemin Kim Co-authored-by: Vijayan Balasubramanian --- build.gradle | 55 +- .../ip2geo/action/PutDatasourceAction.java | 30 + .../ip2geo/action/PutDatasourceRequest.java | 168 ++++++ .../action/PutDatasourceTransportAction.java | 189 +++++++ .../action/RestPutDatasourceHandler.java | 81 +++ .../ip2geo/common/DatasourceHelper.java | 122 ++++ .../ip2geo/common/DatasourceManifest.java | 136 +++++ .../ip2geo/common/DatasourceState.java | 40 ++ .../ip2geo/common/GeoIpDataHelper.java | 354 ++++++++++++ .../ip2geo/common/Ip2GeoExecutorHelper.java | 44 ++ .../ip2geo/common/Ip2GeoSettings.java | 137 +++++ .../ip2geo/jobscheduler/Datasource.java | 528 ++++++++++++++++++ .../jobscheduler/DatasourceExtension.java | 47 ++ .../ip2geo/jobscheduler/DatasourceRunner.java | 278 +++++++++ .../ip2geo/processor/Ip2GeoCache.java | 112 ++++ .../ip2geo/processor/Ip2GeoProcessor.java | 430 ++++++++++++++ .../geospatial/plugin/GeospatialPlugin.java | 52 +- .../plugin-metadata/plugin-security.policy | 12 + ...rch.jobscheduler.spi.JobSchedulerExtension | 11 + .../resources/mappings/ip2geo_datasource.json | 9 + .../action/PutDatasourceRequestTests.java | 40 ++ .../action/RestPutDatasourceHandlerTests.java | 74 +++ .../ip2geo/common/DatasourceHelperTests.java | 78 +++ .../ip2geo/common/GeoIpDataHelperTests.java | 22 + .../ip2geo/common/Ip2GeoSettingsTests.java | 24 + .../ip2geo/jobscheduler/DatasourceTests.java | 64 +++ .../ip2geo/processor/Ip2GeoCacheTests.java | 58 ++ .../plugin/GeospatialPluginTests.java | 49 +- .../resources/rest-api-spec/test/10_basic.yml | 4 +- 29 files changed, 3236 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java create mode 100644 src/main/plugin-metadata/plugin-security.policy create mode 100644 src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension create mode 100644 src/main/resources/mappings/ip2geo_datasource.json create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java diff --git a/build.gradle b/build.gradle index 92f1b13e..ca35448a 100644 --- a/build.gradle +++ b/build.gradle @@ -5,6 +5,8 @@ import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable + apply plugin: 'java' apply plugin: 'idea' apply plugin: 'opensearch.opensearchplugin' @@ -35,6 +37,7 @@ opensearchplugin { classname "${projectPath}.${pathToPlugin}.${pluginClassName}" licenseFile rootProject.file('LICENSE') noticeFile rootProject.file('NOTICE') + extendedPlugins = ['opensearch-job-scheduler'] } // This requires an additional Jar not published as part of build-tools @@ -142,6 +145,10 @@ publishing { } +configurations { + zipArchive +} + //****************************************************************************/ // Dependencies //****************************************************************************/ @@ -154,6 +161,9 @@ dependencies { implementation "org.apache.commons:commons-lang3:3.12.0" implementation "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}" implementation "org.locationtech.jts:jts-core:${versions.jts}" + implementation "org.apache.commons:commons-csv:1.10.0" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" } licenseHeaders.enabled = true @@ -206,8 +216,6 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore @@ -220,6 +228,49 @@ testClusters.integTest { debugPort += 1 } } + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.getSingleFile() + } + } + } + })) + + // opensearch-geospatial plugin is being added to the list of plugins for the testCluster during build before + // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug. + // + // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opensearch-job-scheduler], dependency of [opensearch-geospatial] + // at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:515) + // + // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed. + // See https://github.com/opensearch-project/anomaly-detection/blob/fd547014fdde5114bbc9c8e49fe7aaa37eb6e793/build.gradle#L400-L422 + nodes.each { node -> + def plugins = node.plugins + def firstPlugin = plugins.get(0) + plugins.remove(0) + plugins.add(firstPlugin) + } +} + +testClusters.yamlRestTest { + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.getSingleFile() + } + } + } + })) } run { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java new file mode 100644 index 00000000..8db6278f --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.AcknowledgedResponse; + +/** + * Ip2Geo datasource creation action + */ +public class PutDatasourceAction extends ActionType { + /** + * Put datasource action instance + */ + public static final PutDatasourceAction INSTANCE = new PutDatasourceAction(); + /** + * Put datasource action name + */ + public static final String NAME = "cluster:admin/geospatial/datasource/put"; + + private PutDatasourceAction() { + super(NAME, AcknowledgedResponse::new); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java new file mode 100644 index 00000000..24266b0d --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -0,0 +1,168 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Locale; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.log4j.Log4j2; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; + +/** + * GeoIP datasource creation request + */ +@Getter +@Setter +@Log4j2 +public class PutDatasourceRequest extends AcknowledgedRequest { + private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + /** + * @param datasourceName the datasource name + * @return the datasource name + */ + private String datasourceName; + /** + * @param endpoint url to a manifest file for a datasource + * @return url to a manifest file for a datasource + */ + private String endpoint; + /** + * @param updateIntervalInDays update interval of a datasource + * @return update interval of a datasource + */ + private TimeValue updateIntervalInDays; + + /** + * Parser of a datasource + */ + public static final ObjectParser PARSER; + static { + PARSER = new ObjectParser<>("put_datasource"); + PARSER.declareString((request, val) -> request.setEndpoint(val), ENDPOINT_FIELD); + PARSER.declareLong((request, val) -> request.setUpdateIntervalInDays(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); + } + + /** + * Default constructor + * @param datasourceName name of a datasource + */ + public PutDatasourceRequest(final String datasourceName) { + this.datasourceName = datasourceName; + } + + /** + * Constructor with stream input + * @param in the stream input + * @throws IOException IOException + */ + public PutDatasourceRequest(final StreamInput in) throws IOException { + super(in); + this.datasourceName = in.readString(); + this.endpoint = in.readString(); + this.updateIntervalInDays = in.readTimeValue(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datasourceName); + out.writeString(endpoint); + out.writeTimeValue(updateIntervalInDays); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + validateEndpoint(errors); + validateUpdateInterval(errors); + return errors.validationErrors().isEmpty() ? null : errors; + } + + /** + * Conduct following validation on endpoint + * 1. endpoint format complies with RFC-2396 + * 2. validate manifest file from the endpoint + * + * @param errors the errors to add error messages + */ + private void validateEndpoint(final ActionRequestValidationException errors) { + try { + URL url = new URL(endpoint); + url.toURI(); // Validate URL complies with RFC-2396 + validateManifestFile(url, errors); + } catch (MalformedURLException | URISyntaxException e) { + log.info("Invalid URL[{}] is provided", endpoint, e); + errors.addValidationError("Invalid URL format is provided"); + } + } + + /** + * Conduct following validation on url + * 1. can read manifest file from the endpoint + * 2. the url in the manifest file complies with RFC-2396 + * 3. updateIntervalInDays is less than validForInDays value in the manifest file + * + * @param url the url to validate + * @param errors the errors to add error messages + */ + private void validateManifestFile(final URL url, final ActionRequestValidationException errors) { + DatasourceManifest manifest; + try { + manifest = DatasourceManifest.Builder.build(url); + } catch (Exception e) { + log.info("Error occurred while reading a file from {}", url, e); + errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s", url)); + return; + } + + try { + new URL(manifest.getUrl()).toURI(); // Validate URL complies with RFC-2396 + } catch (MalformedURLException | URISyntaxException e) { + log.info("Invalid URL[{}] is provided for url field in the manifest file", manifest.getUrl(), e); + errors.addValidationError("Invalid URL format is provided for url field in the manifest file"); + return; + } + + if (updateIntervalInDays.days() >= manifest.getValidForInDays()) { + errors.addValidationError( + String.format( + Locale.ROOT, + "updateInterval %d is should be smaller than %d", + updateIntervalInDays.days(), + manifest.getValidForInDays() + ) + ); + } + } + + /** + * Validate updateIntervalInDays is larger than 0 + * + * @param errors the errors to add error messages + */ + private void validateUpdateInterval(final ActionRequestValidationException errors) { + if (updateIntervalInDays.compareTo(TimeValue.timeValueDays(1)) > 0) { + errors.addValidationError("Update interval should be equal to or larger than 1 day"); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java new file mode 100644 index 00000000..291e1087 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.net.URL; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; + +import lombok.extern.log4j.Log4j2; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +/** + * Transport action to create datasource + */ +@Log4j2 +public class PutDatasourceTransportAction extends HandledTransportAction { + private final Client client; + private final ClusterService clusterService; + private final ThreadPool threadPool; + + private TimeValue timeout; + private int indexingBulkSize; + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param client the client + * @param clusterService the cluster service + * @param threadPool the thread pool + * @param settings the settings + * @param clusterSettings the cluster settings + */ + @Inject + public PutDatasourceTransportAction( + final TransportService transportService, + final ActionFilters actionFilters, + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final Settings settings, + final ClusterSettings clusterSettings + ) { + super(PutDatasourceAction.NAME, transportService, actionFilters, PutDatasourceRequest::new); + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(settings); + clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> timeout = newValue); + indexingBulkSize = Ip2GeoSettings.INDEXING_BULK_SIZE.get(settings); + clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.INDEXING_BULK_SIZE, newValue -> indexingBulkSize = newValue); + } + + @Override + protected void doExecute(final Task task, final PutDatasourceRequest request, final ActionListener listener) { + try { + Datasource jobParameter = Datasource.Builder.build(request); + IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) + .id(jobParameter.getId()) + .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .opType(DocWriteRequest.OpType.CREATE); + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(final IndexResponse indexResponse) { + // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread + // pool. + threadPool.generic().submit(() -> { + try { + createDatasource(jobParameter); + } catch (Exception e) { + log.error("Failed to create datasource for {}", jobParameter.getId(), e); + jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); + jobParameter.setState(DatasourceState.FAILED); + try { + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + } catch (Exception ex) { + log.error("Failed to mark datasource state as FAILED for {}", jobParameter.getId(), ex); + } + } + }); + listener.onResponse(new AcknowledgedResponse(true)); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof VersionConflictEngineException) { + listener.onFailure( + new ResourceAlreadyExistsException("datasource [{}] already exists", request.getDatasourceName()) + ); + } else { + listener.onFailure(e); + } + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private void createDatasource(final Datasource jobParameter) throws Exception { + if (!DatasourceState.PREPARING.equals(jobParameter.getState())) { + log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, jobParameter.getState()); + jobParameter.setState(DatasourceState.FAILED); + jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + return; + } + + URL url = new URL(jobParameter.getEndpoint()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(url); + String indexName = setupIndex(manifest, jobParameter); + Instant startTime = Instant.now(); + String[] fields = putIp2GeoData(indexName, manifest); + Instant endTime = Instant.now(); + updateJobParameterAsSucceeded(jobParameter, manifest, fields, startTime, endTime); + log.info("GeoIP database[{}] creation succeeded after {} seconds", jobParameter.getId(), Duration.between(startTime, endTime)); + } + + private void updateJobParameterAsSucceeded( + final Datasource jobParameter, + final DatasourceManifest manifest, + final String[] fields, + final Instant startTime, + final Instant endTime + ) throws IOException { + jobParameter.setDatabase(manifest, fields); + jobParameter.getUpdateStats().setLastSucceededAt(endTime); + jobParameter.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); + jobParameter.enable(); + jobParameter.setState(DatasourceState.AVAILABLE); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + } + + private String setupIndex(final DatasourceManifest manifest, final Datasource jobParameter) throws IOException { + String indexName = jobParameter.indexNameFor(manifest); + jobParameter.getIndices().add(indexName); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + GeoIpDataHelper.createIndexIfNotExists(clusterService, client, indexName, timeout); + return indexName; + } + + private String[] putIp2GeoData(final String indexName, final DatasourceManifest manifest) throws IOException { + String[] fields; + try (CSVParser reader = GeoIpDataHelper.getDatabaseReader(manifest)) { + Iterator iter = reader.iterator(); + fields = iter.next().values(); + GeoIpDataHelper.putGeoData(client, indexName, fields, iter, indexingBulkSize, timeout); + } + return fields; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java new file mode 100644 index 00000000..00c70f19 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.rest.RestRequest.Method.PUT; + +import java.io.IOException; +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +/** + * Rest handler for Ip2Geo datasource creation + * + * This handler handles a request of + * PUT /_plugins/geospatial/ip2geo/datasource/{id} + * { + * "endpoint": {endpoint}, + * "update_interval_in_days": 3 + * } + * + * When request is received, it will create a datasource by downloading GeoIp data from the endpoint. + * After the creation of datasource is completed, it will schedule the next update task after update_interval_in_days. + * + */ +public class RestPutDatasourceHandler extends BaseRestHandler { + private static final String ACTION_NAME = "ip2geo_datasource"; + private String defaultDatasourceEndpoint; + private TimeValue defaultUpdateInterval; + + public RestPutDatasourceHandler(final Settings settings, final ClusterSettings clusterSettings) { + defaultDatasourceEndpoint = Ip2GeoSettings.DATASOURCE_ENDPOINT.get(settings); + clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.DATASOURCE_ENDPOINT, newValue -> defaultDatasourceEndpoint = newValue); + defaultUpdateInterval = Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL.get(settings); + clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL, newValue -> defaultUpdateInterval = newValue); + } + + @Override + public String getName() { + return ACTION_NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("id")); + if (request.hasContentOrSourceParam()) { + try (XContentParser parser = request.contentOrSourceParamParser()) { + PutDatasourceRequest.PARSER.parse(parser, putDatasourceRequest, null); + } + } + if (putDatasourceRequest.getEndpoint() == null) { + putDatasourceRequest.setEndpoint(defaultDatasourceEndpoint); + } + if (putDatasourceRequest.getUpdateIntervalInDays() == null) { + putDatasourceRequest.setUpdateIntervalInDays(defaultUpdateInterval); + } + return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel)); + } + + @Override + public List routes() { + String path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/{id}"); + return List.of(new Route(PUT, path)); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java new file mode 100644 index 00000000..ea89e585 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.io.IOException; +import java.time.Instant; + +import lombok.extern.log4j.Log4j2; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.client.Client; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.index.IndexNotFoundException; + +/** + * Helper class for datasource + */ +@Log4j2 +public class DatasourceHelper { + + /** + * Update datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param client the client + * @param datasource the datasource + * @param timeout the timeout + * @return index response + * @throws IOException exception + */ + public static IndexResponse updateDatasource(final Client client, final Datasource datasource, final TimeValue timeout) + throws IOException { + datasource.setLastUpdateTime(Instant.now()); + IndexRequestBuilder requestBuilder = client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME); + requestBuilder.setId(datasource.getId()); + requestBuilder.setOpType(DocWriteRequest.OpType.INDEX); + requestBuilder.setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + return client.index(requestBuilder.request()).actionGet(timeout); + } + + /** + * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param client the client + * @param id the name of a datasource + * @param timeout the timeout + * @return datasource + * @throws IOException exception + */ + public static Datasource getDatasource(final Client client, final String id, final TimeValue timeout) throws IOException { + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); + GetResponse response; + try { + response = client.get(request).actionGet(timeout); + if (!response.isExists()) { + log.error("Datasource[{}] does not exist in an index[{}]", id, DatasourceExtension.JOB_INDEX_NAME); + return null; + } + } catch (IndexNotFoundException e) { + log.error("Index[{}] is not found", DatasourceExtension.JOB_INDEX_NAME); + return null; + } + + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef() + ); + return Datasource.PARSER.parse(parser, null); + } + + /** + * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param client the client + * @param id the name of a datasource + * @param actionListener the action listener + */ + public static void getDatasource(final Client client, final String id, final ActionListener actionListener) { + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); + client.get(request, new ActionListener() { + @Override + public void onResponse(final GetResponse response) { + if (!response.isExists()) { + actionListener.onResponse(null); + return; + } + + try { + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef() + ); + actionListener.onResponse(Datasource.PARSER.parse(parser, null)); + } catch (IOException e) { + actionListener.onFailure(e); + } + } + + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); + } + }); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java new file mode 100644 index 00000000..0c65382f --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.CharBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import org.opensearch.SpecialPermission; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; + +/** + * Ip2Geo datasource manifest file object + * + * Manifest file is stored in an external endpoint. OpenSearch read the file and store values it in this object. + */ +@Setter +@Getter +@AllArgsConstructor +public class DatasourceManifest { + private static final ParseField URL_FIELD = new ParseField("url"); + private static final ParseField DB_NAME_FIELD = new ParseField("db_name"); + private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); + private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); + private static final ParseField PROVIDER_FIELD = new ParseField("provider"); + + /** + * @param url URL of a ZIP file containing a database + * @return URL of a ZIP file containing a database + */ + private String url; + /** + * @param dbName A database file name inside the ZIP file + * @return A database file name inside the ZIP file + */ + private String dbName; + /** + * @param md5Hash MD5 hash value of a database file + * @return MD5 hash value of a database file + */ + private String md5Hash; + /** + * @param validForInDays A duration in which the database file is valid to use + * @return A duration in which the database file is valid to use + */ + private Long validForInDays; + /** + * @param updatedAt A date when the database was updated + * @return A date when the database was updated + */ + private Long updatedAt; + /** + * @param provider A database provider name + * @return A database provider name + */ + private String provider; + + /** + * Ddatasource manifest parser + */ + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "datasource_manifest", + true, + args -> { + String url = (String) args[0]; + String dbName = (String) args[1]; + String md5hash = (String) args[2]; + Long validForInDays = (Long) args[3]; + Long updatedAt = (Long) args[4]; + String provider = (String) args[5]; + return new DatasourceManifest(url, dbName, md5hash, validForInDays, updatedAt, provider); + } + ); + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), URL_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DB_NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); + } + + /** + * Datasource manifest builder + */ + public static class Builder { + private static final int MANIFEST_FILE_MAX_BYTES = 1024 * 8; + + /** + * Build DatasourceManifest from a given url + * + * @param url url to downloads a manifest file + * @return DatasourceManifest representing the manifest file + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") + public static DatasourceManifest build(final URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) { + CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES); + reader.read(charBuffer); + charBuffer.flip(); + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + charBuffer.toString() + ); + return PARSER.parse(parser, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java new file mode 100644 index 00000000..27523bda --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +/** + * Ip2Geo datasource state + * + * When data source is created, it starts with PREPARING state. Once the first GeoIP data is generated, the state changes to AVAILABLE. + * Only when the first GeoIP data generation failed, the state changes to FAILED. + * Subsequent GeoIP data failure won't change data source state from AVAILABLE to FAILED. + * When delete request is received, the data source state changes to DELETING. + * + * State changed from left to right for the entire lifecycle of a datasource + * (PREPARING) to (FAILED or AVAILABLE) to (DELETING) + * + */ +public enum DatasourceState { + /** + * Data source is being prepared + */ + PREPARING, + /** + * Data source is ready to be used + */ + AVAILABLE, + /** + * Data source preparation failed + */ + FAILED, + /** + * Data source is being deleted + */ + DELETING +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java new file mode 100644 index 00000000..4543ce7f --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java @@ -0,0 +1,354 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import lombok.extern.log4j.Log4j2; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.opensearch.OpenSearchException; +import org.opensearch.SpecialPermission; +import org.opensearch.action.ActionListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.MultiSearchRequestBuilder; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.client.Requests; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.query.QueryBuilders; + +/** + * Helper class for GeoIp data + */ +@Log4j2 +public class GeoIpDataHelper { + private static final String IP_RANGE_FIELD_NAME = "_cidr"; + private static final String DATA_FIELD_NAME = "_data"; + private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); + private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); + + /** + * Create an index of single shard with auto expand replicas to all nodes + * + * @param clusterService cluster service + * @param client client + * @param indexName index name + * @param timeout timeout + */ + public static void createIndexIfNotExists( + final ClusterService clusterService, + final Client client, + final String indexName, + final TimeValue timeout + ) { + if (clusterService.state().metadata().hasIndex(indexName) == true) { + return; + } + final Map indexSettings = new HashMap<>(); + indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); + indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); + client.admin().indices().create(createIndexRequest).actionGet(timeout); + } + + /** + * Generate XContentBuilder representing datasource database index mapping + * + * { + * "dynamic": false, + * "properties": { + * "_cidr": { + * "type": "ip_range", + * "doc_values": false + * } + * } + * } + * + * @return String representing datasource database index mapping + */ + private static String getIndexMapping() { + try { + try (InputStream is = DatasourceHelper.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Create CSVParser of a GeoIP data + * + * @param manifest Datasource manifest + * @return CSVParser for GeoIP data + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read GeoIP database file") + public static CSVParser getDatabaseReader(final DatasourceManifest manifest) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URL zipUrl = new URL(manifest.getUrl()); + ZipInputStream zipIn = new ZipInputStream(zipUrl.openStream()); + ZipEntry zipEntry = zipIn.getNextEntry(); + while (zipEntry != null) { + if (!zipEntry.getName().equalsIgnoreCase(manifest.getDbName())) { + zipEntry = zipIn.getNextEntry(); + continue; + } + return new CSVParser(new BufferedReader(new InputStreamReader(zipIn)), CSVFormat.RFC4180); + } + } catch (IOException e) { + throw new OpenSearchException("failed to read geoip data from {}", manifest.getUrl(), e); + } + throw new OpenSearchException( + "database file [{}] does not exist in the zip file [{}]", + manifest.getDbName(), + manifest.getUrl() + ); + }); + } + + /** + * Create a document in json string format to ingest in datasource database index + * + * It assumes the first field as ip_range. The rest is added under data field. + * + * Document example + * { + * "_cidr":"1.0.0.1/25", + * "_data":{ + * "country": "USA", + * "city": "Seattle", + * "location":"13.23,42.12" + * } + * } + * + * @param fields a list of field name + * @param values a list of values + * @return Document in json string format + */ + public static String createDocument(final String[] fields, final String[] values) { + StringBuilder sb = new StringBuilder(); + sb.append("{\""); + sb.append(IP_RANGE_FIELD_NAME); + sb.append("\":\""); + sb.append(values[0]); + sb.append("\",\""); + sb.append(DATA_FIELD_NAME); + sb.append("\":{"); + for (int i = 1; i < fields.length; i++) { + if (i != 1) { + sb.append(","); + } + sb.append("\""); + sb.append(fields[i]); + sb.append("\":\""); + sb.append(values[i]); + sb.append("\""); + } + sb.append("}}"); + return sb.toString(); + } + + /** + * Query a given index using a given ip address to get geo data + * + * @param client client + * @param indexName index + * @param ip ip address + * @param actionListener action listener + */ + public static void getGeoData( + final Client client, + final String indexName, + final String ip, + final ActionListener> actionListener + ) { + client.prepareSearch(indexName) + .setSize(1) + .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) + .setPreference("_local") + .execute(new ActionListener<>() { + @Override + public void onResponse(final SearchResponse searchResponse) { + if (searchResponse.getHits().getHits().length == 0) { + actionListener.onResponse(Collections.emptyMap()); + } else { + Map geoData = (Map) XContentHelper.convertToMap( + searchResponse.getHits().getAt(0).getSourceRef(), + false, + XContentType.JSON + ).v2().get(DATA_FIELD_NAME); + actionListener.onResponse(geoData); + } + } + + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); + } + }); + } + + /** + * Query a given index using a given ip address iterator to get geo data + * + * This method calls itself recursively until it processes all ip addresses in bulk of {@code bulkSize}. + * + * @param client the client + * @param indexName the index name + * @param ipIterator the iterator of ip addresses + * @param maxBundleSize number of ip address to pass in multi search + * @param maxConcurrentSearches the max concurrent search requests + * @param firstOnly return only the first matching result if true + * @param geoData collected geo data + * @param actionListener the action listener + */ + public static void getGeoData( + final Client client, + final String indexName, + final Iterator ipIterator, + final Integer maxBundleSize, + final Integer maxConcurrentSearches, + final boolean firstOnly, + final Map> geoData, + final ActionListener actionListener + ) { + MultiSearchRequestBuilder mRequestBuilder = client.prepareMultiSearch(); + if (maxConcurrentSearches != 0) { + mRequestBuilder.setMaxConcurrentSearchRequests(maxConcurrentSearches); + } + + List ipsToSearch = new ArrayList<>(maxBundleSize); + while (ipIterator.hasNext() && ipsToSearch.size() < maxBundleSize) { + String ip = ipIterator.next(); + if (geoData.get(ip) == null) { + mRequestBuilder.add( + client.prepareSearch(indexName) + .setSize(1) + .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) + .setPreference("_local") + ); + ipsToSearch.add(ip); + } + } + + if (ipsToSearch.isEmpty()) { + actionListener.onResponse(null); + return; + } + + mRequestBuilder.execute(new ActionListener<>() { + @Override + public void onResponse(final MultiSearchResponse items) { + for (int i = 0; i < ipsToSearch.size(); i++) { + if (items.getResponses()[i].isFailure()) { + actionListener.onFailure(items.getResponses()[i].getFailure()); + return; + } + + if (items.getResponses()[i].getResponse().getHits().getHits().length == 0) { + geoData.put(ipsToSearch.get(i), Collections.emptyMap()); + continue; + } + + Map data = (Map) XContentHelper.convertToMap( + items.getResponses()[i].getResponse().getHits().getAt(0).getSourceRef(), + false, + XContentType.JSON + ).v2().get(DATA_FIELD_NAME); + + geoData.put(ipsToSearch.get(i), data); + + if (firstOnly) { + actionListener.onResponse(null); + return; + } + } + getGeoData(client, indexName, ipIterator, maxBundleSize, maxConcurrentSearches, firstOnly, geoData, actionListener); + } + + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); + } + }); + } + + /** + * Puts GeoIP data from CSVRecord iterator into a given index in bulk + * + * @param client OpenSearch client + * @param indexName Index name to puts the GeoIP data + * @param fields Field name matching with data in CSVRecord in order + * @param iterator GeoIP data to insert + * @param bulkSize Bulk size of data to process + * @param timeout Timeout + */ + public static void putGeoData( + final Client client, + final String indexName, + final String[] fields, + final Iterator iterator, + final int bulkSize, + final TimeValue timeout + ) { + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + while (iterator.hasNext()) { + CSVRecord record = iterator.next(); + String document = createDocument(fields, record.values()); + IndexRequest request = Requests.indexRequest(indexName).source(document, XContentType.JSON); + bulkRequest.add(request); + if (!iterator.hasNext() || bulkRequest.requests().size() == bulkSize) { + BulkResponse response = client.bulk(bulkRequest).actionGet(timeout); + if (response.hasFailures()) { + throw new OpenSearchException( + "error occurred while ingesting GeoIP data in {} with an error {}", + indexName, + response.buildFailureMessage() + ); + } + bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + } + } + client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); + client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java new file mode 100644 index 00000000..7c5d77f9 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.util.concurrent.ExecutorService; + +import org.opensearch.common.settings.Settings; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.FixedExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; + +/** + * Provide a list of static methods related with executors for Ip2Geo + */ +public class Ip2GeoExecutorHelper { + private static final String THREAD_POOL_NAME = "_plugin_geospatial_ip2geo_datasource_update"; + + /** + * We use fixed thread count of 1 for updating datasource as updating datasource is running background + * once a day at most and no need to expedite the task. + * + * @param settings the settings + * @return the executor builder + */ + public static ExecutorBuilder executorBuilder(final Settings settings) { + return new FixedExecutorBuilder(settings, THREAD_POOL_NAME, 1, 1000, THREAD_POOL_NAME, false); + } + + /** + * Return an executor service for datasource update task + * + * @param threadPool the thread pool + * @return the executor service + */ + public static ExecutorService forDatasourceUpdate(final ThreadPool threadPool) { + return threadPool.executor(THREAD_POOL_NAME); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java new file mode 100644 index 00000000..5df30c04 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -0,0 +1,137 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; + +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; + +/** + * Settings for Ip2Geo datasource operations + */ +public class Ip2GeoSettings { + + /** + * Default endpoint to be used in GeoIP datasource creation API + */ + public static final Setting DATASOURCE_ENDPOINT = Setting.simpleString( + "plugins.geospatial.ip2geo.datasource.endpoint", + // TODO: This value is not correct. Update it later once CDN server is ready. + "https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", + new DatasourceEndpointValidator(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Default update interval to be used in Ip2Geo datasource creation API + */ + public static final Setting DATASOURCE_UPDATE_INTERVAL = Setting.timeSetting( + "plugins.geospatial.ip2geo.datasource.update_interval_in_days", + TimeValue.timeValueDays(3), + TimeValue.timeValueDays(1), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Timeout value for Ip2Geo processor + */ + public static final Setting TIMEOUT_IN_SECONDS = Setting.timeSetting( + "plugins.geospatial.ip2geo.timeout_in_seconds", + TimeValue.timeValueSeconds(30), + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Bulk size for indexing GeoIP data + */ + public static final Setting INDEXING_BULK_SIZE = Setting.intSetting( + "plugins.geospatial.ip2geo.datasource.indexing_bulk_size", + 10000, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Cache size for GeoIP data + */ + public static final Setting CACHE_SIZE = Setting.intSetting( + "plugins.geospatial.ip2geo.processor.cache_size", + 1000, + 0, + Setting.Property.NodeScope + ); + + /** + * Multi search bundle size for GeoIP data + * + * Multi search is used only when a field contains a list of ip addresses. + */ + public static final Setting MAX_BUNDLE_SIZE = Setting.intSetting( + "plugins.geospatial.ip2geo.processor.max_bundle_size", + 100, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Multi search max concurrent searches + * + * Multi search is used only when a field contains a list of ip addresses. + * + * When the value is 0, it will use default value which will be decided + * based on node count and search thread pool size. + */ + public static final Setting MAX_CONCURRENT_SEARCHES = Setting.intSetting( + "plugins.geospatial.ip2geo.processor.max_concurrent_searches", + 0, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Return all settings of Ip2Geo feature + * @return a list of all settings for Ip2Geo feature + */ + public static final List> settings() { + return List.of( + DATASOURCE_ENDPOINT, + DATASOURCE_UPDATE_INTERVAL, + TIMEOUT_IN_SECONDS, + INDEXING_BULK_SIZE, + CACHE_SIZE, + MAX_BUNDLE_SIZE, + MAX_CONCURRENT_SEARCHES + ); + } + + /** + * Visible for testing + */ + protected static class DatasourceEndpointValidator implements Setting.Validator { + @Override + public void validate(final String value) { + try { + new URL(value).toURI(); + } catch (MalformedURLException | URISyntaxException e) { + throw new IllegalArgumentException("Invalid URL format is provided"); + } + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java new file mode 100644 index 00000000..4d42f9ad --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -0,0 +1,528 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; + +/** + * Ip2Geo datasource job parameter + */ +@Getter +@Setter +@AllArgsConstructor +public class Datasource implements ScheduledJobParameter { + /** + * Prefix of indices having Ip2Geo data + */ + public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".ip2geo-data"; + private static final long LOCK_DURATION_IN_SECONDS = 60 * 60; + + /** + * Default fields for job scheduling + */ + private static final ParseField ID_FIELD = new ParseField("id"); + private static final ParseField ENABLED_FILED = new ParseField("update_enabled"); + private static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time"); + private static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field"); + private static final ParseField SCHEDULE_FIELD = new ParseField("schedule"); + private static final ParseField ENABLED_TIME_FILED = new ParseField("enabled_time"); + private static final ParseField ENABLED_TIME_FILED_READABLE = new ParseField("enabled_time_field"); + + /** + * Additional fields for datasource + */ + private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + private static final ParseField STATE_FIELD = new ParseField("state"); + private static final ParseField INDICES_FIELD = new ParseField("indices"); + private static final ParseField DATABASE_FIELD = new ParseField("database"); + private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats"); + + /** + * Default variables for job scheduling + */ + + /** + * @param id Id of a datasource + * @return Id of a datasource + */ + private String id; + /** + * @param lastUpdateTime Last update time of a datasource + * @return Last update time of a datasource + */ + private Instant lastUpdateTime; + /** + * @param enabledTime Last time when a scheduling is enabled for a GeoIP data update + * @return Last time when a scheduling is enabled for the job scheduler + */ + private Instant enabledTime; + /** + * @param isEnabled Indicate if GeoIP data update is scheduled or not + * @return Indicate if scheduling is enabled or not + */ + private boolean isEnabled; + /** + * @param schedule Schedule for a GeoIP data update + * @return Schedule for the job scheduler + */ + private IntervalSchedule schedule; + + /** + * Additional variables for datasource + */ + + /** + * @param endpoint URL of a manifest file + * @return URL of a manifest file + */ + private String endpoint; + /** + * @param state State of a datasource + * @return State of a datasource + */ + private DatasourceState state; + /** + * @param indices A list of indices having GeoIP data + * @return A list of indices having GeoIP data + */ + private List indices; + /** + * @param database GeoIP database information + * @return GeoIP database information + */ + private Database database; + /** + * @param updateStats GeoIP database update statistics + * @return GeoIP database update statistics + */ + private UpdateStats updateStats; + + /** + * Datasource parser + */ + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "datasource_metadata", + true, + args -> { + String id = (String) args[0]; + Instant lastUpdateTime = Instant.ofEpochMilli((long) args[1]); + Instant enabledTime = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); + boolean isEnabled = (boolean) args[3]; + IntervalSchedule schedule = (IntervalSchedule) args[4]; + String endpoint = (String) args[5]; + DatasourceState state = DatasourceState.valueOf((String) args[6]); + List indices = (List) args[7]; + Database database = (Database) args[8]; + UpdateStats updateStats = (UpdateStats) args[9]; + Datasource parameter = new Datasource( + id, + lastUpdateTime, + enabledTime, + isEnabled, + schedule, + endpoint, + state, + indices, + database, + updateStats + ); + + return parameter; + } + ); + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_UPDATE_TIME_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ENABLED_TIME_FILED); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FILED); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SCHEDULE_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), ENDPOINT_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), Database.PARSER, DATABASE_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), UpdateStats.PARSER, UPDATE_STATS_FIELD); + + } + + /** + * Visible for testing + */ + protected Datasource() { + this(null, null, null); + } + + public Datasource(final String id, final IntervalSchedule schedule, final String endpoint) { + this( + id, + Instant.now(), + null, + false, + schedule, + endpoint, + DatasourceState.PREPARING, + new ArrayList<>(), + new Database(), + new UpdateStats() + ); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(ID_FIELD.getPreferredName(), id); + builder.timeField( + LAST_UPDATE_TIME_FIELD.getPreferredName(), + LAST_UPDATE_TIME_FIELD_READABLE.getPreferredName(), + lastUpdateTime.toEpochMilli() + ); + if (enabledTime != null) { + builder.timeField( + ENABLED_TIME_FILED.getPreferredName(), + ENABLED_TIME_FILED_READABLE.getPreferredName(), + enabledTime.toEpochMilli() + ); + } + builder.field(ENABLED_FILED.getPreferredName(), isEnabled); + builder.field(SCHEDULE_FIELD.getPreferredName(), schedule); + builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); + builder.field(STATE_FIELD.getPreferredName(), state.name()); + builder.field(INDICES_FIELD.getPreferredName(), indices); + builder.field(DATABASE_FIELD.getPreferredName(), database); + builder.field(UPDATE_STATS_FIELD.getPreferredName(), updateStats); + builder.endObject(); + return builder; + } + + @Override + public String getName() { + return id; + } + + @Override + public Instant getLastUpdateTime() { + return lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return enabledTime; + } + + @Override + public IntervalSchedule getSchedule() { + return schedule; + } + + @Override + public boolean isEnabled() { + return isEnabled; + } + + @Override + public Long getLockDurationSeconds() { + return LOCK_DURATION_IN_SECONDS; + } + + /** + * Jitter in scheduling a task + * + * We want a job to be delayed randomly with range of (0, 5) minutes for the + * next execution time. + * + * @see ScheduledJobParameter#getJitter() + * + * @return the jitter + */ + @Override + public Double getJitter() { + return 5.0 / (schedule.getInterval() * 24 * 60); + } + + /** + * Enable auto update of GeoIP data + */ + public void enable() { + enabledTime = Instant.now(); + isEnabled = true; + } + + /** + * Disable auto update of GeoIP data + */ + public void disable() { + enabledTime = null; + isEnabled = false; + } + + /** + * Current index name of a datasource + * + * @return Current index name of a datasource + */ + public String currentIndexName() { + return indexNameFor(database.updatedAt.toEpochMilli()); + } + + /** + * Index name for a given manifest + * + * @param manifest manifest + * @return Index name for a given manifest + */ + public String indexNameFor(final DatasourceManifest manifest) { + return indexNameFor(manifest.getUpdatedAt()); + } + + private String indexNameFor(final long suffix) { + return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, suffix); + } + + public boolean isExpired() { + if (database.validForInDays == null) { + return false; + } + + Instant lastCheckedAt; + if (updateStats.lastSkippedAt == null) { + lastCheckedAt = updateStats.lastSucceededAt; + } else { + lastCheckedAt = updateStats.lastSucceededAt.isBefore(updateStats.lastSkippedAt) + ? updateStats.lastSkippedAt + : updateStats.lastSucceededAt; + } + return Instant.now().isAfter(lastCheckedAt.plus(database.validForInDays, ChronoUnit.DAYS)); + } + + public void setDatabase(final DatasourceManifest datasourceManifest, final String[] fields) { + this.database.setProvider(datasourceManifest.getProvider()); + this.database.setMd5Hash(datasourceManifest.getMd5Hash()); + this.database.setUpdatedAt(Instant.ofEpochMilli(datasourceManifest.getUpdatedAt())); + this.database.setValidForInDays(database.validForInDays); + this.database.setFields(Arrays.asList(fields)); + } + + /** + * Database of a datasource + */ + @Getter + @Setter + @AllArgsConstructor(access = AccessLevel.PRIVATE) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class Database implements ToXContent { + private static final ParseField PROVIDER_FIELD = new ParseField("provider"); + private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); + private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at_field"); + private static final ParseField FIELDS_FIELD = new ParseField("fields"); + private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); + + /** + * @param provider A database provider name + * @return A database provider name + */ + private String provider; + /** + * @param md5Hash MD5 hash value of a database file + * @return MD5 hash value of a database file + */ + private String md5Hash; + /** + * @param updatedAt A date when the database was updated + * @return A date when the database was updated + */ + private Instant updatedAt; + /** + * @param validForInDays A duration in which the database file is valid to use + * @return A duration in which the database file is valid to use + */ + private Long validForInDays; + /** + * @param fields A list of available fields in the database + * @return A list of available fields in the database + */ + private List fields; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "datasource_metadata_database", + true, + args -> { + String provider = (String) args[0]; + String md5Hash = (String) args[1]; + Instant updatedAt = args[2] == null ? null : Instant.ofEpochMilli((Long) args[2]); + Long validForInDays = (Long) args[3]; + List fields = (List) args[4]; + return new Database(provider, md5Hash, updatedAt, validForInDays, fields); + } + ); + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + if (provider != null) { + builder.field(PROVIDER_FIELD.getPreferredName(), provider); + } + if (md5Hash != null) { + builder.field(MD5_HASH_FIELD.getPreferredName(), md5Hash); + } + if (updatedAt != null) { + builder.timeField( + UPDATED_AT_FIELD.getPreferredName(), + UPDATED_AT_FIELD_READABLE.getPreferredName(), + updatedAt.toEpochMilli() + ); + } + if (validForInDays != null) { + builder.field(VALID_FOR_IN_DAYS_FIELD.getPreferredName(), validForInDays); + } + if (fields != null) { + builder.startArray(FIELDS_FIELD.getPreferredName()); + for (String field : fields) { + builder.value(field); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + } + + /** + * Update stats of a datasource + */ + @Getter + @Setter + @AllArgsConstructor(access = AccessLevel.PRIVATE) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class UpdateStats implements ToXContent { + private static final ParseField LAST_SUCCEEDED_AT_FIELD = new ParseField("last_succeeded_at"); + private static final ParseField LAST_SUCCEEDED_AT_FIELD_READABLE = new ParseField("last_succeeded_at_field"); + private static final ParseField LAST_PROCESSING_TIME_IN_MILLIS_FIELD = new ParseField("last_processing_time_in_millis"); + private static final ParseField LAST_FAILED_AT_FIELD = new ParseField("last_failed_at"); + private static final ParseField LAST_FAILED_AT_FIELD_READABLE = new ParseField("last_failed_at_field"); + private static final ParseField LAST_SKIPPED_AT = new ParseField("last_skipped_at"); + private static final ParseField LAST_SKIPPED_AT_READABLE = new ParseField("last_skipped_at_field"); + + /** + * @param lastSucceededAt The last time when GeoIP data update was succeeded + * @return The last time when GeoIP data update was succeeded + */ + private Instant lastSucceededAt; + /** + * @param lastProcessingTimeInMillis The last processing time when GeoIP data update was succeeded + * @return The last processing time when GeoIP data update was succeeded + */ + private Long lastProcessingTimeInMillis; + /** + * @param lastFailedAt The last time when GeoIP data update was failed + * @return The last time when GeoIP data update was failed + */ + private Instant lastFailedAt; + /** + * @param lastSkippedAt The last time when GeoIP data update was skipped as there was no new update from an endpoint + * @return The last time when GeoIP data update was skipped as there was no new update from an endpoint + */ + private Instant lastSkippedAt; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "datasource_metadata_update_stats", + true, + args -> { + Instant lastSucceededAt = args[0] == null ? null : Instant.ofEpochMilli((long) args[0]); + Long lastProcessingTimeInMillis = (Long) args[1]; + Instant lastFailedAt = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); + Instant lastSkippedAt = args[3] == null ? null : Instant.ofEpochMilli((long) args[3]); + return new UpdateStats(lastSucceededAt, lastProcessingTimeInMillis, lastFailedAt, lastSkippedAt); + } + ); + + static { + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_SUCCEEDED_AT_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_PROCESSING_TIME_IN_MILLIS_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_FAILED_AT_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_SKIPPED_AT); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + if (lastSucceededAt != null) { + builder.timeField( + LAST_SUCCEEDED_AT_FIELD.getPreferredName(), + LAST_SUCCEEDED_AT_FIELD_READABLE.getPreferredName(), + lastSucceededAt.toEpochMilli() + ); + } + if (lastProcessingTimeInMillis != null) { + builder.field(LAST_PROCESSING_TIME_IN_MILLIS_FIELD.getPreferredName(), lastProcessingTimeInMillis); + } + if (lastFailedAt != null) { + builder.timeField( + LAST_FAILED_AT_FIELD.getPreferredName(), + LAST_FAILED_AT_FIELD_READABLE.getPreferredName(), + lastFailedAt.toEpochMilli() + ); + } + if (lastSkippedAt != null) { + builder.timeField( + LAST_SKIPPED_AT.getPreferredName(), + LAST_SKIPPED_AT_READABLE.getPreferredName(), + lastSkippedAt.toEpochMilli() + ); + } + builder.endObject(); + return builder; + } + } + + /** + * Builder class for Datasource + */ + public static class Builder { + public static Datasource build(final PutDatasourceRequest request) { + String id = request.getDatasourceName(); + IntervalSchedule schedule = new IntervalSchedule( + Instant.now(), + (int) request.getUpdateIntervalInDays().days(), + ChronoUnit.DAYS + ); + String endpoint = request.getEndpoint(); + return new Datasource(id, schedule, endpoint); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java new file mode 100644 index 00000000..ed94f6fe --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; + +/** + * Datasource job scheduler extension + * + * This extension is responsible for scheduling Ip2Geo data update task + * + * See https://github.com/opensearch-project/job-scheduler/blob/main/README.md#getting-started + */ +public class DatasourceExtension implements JobSchedulerExtension { + /** + * Job index name for a datasource + */ + public static final String JOB_INDEX_NAME = ".scheduler_geospatial_ip2geo_datasource"; + + @Override + public String getJobType() { + return "scheduler_geospatial_ip2geo_datasource"; + } + + @Override + public String getJobIndex() { + return JOB_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + return DatasourceRunner.getJobRunnerInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + return (parser, id, jobDocVersion) -> Datasource.PARSER.parse(parser, null); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java new file mode 100644 index 00000000..c408ff3b --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -0,0 +1,278 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import java.net.URL; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import lombok.extern.log4j.Log4j2; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutorHelper; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.threadpool.ThreadPool; + +/** + * Datasource update task + * + * This is a background task which is responsible for updating Ip2Geo datasource + */ +@Log4j2 +public class DatasourceRunner implements ScheduledJobRunner { + + private static DatasourceRunner INSTANCE; + + /** + * Return a singleton job runner instance + * @return job runner + */ + public static DatasourceRunner getJobRunnerInstance() { + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (DatasourceRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new DatasourceRunner(); + return INSTANCE; + } + } + + private ClusterService clusterService; + private ThreadPool threadPool; + private Client client; + private TimeValue timeout; + private Integer indexingBulkSize; + private boolean initialized; + + private DatasourceRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + /** + * Initialize timeout and indexingBulkSize from settings + */ + public void initialize(final ClusterService clusterService, final ThreadPool threadPool, final Client client) { + this.clusterService = clusterService; + this.threadPool = threadPool; + this.client = client; + + this.timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(clusterService.getSettings()); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> this.timeout = newValue); + this.indexingBulkSize = Ip2GeoSettings.INDEXING_BULK_SIZE.get(clusterService.getSettings()); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(Ip2GeoSettings.INDEXING_BULK_SIZE, newValue -> this.indexingBulkSize = newValue); + this.initialized = true; + } + + @Override + public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { + if (initialized == false) { + throw new AssertionError("this instance is not initialized"); + } + + log.info("Update job started for a datasource[{}]", jobParameter.getName()); + if (jobParameter instanceof Datasource == false) { + throw new IllegalStateException( + "job parameter is not instance of DatasourceUpdateJobParameter, type: " + jobParameter.getClass().getCanonicalName() + ); + } + + Ip2GeoExecutorHelper.forDatasourceUpdate(threadPool).submit(updateDatasourceRunner(jobParameter, context)); + } + + /** + * Update GeoIP data + * + * Lock is used so that only one of nodes run this task. + * Lock duration is 1 hour to avoid refreshing. This is okay because update interval is 1 day minimum. + * + * @param jobParameter job parameter + * @param context context + */ + private Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { + final LockService lockService = context.getLockService(); + return () -> { + if (jobParameter.getLockDurationSeconds() != null) { + lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { + if (lock == null) { + return; + } + try { + Datasource datasource = DatasourceHelper.getDatasource(client, jobParameter.getName(), timeout); + if (datasource == null) { + log.info("Datasource[{}] is already deleted", jobParameter.getName()); + return; + } + + try { + deleteUnusedIndices(datasource); + updateDatasource(datasource); + deleteUnusedIndices(datasource); + } catch (Exception e) { + log.error("Failed to update datasource for {}", datasource.getId(), e); + datasource.getUpdateStats().setLastFailedAt(Instant.now()); + DatasourceHelper.updateDatasource(client, datasource, timeout); + } + } finally { + lockService.release( + lock, + ActionListener.wrap(released -> {}, exception -> { log.error("Failed to release lock [{}]", lock); }) + ); + } + }, exception -> { log.error("Failed to acquire lock for job [{}]", jobParameter.getName()); })); + } + }; + + } + + /** + * Delete all indices except the one which are being used + * + * @param parameter + */ + private void deleteUnusedIndices(final Datasource parameter) { + try { + List deletedIndices = new ArrayList<>(); + for (String index : parameter.getIndices()) { + if (index.equals(parameter.currentIndexName())) { + continue; + } + + if (!clusterService.state().metadata().hasIndex(index)) { + deletedIndices.add(index); + continue; + } + + try { + if (client.admin() + .indices() + .prepareDelete(index) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .execute() + .actionGet(timeout) + .isAcknowledged()) { + deletedIndices.add(index); + } else { + log.error("Failed to delete an index [{}]", index); + } + } catch (Exception e) { + log.error("Failed to delete an index [{}]", index, e); + } + } + if (!deletedIndices.isEmpty()) { + parameter.getIndices().removeAll(deletedIndices); + DatasourceHelper.updateDatasource(client, parameter, timeout); + } + } catch (Exception e) { + log.error("Failed to delete old indices for {}", parameter.getId(), e); + } + } + + /** + * Update GeoIP data internal + * + * @param jobParameter + * @throws Exception + */ + private void updateDatasource(final Datasource jobParameter) throws Exception { + if (!DatasourceState.AVAILABLE.equals(jobParameter.getState())) { + log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, jobParameter.getState()); + jobParameter.disable(); + jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + return; + } + + URL url = new URL(jobParameter.getEndpoint()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(url); + + if (skipUpdate(jobParameter, manifest)) { + log.info("Skipping GeoIP database update. Update is not required for {}", jobParameter.getId()); + jobParameter.getUpdateStats().setLastSkippedAt(Instant.now()); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + return; + } + + Instant startTime = Instant.now(); + String indexName = jobParameter.indexNameFor(manifest); + jobParameter.getIndices().add(indexName); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + GeoIpDataHelper.createIndexIfNotExists(clusterService, client, indexName, timeout); + String[] fields; + try (CSVParser reader = GeoIpDataHelper.getDatabaseReader(manifest)) { + Iterator iter = reader.iterator(); + fields = iter.next().values(); + if (!jobParameter.getDatabase().getFields().equals(Arrays.asList(fields))) { + throw new OpenSearchException( + "fields does not match between old [{}] and new [{}]", + jobParameter.getDatabase().getFields().toString(), + Arrays.asList(fields).toString() + ); + } + GeoIpDataHelper.putGeoData(client, indexName, fields, iter, indexingBulkSize, timeout); + } + + Instant endTime = Instant.now(); + jobParameter.getDatabase().setProvider(manifest.getProvider()); + jobParameter.getDatabase().setMd5Hash(manifest.getMd5Hash()); + jobParameter.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); + jobParameter.getDatabase().setValidForInDays(manifest.getValidForInDays()); + jobParameter.getDatabase().setFields(Arrays.asList(fields)); + jobParameter.getUpdateStats().setLastSucceededAt(endTime); + jobParameter.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); + DatasourceHelper.updateDatasource(client, jobParameter, timeout); + log.info( + "GeoIP database creation succeeded for {} and took {} seconds", + jobParameter.getId(), + Duration.between(startTime, endTime) + ); + } + + /** + * Determine if update is needed or not + * + * Update is needed when all following conditions are met + * 1. MD5 hash value in datasource is different with MD5 hash value in manifest + * 2. updatedAt value in datasource is before updateAt value in manifest + * + * @param parameter + * @param manifest + * @return + */ + private boolean skipUpdate(final Datasource parameter, final DatasourceManifest manifest) { + if (manifest.getMd5Hash().equals(parameter.getDatabase().getMd5Hash())) { + return true; + } + + return parameter.getDatabase().getUpdatedAt().toEpochMilli() >= manifest.getUpdatedAt(); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java new file mode 100644 index 00000000..2bcc1ed5 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.processor; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +import org.opensearch.common.cache.Cache; +import org.opensearch.common.cache.CacheBuilder; +import org.opensearch.common.unit.TimeValue; + +/** + * The in-memory cache for the ip2geo data. There should only be 1 instance of this class. + */ +public class Ip2GeoCache { + private static final TimeValue CACHING_PERIOD = TimeValue.timeValueMinutes(1); + private final Cache> cache; + + /** + * Default constructor + * + * @param maxSize size of a cache + */ + public Ip2GeoCache(final long maxSize) { + if (maxSize < 0) { + throw new IllegalArgumentException("ip2geo max cache size must be 0 or greater"); + } + this.cache = CacheBuilder.>builder() + .setMaximumWeight(maxSize) + .setExpireAfterWrite(CACHING_PERIOD) + .build(); + } + + /** + * Put data in a cache if it is absent and return the data + * + * @param ip the first part of a key + * @param datasourceName the second part of a key + * @param retrieveFunction function to retrieve a data to be stored in a cache + * @return data in a cache + */ + public Map putIfAbsent( + final String ip, + final String datasourceName, + final Function> retrieveFunction + ) { + CacheKey cacheKey = new CacheKey(ip, datasourceName); + Map response = cache.get(cacheKey); + if (response == null) { + response = retrieveFunction.apply(ip); + response = response == null ? Collections.emptyMap() : response; + cache.put(cacheKey, response); + } + return response; + } + + /** + * Put data in a cache + * + * @param ip the first part of a key + * @param datasourceName the second part of a key + * @param data the data + */ + public void put(final String ip, final String datasourceName, final Map data) { + CacheKey cacheKey = new CacheKey(ip, datasourceName); + cache.put(cacheKey, data); + } + + protected Map get(final String ip, final String datasourceName) { + CacheKey cacheKey = new CacheKey(ip, datasourceName); + return cache.get(cacheKey); + } + + /** + * The key to use for the cache. Since this cache can span multiple ip2geo processors that all use different datasource, the datasource + * name is needed to be included in the cache key. For example, if we only used the IP address as the key the same IP may be in multiple + * datasource with different values. The datasource name scopes the IP to the correct datasource + */ + private static class CacheKey { + + private final String ip; + private final String datasourceName; + + private CacheKey(final String ip, final String datasourceName) { + this.ip = ip; + this.datasourceName = datasourceName; + } + + // generated + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CacheKey cacheKey = (CacheKey) o; + return Objects.equals(ip, cacheKey.ip) && Objects.equals(datasourceName, cacheKey.datasourceName); + } + + // generated + @Override + public int hashCode() { + return Objects.hash(ip, datasourceName); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java new file mode 100644 index 00000000..0d6ddd8c --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -0,0 +1,430 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.geospatial.ip2geo.processor; + +import static org.opensearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME; +import static org.opensearch.ingest.ConfigurationUtils.newConfigurationException; +import static org.opensearch.ingest.ConfigurationUtils.readBooleanProperty; +import static org.opensearch.ingest.ConfigurationUtils.readOptionalList; +import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import lombok.extern.log4j.Log4j2; + +import org.opensearch.action.ActionListener; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.ingest.AbstractProcessor; +import org.opensearch.ingest.IngestDocument; +import org.opensearch.ingest.IngestService; +import org.opensearch.ingest.Processor; + +/** + * Ip2Geo processor + */ +@Log4j2 +public final class Ip2GeoProcessor extends AbstractProcessor { + private static final Map DATA_EXPIRED = Map.of("error", "ip2geo_data_expired"); + private static final String PROPERTY_IP = "ip"; + private final String field; + private final String targetField; + private final String datasourceName; + private final Set properties; + private final boolean ignoreMissing; + private final boolean firstOnly; + private final Ip2GeoCache cache; + private final Client client; + private final ClusterService clusterService; + + private int maxBundleSize; + private int maxConcurrentSearches; + + /** + * Ip2Geo processor type + */ + public static final String TYPE = "ip2geo"; + + /** + * Construct an Ip2Geo processor. + * @param tag the processor tag + * @param description the processor description + * @param field the source field to geo-IP map + * @param targetField the target field + * @param datasourceName the datasourceName + * @param properties the properties + * @param ignoreMissing true if documents with a missing value for the field should be ignored + * @param firstOnly true if only first result should be returned in case of array + * @param cache the Ip2Geo cache + * @param client the client + * @param clusterService the cluster service + */ + public Ip2GeoProcessor( + final String tag, + final String description, + final String field, + final String targetField, + final String datasourceName, + final Set properties, + final boolean ignoreMissing, + final boolean firstOnly, + final Ip2GeoCache cache, + final Client client, + final ClusterService clusterService + ) { + super(tag, description); + this.field = field; + this.targetField = targetField; + this.datasourceName = datasourceName; + this.properties = properties; + this.ignoreMissing = ignoreMissing; + this.firstOnly = firstOnly; + this.cache = cache; + this.client = client; + this.clusterService = clusterService; + + maxBundleSize = clusterService.getClusterSettings().get(Ip2GeoSettings.MAX_BUNDLE_SIZE); + clusterService.getClusterSettings().addSettingsUpdateConsumer(Ip2GeoSettings.MAX_BUNDLE_SIZE, newValue -> maxBundleSize = newValue); + maxConcurrentSearches = clusterService.getClusterSettings().get(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES, newValue -> maxConcurrentSearches = newValue); + } + + /** + * Add geo data of a given ip address to ingestDocument in asynchronous way + * + * @param ingestDocument the document + * @param handler the handler + */ + @Override + public void execute(IngestDocument ingestDocument, BiConsumer handler) { + Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); + + if (ip == null && ignoreMissing) { + handler.accept(ingestDocument, null); + return; + } else if (ip == null) { + handler.accept(null, new IllegalArgumentException("field [" + field + "] is null, cannot extract geo information.")); + return; + } + + if (ip instanceof String) { + executeInternal(ingestDocument, handler, (String) ip); + } else if (ip instanceof List) { + executeInternal(ingestDocument, handler, ((List) ip)); + } else { + throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings"); + } + } + + /** + * Use {@code execute(IngestDocument, BiConsumer)} instead + * + * @param ingestDocument the document + * @return none + */ + @Override + public IngestDocument execute(IngestDocument ingestDocument) { + throw new IllegalStateException("Not implemented"); + } + + /** + * Handle single ip + * + * @param ingestDocument the document + * @param handler the handler + * @param ip the ip + */ + private void executeInternal( + final IngestDocument ingestDocument, + final BiConsumer handler, + final String ip + ) { + Map geoData = cache.get(ip, datasourceName); + if (geoData != null) { + if (!geoData.isEmpty()) { + ingestDocument.setFieldValue(targetField, filteredGeoData(geoData, ip)); + } + handler.accept(ingestDocument, null); + return; + } + + DatasourceHelper.getDatasource(client, datasourceName, new ActionListener<>() { + @Override + public void onResponse(final Datasource datasource) { + if (datasource == null) { + handler.accept(null, new IllegalStateException("datasource does not exist")); + return; + } + + if (datasource.isExpired()) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); + return; + } + + GeoIpDataHelper.getGeoData(client, datasource.currentIndexName(), ip, new ActionListener<>() { + @Override + public void onResponse(final Map stringObjectMap) { + cache.put(ip, datasourceName, stringObjectMap); + if (!stringObjectMap.isEmpty()) { + ingestDocument.setFieldValue(targetField, filteredGeoData(stringObjectMap, ip)); + } + handler.accept(ingestDocument, null); + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); + } + }); + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); + } + }); + } + + /** + * Handle multiple ips + * + * @param ingestDocument the document + * @param handler the handler + * @param ips the ip list + */ + private void executeInternal( + final IngestDocument ingestDocument, + final BiConsumer handler, + final List ips + ) { + Map> data = new HashMap<>(); + for (Object ip : ips) { + if (ip instanceof String == false) { + throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); + } + String ipAddr = (String) ip; + data.put(ipAddr, cache.get(ipAddr, datasourceName)); + } + List ipList = (List) ips; + DatasourceHelper.getDatasource(client, datasourceName, new ActionListener<>() { + @Override + public void onResponse(final Datasource datasource) { + if (datasource == null) { + handler.accept(null, new IllegalStateException("datasource does not exist")); + return; + } + + if (datasource.isExpired()) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); + return; + } + GeoIpDataHelper.getGeoData( + client, + datasource.currentIndexName(), + ipList.iterator(), + maxBundleSize, + maxConcurrentSearches, + firstOnly, + data, + new ActionListener<>() { + @Override + public void onResponse(final Object obj) { + for (Map.Entry> entry : data.entrySet()) { + cache.put(entry.getKey(), datasourceName, entry.getValue()); + } + + if (firstOnly) { + for (String ipAddr : ipList) { + Map geoData = data.get(ipAddr); + // GeoData for ipAddr won't be null + if (!geoData.isEmpty()) { + ingestDocument.setFieldValue(targetField, geoData); + handler.accept(ingestDocument, null); + return; + } + } + handler.accept(ingestDocument, null); + } else { + boolean match = false; + List> geoDataList = new ArrayList<>(ipList.size()); + for (String ipAddr : ipList) { + Map geoData = data.get(ipAddr); + // GeoData for ipAddr won't be null + geoDataList.add(geoData.isEmpty() ? null : geoData); + if (!geoData.isEmpty()) { + match = true; + } + } + if (match) { + ingestDocument.setFieldValue(targetField, geoDataList); + } + handler.accept(ingestDocument, null); + } + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); + } + } + ); + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); + } + }); + } + + private Map filteredGeoData(final Map geoData, final String ip) { + Map filteredGeoData; + if (properties == null) { + filteredGeoData = geoData; + } else { + filteredGeoData = new HashMap<>(); + for (String property : this.properties) { + if (property.equals(PROPERTY_IP)) { + filteredGeoData.put(PROPERTY_IP, ip); + } else { + filteredGeoData.put(property, geoData.get(property)); + } + } + } + return filteredGeoData; + } + + @Override + public String getType() { + return TYPE; + } + + /** + * Ip2Geo processor factory + */ + public static final class Factory implements Processor.Factory { + private final Ip2GeoCache cache; + private final Client client; + private final IngestService ingestService; + private TimeValue timeout; + + /** + * Default constructor + * + * @param cache the cache + * @param client the client + * @param ingestService the ingest service + */ + public Factory(final Ip2GeoCache cache, final Client client, final IngestService ingestService) { + this.cache = cache; + this.client = client; + this.ingestService = ingestService; + + timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(client.settings()); + ClusterSettings clusterSettings = ingestService.getClusterService().getClusterSettings(); + clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> timeout = newValue); + } + + /** + * When a user create a processor, this method is called twice. Once to validate the new processor and another + * to apply cluster state change after the processor is added. + * + * The second call is made by ClusterApplierService. Therefore, we cannot access cluster state in the call. + * That means, we cannot even query an index inside the call. + * + * Because the processor is validated in the first call, we skip the validation in the second call. + * + * @see org.opensearch.cluster.service.ClusterApplierService#state() + */ + @Override + public Ip2GeoProcessor create( + final Map registry, + final String processorTag, + final String description, + final Map config + ) throws IOException { + String ipField = readStringProperty(TYPE, processorTag, config, "field"); + String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "ip2geo"); + String datasourceName = readStringProperty(TYPE, processorTag, config, "datasource"); + List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); + boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); + + // Skip validation for the call by cluster applier service + if (!Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME)) { + validate(processorTag, datasourceName, propertyNames); + } + + return new Ip2GeoProcessor( + processorTag, + description, + ipField, + targetField, + datasourceName, + propertyNames == null ? null : new HashSet<>(propertyNames), + ignoreMissing, + firstOnly, + cache, + client, + ingestService.getClusterService() + ); + } + + private void validate(final String processorTag, final String datasourceName, final List propertyNames) throws IOException { + Datasource datasource = DatasourceHelper.getDatasource(client, datasourceName, timeout); + + if (datasource == null) { + throw newConfigurationException(TYPE, processorTag, "datasource", "datasource [" + datasourceName + "] doesn't exist"); + } + + if (!DatasourceState.AVAILABLE.equals(datasource.getState())) { + throw newConfigurationException( + TYPE, + processorTag, + "datasource", + "datasource [" + datasourceName + "] is not in an available state" + ); + } + + if (propertyNames == null) { + return; + } + + // Validate properties are valid. If not add all available properties. + final Set availableProperties = new HashSet<>(datasource.getDatabase().getFields()); + availableProperties.add(PROPERTY_IP); + for (String fieldName : propertyNames) { + if (!availableProperties.contains(fieldName)) { + throw newConfigurationException( + TYPE, + processorTag, + "properties", + "property [" + fieldName + "] is not available in the datasource [" + datasourceName + "]" + ); + } + } + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 085bbf92..1809cef6 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -5,6 +5,9 @@ package org.opensearch.geospatial.plugin; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; + +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -19,6 +22,7 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -32,6 +36,14 @@ import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapper; import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldTypeParser; import org.opensearch.geospatial.index.query.xyshape.XYShapeQueryBuilder; +import org.opensearch.geospatial.ip2geo.action.PutDatasourceAction; +import org.opensearch.geospatial.ip2geo.action.PutDatasourceTransportAction; +import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutorHelper; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoCache; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.geospatial.search.aggregations.bucket.geogrid.GeoHexGrid; @@ -41,16 +53,19 @@ import org.opensearch.geospatial.stats.upload.UploadStatsAction; import org.opensearch.geospatial.stats.upload.UploadStatsTransportAction; import org.opensearch.index.mapper.Mapper; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.ingest.Processor; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; +import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; @@ -58,15 +73,40 @@ * Entry point for Geospatial features. It provides additional Processors, Actions * to interact with Cluster. */ -public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlugin, MapperPlugin, SearchPlugin { +public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlugin, MapperPlugin, SearchPlugin, SystemIndexPlugin { + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return List.of(new SystemIndexDescriptor(IP2GEO_DATA_INDEX_NAME_PREFIX, "System index used for Ip2Geo data")); + } @Override public Map getProcessors(Processor.Parameters parameters) { return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) + .put( + Ip2GeoProcessor.TYPE, + new Ip2GeoProcessor.Factory( + new Ip2GeoCache(Ip2GeoSettings.CACHE_SIZE.get(parameters.client.settings())), + parameters.client, + parameters.ingestService + ) + ) .immutableMap(); } + @Override + public List> getExecutorBuilders(Settings settings) { + List> executorBuilders = new ArrayList<>(); + executorBuilders.add(Ip2GeoExecutorHelper.executorBuilder(settings)); + return executorBuilders; + } + + @Override + public List> getSettings() { + return Ip2GeoSettings.settings(); + } + @Override public Collection createComponents( Client client, @@ -81,6 +121,9 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { + // Initialize DatasourceUpdateRunner + DatasourceRunner.getJobRunnerInstance().initialize(clusterService, threadPool, client); + return List.of(UploadStats.getInstance()); } @@ -94,16 +137,15 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - RestUploadGeoJSONAction uploadGeoJSONAction = new RestUploadGeoJSONAction(); - RestUploadStatsAction statsAction = new RestUploadStatsAction(); - return List.of(statsAction, uploadGeoJSONAction); + return List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction(), new RestPutDatasourceHandler(settings, clusterSettings)); } @Override public List> getActions() { return List.of( new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class), - new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class) + new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class), + new ActionHandler<>(PutDatasourceAction.INSTANCE, PutDatasourceTransportAction.class) ); } diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000..6e9e1030 --- /dev/null +++ b/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + // needed by Ip2Geo datasource to get GeoIP database + permission java.net.SocketPermission "*", "connect,resolve"; +}; diff --git a/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension b/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension new file mode 100644 index 00000000..e3d6fe6f --- /dev/null +++ b/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension @@ -0,0 +1,11 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +# This file is needed to register DatasourceExtension in job scheduler framework +# See https://github.com/opensearch-project/job-scheduler/blob/main/README.md#getting-started +org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension diff --git a/src/main/resources/mappings/ip2geo_datasource.json b/src/main/resources/mappings/ip2geo_datasource.json new file mode 100644 index 00000000..3179ef0d --- /dev/null +++ b/src/main/resources/mappings/ip2geo_datasource.json @@ -0,0 +1,9 @@ +{ + "dynamic": false, + "properties": { + "_cidr": { + "type": "ip_range", + "doc_values": false + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java new file mode 100644 index 00000000..3198f26a --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.util.Locale; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; + +public class PutDatasourceRequestTests extends OpenSearchTestCase { + + public void testValidateInvalidUrl() { + PutDatasourceRequest request = new PutDatasourceRequest("test"); + request.setEndpoint("invalidUrl"); + request.setUpdateIntervalInDays(TimeValue.ZERO); + ActionRequestValidationException exception = request.validate(); + assertEquals(1, exception.validationErrors().size()); + assertEquals("Invalid URL format is provided", exception.validationErrors().get(0)); + } + + public void testValidateInvalidManifestFile() { + PutDatasourceRequest request = new PutDatasourceRequest("test"); + request.setDatasourceName("test"); + request.setEndpoint("https://hi.com"); + request.setUpdateIntervalInDays(TimeValue.ZERO); + ActionRequestValidationException exception = request.validate(); + assertEquals(1, exception.validationErrors().size()); + assertEquals( + String.format(Locale.ROOT, "Error occurred while reading a file from %s", request.getEndpoint()), + exception.validationErrors().get(0) + ); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java new file mode 100644 index 00000000..82cae4ad --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.util.HashSet; + +import org.junit.Before; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; + +public class RestPutDatasourceHandlerTests extends RestActionTestCase { + private RestPutDatasourceHandler action; + + @Before + public void setupAction() { + action = new RestPutDatasourceHandler(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings()))); + controller().registerHandler(action); + } + + public void testPrepareRequest() { + String content = "{\"endpoint\":\"https://test.com\", \"update_interval\":1}"; + RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath("/_geoip/datasource/test") + .withContent(new BytesArray(content), XContentType.JSON) + .build(); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof PutDatasourceRequest); + PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; + assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); + assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateIntervalInDays()); + assertEquals("test", putDatasourceRequest.getDatasourceName()); + return null; + }); + + dispatchRequest(restRequest); + } + + public void testPrepareRequestDefaultValue() { + RestRequest restRequestWithEmptyContent = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath("/_geoip/datasource/test") + .withContent(new BytesArray("{}"), XContentType.JSON) + .build(); + + RestRequest restRequestWithoutContent = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath("/_geoip/datasource/test") + .build(); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof PutDatasourceRequest); + PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; + assertEquals("https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", putDatasourceRequest.getEndpoint()); + assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateIntervalInDays()); + assertEquals("test", putDatasourceRequest.getDatasourceName()); + return null; + }); + + dispatchRequest(restRequestWithEmptyContent); + dispatchRequest(restRequestWithoutContent); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java new file mode 100644 index 00000000..02ff8d58 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.test.rest.RestActionTestCase; + +public class DatasourceHelperTests extends RestActionTestCase { + + public void testUpdateDatasource() throws Exception { + Instant previousTime = Instant.now().minusMillis(1); + Datasource datasource = new Datasource( + "testId", + previousTime, + null, + false, + null, + null, + DatasourceState.PREPARING, + null, + null, + null + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest request = (IndexRequest) actionRequest; + assertEquals(datasource.getId(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + return null; + }); + + DatasourceHelper.updateDatasource(verifyingClient, datasource, TimeValue.timeValueSeconds(30)); + assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); + } + + public void testGetDatasourceException() throws Exception { + Datasource datasource = new Datasource( + "testId", + Instant.now(), + null, + false, + new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + "https://test.com", + DatasourceState.PREPARING, + null, + null, + null + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof GetRequest); + GetRequest request = (GetRequest) actionRequest; + assertEquals(datasource.getId(), request.id()); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + throw new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME); + }); + + assertNull(DatasourceHelper.getDatasource(verifyingClient, datasource.getId(), TimeValue.timeValueSeconds(30))); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java new file mode 100644 index 00000000..6b65026d --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import org.opensearch.test.OpenSearchTestCase; + +public class GeoIpDataHelperTests extends OpenSearchTestCase { + public void testCreateDocument() { + String[] names = { "ip", "country", "city" }; + String[] values = { "1.0.0.0/25", "USA", "Seattle" }; + assertEquals( + "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", + GeoIpDataHelper.createDocument(names, values) + ); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java new file mode 100644 index 00000000..451d0f8e --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import org.opensearch.test.OpenSearchTestCase; + +public class Ip2GeoSettingsTests extends OpenSearchTestCase { + public void testValidateInvalidUrl() { + Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator(); + Exception e = expectThrows(IllegalArgumentException.class, () -> validator.validate("InvalidUrl")); + assertEquals("Invalid URL format is provided", e.getMessage()); + } + + public void testValidateValidUrl() { + Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator(); + validator.validate("https://test.com"); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java new file mode 100644 index 00000000..0faaa3e2 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Locale; + +import org.opensearch.common.Randomness; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.test.OpenSearchTestCase; + +public class DatasourceTests extends OpenSearchTestCase { + public void testCurrentIndexName() { + String id = GeospatialTestHelper.randomLowerCaseString(); + Instant now = Instant.now(); + Datasource datasource = new Datasource(); + datasource.setId(id); + datasource.getDatabase().setProvider("provider"); + datasource.getDatabase().setMd5Hash("md5Hash"); + datasource.getDatabase().setUpdatedAt(now); + datasource.getDatabase().setValidForInDays(10l); + datasource.getDatabase().setFields(new ArrayList<>()); + assertEquals( + String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, now.toEpochMilli()), + datasource.currentIndexName() + ); + } + + public void testGetIndexNameFor() { + long updatedAt = Randomness.get().nextLong(); + DatasourceManifest manifest = mock(DatasourceManifest.class); + when(manifest.getUpdatedAt()).thenReturn(updatedAt); + + String id = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource(); + datasource.setId(id); + assertEquals( + String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, updatedAt), + datasource.indexNameFor(manifest) + ); + } + + public void testGetJitter() { + Datasource datasource = new Datasource(); + datasource.setSchedule(new IntervalSchedule(Instant.now(), Randomness.get().nextInt(31), ChronoUnit.DAYS)); + long intervalInMinutes = datasource.getSchedule().getInterval() * 60 * 24; + double sixMinutes = 6; + assertTrue(datasource.getJitter() * intervalInMinutes <= sixMinutes); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java new file mode 100644 index 00000000..ad340f47 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.processor; + +import java.util.HashMap; +import java.util.Map; + +import org.opensearch.OpenSearchException; +import org.opensearch.test.OpenSearchTestCase; + +public class Ip2GeoCacheTests extends OpenSearchTestCase { + public void testCachesAndEvictsResults() { + Ip2GeoCache cache = new Ip2GeoCache(1); + String datasource = "datasource"; + Map response1 = new HashMap<>(); + Map response2 = new HashMap<>(); + assertNotSame(response1, response2); + + // add a key + Map cachedResponse = cache.putIfAbsent("127.0.0.1", datasource, key -> response1); + assertSame(cachedResponse, response1); + assertSame(cachedResponse, cache.putIfAbsent("127.0.0.1", datasource, key -> response2)); + assertSame(cachedResponse, cache.get("127.0.0.1", datasource)); + + // evict old key by adding another value + cachedResponse = cache.putIfAbsent("127.0.0.2", datasource, key -> response2); + assertSame(cachedResponse, response2); + assertSame(cachedResponse, cache.putIfAbsent("127.0.0.2", datasource, ip -> response2)); + assertSame(cachedResponse, cache.get("127.0.0.2", datasource)); + + assertNotSame(response1, cache.get("127.0.0.1", datasource)); + } + + public void testThrowsFunctionsException() { + Ip2GeoCache cache = new Ip2GeoCache(1); + expectThrows( + OpenSearchException.class, + () -> cache.putIfAbsent("127.0.0.1", "datasource", ip -> { throw new OpenSearchException("bad"); }) + ); + } + + public void testNoExceptionForNullValue() { + Ip2GeoCache cache = new Ip2GeoCache(1); + Map response = cache.putIfAbsent("127.0.0.1", "datasource", ip -> null); + assertTrue(response.isEmpty()); + } + + public void testInvalidInit() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Ip2GeoCache(-1)); + assertEquals("ip2geo max cache size must be 0 or greater", ex.getMessage()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 1a775955..d7ce5594 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -5,25 +5,53 @@ package org.opensearch.geospatial.plugin; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashSet; import java.util.List; import java.util.Map; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; +import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.geospatial.stats.upload.RestUploadStatsAction; +import org.opensearch.ingest.IngestService; import org.opensearch.ingest.Processor; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IngestPlugin; import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; import org.opensearch.test.OpenSearchTestCase; public class GeospatialPluginTests extends OpenSearchTestCase { + private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings())); + private final List SUPPORTED_REST_HANDLERS = List.of( + new RestUploadGeoJSONAction(), + new RestUploadStatsAction(), + new RestPutDatasourceHandler(Settings.EMPTY, clusterSettings) + ); + private final Client client; + private final ClusterService clusterService; + private final IngestService ingestService; - private final List SUPPORTED_REST_HANDLERS = List.of(new RestUploadGeoJSONAction(), new RestUploadStatsAction()); + public GeospatialPluginTests() { + client = mock(Client.class); + when(client.settings()).thenReturn(Settings.EMPTY); + clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + ingestService = mock(IngestService.class); + when(ingestService.getClusterService()).thenReturn(clusterService); + } public void testIsAnIngestPlugin() { GeospatialPlugin plugin = new GeospatialPlugin(); @@ -31,15 +59,30 @@ public void testIsAnIngestPlugin() { } public void testFeatureProcessorIsAdded() { + Processor.Parameters parameters = new Processor.Parameters( + mock(Environment.class), + mock(ScriptService.class), + null, + null, + null, + null, + ingestService, + client, + null + ); + GeospatialPlugin plugin = new GeospatialPlugin(); - Map processors = plugin.getProcessors(null); + Map processors = plugin.getProcessors(parameters); assertTrue(processors.containsKey(FeatureProcessor.TYPE)); assertTrue(processors.get(FeatureProcessor.TYPE) instanceof FeatureProcessor.Factory); } public void testTotalRestHandlers() { GeospatialPlugin plugin = new GeospatialPlugin(); - assertEquals(SUPPORTED_REST_HANDLERS.size(), plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null).size()); + assertEquals( + SUPPORTED_REST_HANDLERS.size(), + plugin.getRestHandlers(Settings.EMPTY, null, clusterSettings, null, null, null, null).size() + ); } public void testUploadGeoJSONTransportIsAdded() { diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index 21c13a60..264ec585 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -1,8 +1,8 @@ -"Test that geospatial plugin is loaded in OpenSearch": +"Test that geospatial and job scheduler plugins are loaded in OpenSearch": - do: cat.plugins: local: true h: component - match: - $body: /^opensearch-geospatial\n$/ + $body: /^opensearch-geospatial\nopensearch-job-scheduler\n$/ From a3627a68370b21a5c9e0ca145b4c9a2342df82dd Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 3 May 2023 10:44:33 -0700 Subject: [PATCH 12/61] Added unit tests with some refactoring of codes (#271) * Add Unit tests * Set cache true for search query * Remove in memory cache implementation (Two way door decision) * Relying on search cache without custom cache * Renamed datasource state from FAILED to CREATE_FAILED * Renamed class name from *Helper to *Facade * Changed updateIntervalInDays to updateInterval * Changed value type of default update_interval from TimeValue to Long * Read setting value from cluster settings directly Signed-off-by: Heemin Kim --- .../annotation/VisibleForTesting.java | 12 + .../ip2geo/action/PutDatasourceRequest.java | 26 +- .../action/PutDatasourceTransportAction.java | 163 +++------ .../action/RestPutDatasourceHandler.java | 17 +- ...ourceHelper.java => DatasourceFacade.java} | 33 +- .../ip2geo/common/DatasourceState.java | 16 +- ...IpDataHelper.java => GeoIpDataFacade.java} | 115 +++--- ...xecutorHelper.java => Ip2GeoExecutor.java} | 10 +- .../ip2geo/common/Ip2GeoSettings.java | 23 +- .../ip2geo/jobscheduler/Datasource.java | 80 +++- .../jobscheduler/DatasourceExtension.java | 2 +- .../ip2geo/jobscheduler/DatasourceRunner.java | 231 +++--------- .../jobscheduler/DatasourceUpdateService.java | 218 +++++++++++ .../ip2geo/processor/Ip2GeoCache.java | 112 ------ .../ip2geo/processor/Ip2GeoProcessor.java | 275 +++++++------- .../geospatial/plugin/GeospatialPlugin.java | 36 +- .../geospatial/ip2geo/Ip2GeoTestCase.java | 227 ++++++++++++ .../action/PutDatasourceRequestTests.java | 96 ++++- .../PutDatasourceTransportActionTests.java | 118 ++++++ .../action/RestPutDatasourceHandlerTests.java | 51 ++- .../ip2geo/common/DatasourceFacadeTests.java | 130 +++++++ .../ip2geo/common/DatasourceHelperTests.java | 78 ---- .../ip2geo/common/GeoIpDataFacadeTests.java | 324 ++++++++++++++++ .../ip2geo/common/GeoIpDataHelperTests.java | 22 -- .../DatasourceExtensionTests.java | 47 +++ .../jobscheduler/DatasourceRunnerTests.java | 110 ++++++ .../ip2geo/jobscheduler/DatasourceTests.java | 54 ++- .../DatasourceUpdateServiceTests.java | 152 ++++++++ .../ip2geo/processor/Ip2GeoCacheTests.java | 58 --- .../processor/Ip2GeoProcessorTests.java | 346 ++++++++++++++++++ .../plugin/GeospatialPluginTests.java | 118 +++++- src/test/resources/ip2geo/manifest.json | 8 + .../ip2geo/manifest_invalid_url.json | 8 + .../resources/ip2geo/manifest_template.json | 8 + .../sample_invalid_less_than_two_fields.csv | 2 + src/test/resources/ip2geo/sample_valid.csv | 3 + src/test/resources/ip2geo/sample_valid.zip | Bin 0 -> 250 bytes 37 files changed, 2451 insertions(+), 878 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/annotation/VisibleForTesting.java rename src/main/java/org/opensearch/geospatial/ip2geo/common/{DatasourceHelper.java => DatasourceFacade.java} (81%) rename src/main/java/org/opensearch/geospatial/ip2geo/common/{GeoIpDataHelper.java => GeoIpDataFacade.java} (76%) rename src/main/java/org/opensearch/geospatial/ip2geo/common/{Ip2GeoExecutorHelper.java => Ip2GeoExecutor.java} (85%) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java delete mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java delete mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java delete mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java delete mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java create mode 100644 src/test/resources/ip2geo/manifest.json create mode 100644 src/test/resources/ip2geo/manifest_invalid_url.json create mode 100644 src/test/resources/ip2geo/manifest_template.json create mode 100644 src/test/resources/ip2geo/sample_invalid_less_than_two_fields.csv create mode 100644 src/test/resources/ip2geo/sample_valid.csv create mode 100644 src/test/resources/ip2geo/sample_valid.zip diff --git a/src/main/java/org/opensearch/geospatial/annotation/VisibleForTesting.java b/src/main/java/org/opensearch/geospatial/annotation/VisibleForTesting.java new file mode 100644 index 00000000..d48c6dc2 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/annotation/VisibleForTesting.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.annotation; + +public @interface VisibleForTesting { +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 24266b0d..82e513f0 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -14,6 +14,7 @@ import java.net.URL; import java.util.Locale; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.extern.log4j.Log4j2; @@ -28,11 +29,12 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; /** - * GeoIP datasource creation request + * Ip2Geo datasource creation request */ @Getter @Setter @Log4j2 +@EqualsAndHashCode public class PutDatasourceRequest extends AcknowledgedRequest { private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); @@ -47,10 +49,10 @@ public class PutDatasourceRequest extends AcknowledgedRequest("put_datasource"); PARSER.declareString((request, val) -> request.setEndpoint(val), ENDPOINT_FIELD); - PARSER.declareLong((request, val) -> request.setUpdateIntervalInDays(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); + PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); } /** @@ -79,7 +81,7 @@ public PutDatasourceRequest(final StreamInput in) throws IOException { super(in); this.datasourceName = in.readString(); this.endpoint = in.readString(); - this.updateIntervalInDays = in.readTimeValue(); + this.updateInterval = in.readTimeValue(); } @Override @@ -87,7 +89,7 @@ public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); out.writeString(datasourceName); out.writeString(endpoint); - out.writeTimeValue(updateIntervalInDays); + out.writeTimeValue(updateInterval); } @Override @@ -120,7 +122,7 @@ private void validateEndpoint(final ActionRequestValidationException errors) { * Conduct following validation on url * 1. can read manifest file from the endpoint * 2. the url in the manifest file complies with RFC-2396 - * 3. updateIntervalInDays is less than validForInDays value in the manifest file + * 3. updateInterval is less than validForInDays value in the manifest file * * @param url the url to validate * @param errors the errors to add error messages @@ -143,12 +145,12 @@ private void validateManifestFile(final URL url, final ActionRequestValidationEx return; } - if (updateIntervalInDays.days() >= manifest.getValidForInDays()) { + if (updateInterval.days() >= manifest.getValidForInDays()) { errors.addValidationError( String.format( Locale.ROOT, - "updateInterval %d is should be smaller than %d", - updateIntervalInDays.days(), + "updateInterval %d should be smaller than %d", + updateInterval.days(), manifest.getValidForInDays() ) ); @@ -156,12 +158,12 @@ private void validateManifestFile(final URL url, final ActionRequestValidationEx } /** - * Validate updateIntervalInDays is larger than 0 + * Validate updateInterval is equal or larger than 1 * * @param errors the errors to add error messages */ private void validateUpdateInterval(final ActionRequestValidationException errors) { - if (updateIntervalInDays.compareTo(TimeValue.timeValueDays(1)) > 0) { + if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { errors.addValidationError("Update interval should be equal to or larger than 1 day"); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index 291e1087..c1ca0737 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -8,16 +8,10 @@ package org.opensearch.geospatial.ip2geo.action; -import java.io.IOException; -import java.net.URL; -import java.time.Duration; import java.time.Instant; -import java.util.Iterator; import lombok.extern.log4j.Log4j2; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; @@ -28,19 +22,14 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; -import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -52,138 +41,96 @@ @Log4j2 public class PutDatasourceTransportAction extends HandledTransportAction { private final Client client; - private final ClusterService clusterService; private final ThreadPool threadPool; - - private TimeValue timeout; - private int indexingBulkSize; + private final DatasourceFacade datasourceFacade; + private final DatasourceUpdateService datasourceUpdateService; /** * Default constructor * @param transportService the transport service * @param actionFilters the action filters * @param client the client - * @param clusterService the cluster service * @param threadPool the thread pool - * @param settings the settings - * @param clusterSettings the cluster settings */ @Inject public PutDatasourceTransportAction( final TransportService transportService, final ActionFilters actionFilters, final Client client, - final ClusterService clusterService, final ThreadPool threadPool, - final Settings settings, - final ClusterSettings clusterSettings + final DatasourceFacade datasourceFacade, + final DatasourceUpdateService datasourceUpdateService ) { super(PutDatasourceAction.NAME, transportService, actionFilters, PutDatasourceRequest::new); this.client = client; - this.clusterService = clusterService; this.threadPool = threadPool; - timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(settings); - clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> timeout = newValue); - indexingBulkSize = Ip2GeoSettings.INDEXING_BULK_SIZE.get(settings); - clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.INDEXING_BULK_SIZE, newValue -> indexingBulkSize = newValue); + this.datasourceFacade = datasourceFacade; + this.datasourceUpdateService = datasourceUpdateService; } @Override protected void doExecute(final Task task, final PutDatasourceRequest request, final ActionListener listener) { try { - Datasource jobParameter = Datasource.Builder.build(request); + Datasource datasource = Datasource.Builder.build(request); IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) - .id(jobParameter.getId()) - .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .id(datasource.getId()) + .source(datasource.toXContent(JsonXContent.contentBuilder(), null)) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .opType(DocWriteRequest.OpType.CREATE); - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(final IndexResponse indexResponse) { - // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread - // pool. - threadPool.generic().submit(() -> { - try { - createDatasource(jobParameter); - } catch (Exception e) { - log.error("Failed to create datasource for {}", jobParameter.getId(), e); - jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); - jobParameter.setState(DatasourceState.FAILED); - try { - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - } catch (Exception ex) { - log.error("Failed to mark datasource state as FAILED for {}", jobParameter.getId(), ex); - } - } - }); - listener.onResponse(new AcknowledgedResponse(true)); - } - - @Override - public void onFailure(final Exception e) { - if (e instanceof VersionConflictEngineException) { - listener.onFailure( - new ResourceAlreadyExistsException("datasource [{}] already exists", request.getDatasourceName()) - ); - } else { - listener.onFailure(e); - } - } - }); + client.index(indexRequest, getIndexResponseListener(datasource, listener)); } catch (Exception e) { listener.onFailure(e); } } - private void createDatasource(final Datasource jobParameter) throws Exception { - if (!DatasourceState.PREPARING.equals(jobParameter.getState())) { - log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, jobParameter.getState()); - jobParameter.setState(DatasourceState.FAILED); - jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - return; - } + @VisibleForTesting + protected ActionListener getIndexResponseListener( + final Datasource datasource, + final ActionListener listener + ) { + return new ActionListener<>() { + @Override + public void onResponse(final IndexResponse indexResponse) { + // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread + // pool. + threadPool.generic().submit(() -> { createDatasource(datasource); }); + listener.onResponse(new AcknowledgedResponse(true)); + } - URL url = new URL(jobParameter.getEndpoint()); - DatasourceManifest manifest = DatasourceManifest.Builder.build(url); - String indexName = setupIndex(manifest, jobParameter); - Instant startTime = Instant.now(); - String[] fields = putIp2GeoData(indexName, manifest); - Instant endTime = Instant.now(); - updateJobParameterAsSucceeded(jobParameter, manifest, fields, startTime, endTime); - log.info("GeoIP database[{}] creation succeeded after {} seconds", jobParameter.getId(), Duration.between(startTime, endTime)); + @Override + public void onFailure(final Exception e) { + if (e instanceof VersionConflictEngineException) { + listener.onFailure(new ResourceAlreadyExistsException("datasource [{}] already exists", datasource.getId())); + } else { + listener.onFailure(e); + } + } + }; } - private void updateJobParameterAsSucceeded( - final Datasource jobParameter, - final DatasourceManifest manifest, - final String[] fields, - final Instant startTime, - final Instant endTime - ) throws IOException { - jobParameter.setDatabase(manifest, fields); - jobParameter.getUpdateStats().setLastSucceededAt(endTime); - jobParameter.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); - jobParameter.enable(); - jobParameter.setState(DatasourceState.AVAILABLE); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - } + @VisibleForTesting + protected void createDatasource(final Datasource datasource) { + if (DatasourceState.CREATING.equals(datasource.getState()) == false) { + log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.CREATING, datasource.getState()); + markDatasourceAsCreateFailed(datasource); + return; + } - private String setupIndex(final DatasourceManifest manifest, final Datasource jobParameter) throws IOException { - String indexName = jobParameter.indexNameFor(manifest); - jobParameter.getIndices().add(indexName); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - GeoIpDataHelper.createIndexIfNotExists(clusterService, client, indexName, timeout); - return indexName; + try { + datasourceUpdateService.updateOrCreateGeoIpData(datasource); + } catch (Exception e) { + log.error("Failed to create datasource for {}", datasource.getId(), e); + markDatasourceAsCreateFailed(datasource); + } } - private String[] putIp2GeoData(final String indexName, final DatasourceManifest manifest) throws IOException { - String[] fields; - try (CSVParser reader = GeoIpDataHelper.getDatabaseReader(manifest)) { - Iterator iter = reader.iterator(); - fields = iter.next().values(); - GeoIpDataHelper.putGeoData(client, indexName, fields, iter, indexingBulkSize, timeout); + private void markDatasourceAsCreateFailed(final Datasource datasource) { + datasource.getUpdateStats().setLastFailedAt(Instant.now()); + datasource.setState(DatasourceState.CREATE_FAILED); + try { + datasourceFacade.updateDatasource(datasource); + } catch (Exception e) { + log.error("Failed to mark datasource state as CREATE_FAILED for {}", datasource.getId(), e); } - return fields; } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java index 00c70f19..3ccffa06 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java @@ -17,7 +17,6 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; @@ -41,14 +40,10 @@ */ public class RestPutDatasourceHandler extends BaseRestHandler { private static final String ACTION_NAME = "ip2geo_datasource"; - private String defaultDatasourceEndpoint; - private TimeValue defaultUpdateInterval; + private final ClusterSettings clusterSettings; - public RestPutDatasourceHandler(final Settings settings, final ClusterSettings clusterSettings) { - defaultDatasourceEndpoint = Ip2GeoSettings.DATASOURCE_ENDPOINT.get(settings); - clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.DATASOURCE_ENDPOINT, newValue -> defaultDatasourceEndpoint = newValue); - defaultUpdateInterval = Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL.get(settings); - clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL, newValue -> defaultUpdateInterval = newValue); + public RestPutDatasourceHandler(final ClusterSettings clusterSettings) { + this.clusterSettings = clusterSettings; } @Override @@ -65,10 +60,10 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } } if (putDatasourceRequest.getEndpoint() == null) { - putDatasourceRequest.setEndpoint(defaultDatasourceEndpoint); + putDatasourceRequest.setEndpoint(clusterSettings.get(Ip2GeoSettings.DATASOURCE_ENDPOINT)); } - if (putDatasourceRequest.getUpdateIntervalInDays() == null) { - putDatasourceRequest.setUpdateIntervalInDays(defaultUpdateInterval); + if (putDatasourceRequest.getUpdateInterval() == null) { + putDatasourceRequest.setUpdateInterval(TimeValue.timeValueDays(clusterSettings.get(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL))); } return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel)); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java similarity index 81% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java rename to src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index ea89e585..b16e5478 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelper.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -20,7 +20,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.client.Client; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; @@ -32,43 +32,45 @@ import org.opensearch.index.IndexNotFoundException; /** - * Helper class for datasource + * Facade class for datasource */ @Log4j2 -public class DatasourceHelper { +public class DatasourceFacade { + private final Client client; + private final ClusterSettings clusterSettings; + + public DatasourceFacade(final Client client, final ClusterSettings clusterSettings) { + this.client = client; + this.clusterSettings = clusterSettings; + } /** * Update datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param client the client * @param datasource the datasource - * @param timeout the timeout * @return index response * @throws IOException exception */ - public static IndexResponse updateDatasource(final Client client, final Datasource datasource, final TimeValue timeout) - throws IOException { + public IndexResponse updateDatasource(final Datasource datasource) throws IOException { datasource.setLastUpdateTime(Instant.now()); IndexRequestBuilder requestBuilder = client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME); requestBuilder.setId(datasource.getId()); requestBuilder.setOpType(DocWriteRequest.OpType.INDEX); requestBuilder.setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); - return client.index(requestBuilder.request()).actionGet(timeout); + return client.index(requestBuilder.request()).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); } /** * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param client the client * @param id the name of a datasource - * @param timeout the timeout * @return datasource * @throws IOException exception */ - public static Datasource getDatasource(final Client client, final String id, final TimeValue timeout) throws IOException { + public Datasource getDatasource(final String id) throws IOException { GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); GetResponse response; try { - response = client.get(request).actionGet(timeout); - if (!response.isExists()) { + response = client.get(request).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + if (response.isExists() == false) { log.error("Datasource[{}] does not exist in an index[{}]", id, DatasourceExtension.JOB_INDEX_NAME); return null; } @@ -87,16 +89,15 @@ public static Datasource getDatasource(final Client client, final String id, fin /** * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param client the client * @param id the name of a datasource * @param actionListener the action listener */ - public static void getDatasource(final Client client, final String id, final ActionListener actionListener) { + public void getDatasource(final String id, final ActionListener actionListener) { GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); client.get(request, new ActionListener() { @Override public void onResponse(final GetResponse response) { - if (!response.isExists()) { + if (response.isExists() == false) { actionListener.onResponse(null); return; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java index 27523bda..85b0aecf 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java @@ -11,28 +11,28 @@ /** * Ip2Geo datasource state * - * When data source is created, it starts with PREPARING state. Once the first GeoIP data is generated, the state changes to AVAILABLE. - * Only when the first GeoIP data generation failed, the state changes to FAILED. - * Subsequent GeoIP data failure won't change data source state from AVAILABLE to FAILED. + * When data source is created, it starts with CREATING state. Once the first GeoIP data is generated, the state changes to AVAILABLE. + * Only when the first GeoIP data generation failed, the state changes to CREATE_FAILED. + * Subsequent GeoIP data failure won't change data source state from AVAILABLE to CREATE_FAILED. * When delete request is received, the data source state changes to DELETING. * * State changed from left to right for the entire lifecycle of a datasource - * (PREPARING) to (FAILED or AVAILABLE) to (DELETING) + * (CREATING) to (CREATE_FAILED or AVAILABLE) to (DELETING) * */ public enum DatasourceState { /** - * Data source is being prepared + * Data source is being created */ - PREPARING, + CREATING, /** * Data source is ready to be used */ AVAILABLE, /** - * Data source preparation failed + * Data source creation failed */ - FAILED, + CREATE_FAILED, /** * Data source is being deleted */ diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java similarity index 76% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java rename to src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 4543ce7f..c1fb3a70 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelper.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -8,6 +8,8 @@ package org.opensearch.geospatial.ip2geo.common; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -41,41 +43,45 @@ import org.opensearch.action.search.MultiSearchRequestBuilder; import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.QueryBuilders; /** - * Helper class for GeoIp data + * Facade class for GeoIp data */ @Log4j2 -public class GeoIpDataHelper { +public class GeoIpDataFacade { private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final Client client; + + public GeoIpDataFacade(final ClusterService clusterService, final Client client) { + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.client = client; + } /** * Create an index of single shard with auto expand replicas to all nodes * - * @param clusterService cluster service - * @param client client * @param indexName index name - * @param timeout timeout */ - public static void createIndexIfNotExists( - final ClusterService clusterService, - final Client client, - final String indexName, - final TimeValue timeout - ) { + public void createIndexIfNotExists(final String indexName) { if (clusterService.state().metadata().hasIndex(indexName) == true) { return; } @@ -83,7 +89,7 @@ public static void createIndexIfNotExists( indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); - client.admin().indices().create(createIndexRequest).actionGet(timeout); + client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); } /** @@ -101,11 +107,11 @@ public static void createIndexIfNotExists( * * @return String representing datasource database index mapping */ - private static String getIndexMapping() { + private String getIndexMapping() { try { - try (InputStream is = DatasourceHelper.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { + try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - return reader.lines().collect(Collectors.joining()); + return reader.lines().map(String::trim).collect(Collectors.joining()); } } } catch (IOException e) { @@ -120,7 +126,7 @@ private static String getIndexMapping() { * @return CSVParser for GeoIP data */ @SuppressForbidden(reason = "Need to connect to http endpoint to read GeoIP database file") - public static CSVParser getDatabaseReader(final DatasourceManifest manifest) { + public CSVParser getDatabaseReader(final DatasourceManifest manifest) { SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -128,7 +134,7 @@ public static CSVParser getDatabaseReader(final DatasourceManifest manifest) { ZipInputStream zipIn = new ZipInputStream(zipUrl.openStream()); ZipEntry zipEntry = zipIn.getNextEntry(); while (zipEntry != null) { - if (!zipEntry.getName().equalsIgnoreCase(manifest.getDbName())) { + if (zipEntry.getName().equalsIgnoreCase(manifest.getDbName()) == false) { zipEntry = zipIn.getNextEntry(); continue; } @@ -164,7 +170,10 @@ public static CSVParser getDatabaseReader(final DatasourceManifest manifest) { * @param values a list of values * @return Document in json string format */ - public static String createDocument(final String[] fields, final String[] values) { + public String createDocument(final String[] fields, final String[] values) { + if (fields.length != values.length) { + throw new OpenSearchException("header[{}] and record[{}] length does not match", fields, values); + } StringBuilder sb = new StringBuilder(); sb.append("{\""); sb.append(IP_RANGE_FIELD_NAME); @@ -188,35 +197,30 @@ public static String createDocument(final String[] fields, final String[] values } /** - * Query a given index using a given ip address to get geo data + * Query a given index using a given ip address to get geoip data * - * @param client client * @param indexName index * @param ip ip address * @param actionListener action listener */ - public static void getGeoData( - final Client client, - final String indexName, - final String ip, - final ActionListener> actionListener - ) { + public void getGeoIpData(final String indexName, final String ip, final ActionListener> actionListener) { client.prepareSearch(indexName) .setSize(1) .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) .setPreference("_local") + .setRequestCache(true) .execute(new ActionListener<>() { @Override public void onResponse(final SearchResponse searchResponse) { if (searchResponse.getHits().getHits().length == 0) { actionListener.onResponse(Collections.emptyMap()); } else { - Map geoData = (Map) XContentHelper.convertToMap( + Map geoIpData = (Map) XContentHelper.convertToMap( searchResponse.getHits().getAt(0).getSourceRef(), false, XContentType.JSON ).v2().get(DATA_FIELD_NAME); - actionListener.onResponse(geoData); + actionListener.onResponse(geoIpData); } } @@ -228,28 +232,26 @@ public void onFailure(final Exception e) { } /** - * Query a given index using a given ip address iterator to get geo data + * Query a given index using a given ip address iterator to get geoip data * * This method calls itself recursively until it processes all ip addresses in bulk of {@code bulkSize}. * - * @param client the client * @param indexName the index name * @param ipIterator the iterator of ip addresses * @param maxBundleSize number of ip address to pass in multi search * @param maxConcurrentSearches the max concurrent search requests * @param firstOnly return only the first matching result if true - * @param geoData collected geo data + * @param geoIpData collected geo data * @param actionListener the action listener */ - public static void getGeoData( - final Client client, + public void getGeoIpData( final String indexName, final Iterator ipIterator, final Integer maxBundleSize, final Integer maxConcurrentSearches, final boolean firstOnly, - final Map> geoData, - final ActionListener actionListener + final Map> geoIpData, + final ActionListener>> actionListener ) { MultiSearchRequestBuilder mRequestBuilder = client.prepareMultiSearch(); if (maxConcurrentSearches != 0) { @@ -259,19 +261,20 @@ public static void getGeoData( List ipsToSearch = new ArrayList<>(maxBundleSize); while (ipIterator.hasNext() && ipsToSearch.size() < maxBundleSize) { String ip = ipIterator.next(); - if (geoData.get(ip) == null) { + if (geoIpData.get(ip) == null) { mRequestBuilder.add( client.prepareSearch(indexName) .setSize(1) .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) .setPreference("_local") + .setRequestCache(true) ); ipsToSearch.add(ip); } } if (ipsToSearch.isEmpty()) { - actionListener.onResponse(null); + actionListener.onResponse(geoIpData); return; } @@ -285,7 +288,7 @@ public void onResponse(final MultiSearchResponse items) { } if (items.getResponses()[i].getResponse().getHits().getHits().length == 0) { - geoData.put(ipsToSearch.get(i), Collections.emptyMap()); + geoIpData.put(ipsToSearch.get(i), Collections.emptyMap()); continue; } @@ -295,14 +298,14 @@ public void onResponse(final MultiSearchResponse items) { XContentType.JSON ).v2().get(DATA_FIELD_NAME); - geoData.put(ipsToSearch.get(i), data); + geoIpData.put(ipsToSearch.get(i), data); if (firstOnly) { - actionListener.onResponse(null); + actionListener.onResponse(geoIpData); return; } } - getGeoData(client, indexName, ipIterator, maxBundleSize, maxConcurrentSearches, firstOnly, geoData, actionListener); + getGeoIpData(indexName, ipIterator, maxBundleSize, maxConcurrentSearches, firstOnly, geoIpData, actionListener); } @Override @@ -315,28 +318,20 @@ public void onFailure(final Exception e) { /** * Puts GeoIP data from CSVRecord iterator into a given index in bulk * - * @param client OpenSearch client * @param indexName Index name to puts the GeoIP data * @param fields Field name matching with data in CSVRecord in order * @param iterator GeoIP data to insert * @param bulkSize Bulk size of data to process - * @param timeout Timeout */ - public static void putGeoData( - final Client client, - final String indexName, - final String[] fields, - final Iterator iterator, - final int bulkSize, - final TimeValue timeout - ) { + public void putGeoIpData(final String indexName, final String[] fields, final Iterator iterator, final int bulkSize) { + TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); while (iterator.hasNext()) { CSVRecord record = iterator.next(); String document = createDocument(fields, record.values()); IndexRequest request = Requests.indexRequest(indexName).source(document, XContentType.JSON); bulkRequest.add(request); - if (!iterator.hasNext() || bulkRequest.requests().size() == bulkSize) { + if (iterator.hasNext() == false || bulkRequest.requests().size() == bulkSize) { BulkResponse response = client.bulk(bulkRequest).actionGet(timeout); if (response.hasFailures()) { throw new OpenSearchException( @@ -351,4 +346,20 @@ public static void putGeoData( client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); } + + public AcknowledgedResponse deleteIp2GeoDataIndex(final String index) { + if (index == null || index.startsWith(IP2GEO_DATA_INDEX_NAME_PREFIX) == false) { + throw new OpenSearchException( + "the index[{}] is not ip2geo data index which should start with {}", + index, + IP2GEO_DATA_INDEX_NAME_PREFIX + ); + } + return client.admin() + .indices() + .prepareDelete(index) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java similarity index 85% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java rename to src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java index 7c5d77f9..c2127482 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutorHelper.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java @@ -18,8 +18,13 @@ /** * Provide a list of static methods related with executors for Ip2Geo */ -public class Ip2GeoExecutorHelper { +public class Ip2GeoExecutor { private static final String THREAD_POOL_NAME = "_plugin_geospatial_ip2geo_datasource_update"; + private final ThreadPool threadPool; + + public Ip2GeoExecutor(final ThreadPool threadPool) { + this.threadPool = threadPool; + } /** * We use fixed thread count of 1 for updating datasource as updating datasource is running background @@ -35,10 +40,9 @@ public static ExecutorBuilder executorBuilder(final Settings settings) { /** * Return an executor service for datasource update task * - * @param threadPool the thread pool * @return the executor service */ - public static ExecutorService forDatasourceUpdate(final ThreadPool threadPool) { + public ExecutorService forDatasourceUpdate() { return threadPool.executor(THREAD_POOL_NAME); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index 5df30c04..f8348435 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -36,10 +36,10 @@ public class Ip2GeoSettings { /** * Default update interval to be used in Ip2Geo datasource creation API */ - public static final Setting DATASOURCE_UPDATE_INTERVAL = Setting.timeSetting( + public static final Setting DATASOURCE_UPDATE_INTERVAL = Setting.longSetting( "plugins.geospatial.ip2geo.datasource.update_interval_in_days", - TimeValue.timeValueDays(3), - TimeValue.timeValueDays(1), + 3l, + 1l, Setting.Property.NodeScope, Setting.Property.Dynamic ); @@ -47,8 +47,8 @@ public class Ip2GeoSettings { /** * Timeout value for Ip2Geo processor */ - public static final Setting TIMEOUT_IN_SECONDS = Setting.timeSetting( - "plugins.geospatial.ip2geo.timeout_in_seconds", + public static final Setting TIMEOUT = Setting.timeSetting( + "plugins.geospatial.ip2geo.timeout", TimeValue.timeValueSeconds(30), TimeValue.timeValueSeconds(1), Setting.Property.NodeScope, @@ -66,16 +66,6 @@ public class Ip2GeoSettings { Setting.Property.Dynamic ); - /** - * Cache size for GeoIP data - */ - public static final Setting CACHE_SIZE = Setting.intSetting( - "plugins.geospatial.ip2geo.processor.cache_size", - 1000, - 0, - Setting.Property.NodeScope - ); - /** * Multi search bundle size for GeoIP data * @@ -113,9 +103,8 @@ public static final List> settings() { return List.of( DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, - TIMEOUT_IN_SECONDS, + TIMEOUT, INDEXING_BULK_SIZE, - CACHE_SIZE, MAX_BUNDLE_SIZE, MAX_CONCURRENT_SEARCHES ); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 4d42f9ad..dfc26d71 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -12,20 +12,24 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import lombok.AccessLevel; import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.ToString; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; @@ -38,6 +42,8 @@ */ @Getter @Setter +@ToString +@EqualsAndHashCode @AllArgsConstructor public class Datasource implements ScheduledJobParameter { /** @@ -45,6 +51,9 @@ public class Datasource implements ScheduledJobParameter { */ public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".ip2geo-data"; private static final long LOCK_DURATION_IN_SECONDS = 60 * 60; + private static final long MAX_JITTER_IN_MINUTES = 5; + private static final long ONE_DAY_IN_HOURS = 24; + private static final long ONE_HOUR_IN_MINUTES = 60; /** * Default fields for job scheduling @@ -173,22 +182,20 @@ public class Datasource implements ScheduledJobParameter { } - /** - * Visible for testing - */ - protected Datasource() { + @VisibleForTesting + public Datasource() { this(null, null, null); } public Datasource(final String id, final IntervalSchedule schedule, final String endpoint) { this( id, - Instant.now(), + Instant.now().truncatedTo(ChronoUnit.MILLIS), null, false, schedule, endpoint, - DatasourceState.PREPARING, + DatasourceState.CREATING, new ArrayList<>(), new Database(), new UpdateStats() @@ -264,14 +271,17 @@ public Long getLockDurationSeconds() { */ @Override public Double getJitter() { - return 5.0 / (schedule.getInterval() * 24 * 60); + return MAX_JITTER_IN_MINUTES / ((double) schedule.getInterval() * ONE_DAY_IN_HOURS * ONE_HOUR_IN_MINUTES); } /** * Enable auto update of GeoIP data */ public void enable() { - enabledTime = Instant.now(); + if (isEnabled == true) { + return; + } + enabledTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); isEnabled = true; } @@ -306,6 +316,11 @@ private String indexNameFor(final long suffix) { return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, suffix); } + /** + * Checks if datasource is expired or not + * + * @return true if datasource is expired false otherwise + */ public boolean isExpired() { if (database.validForInDays == null) { return false; @@ -322,12 +337,45 @@ public boolean isExpired() { return Instant.now().isAfter(lastCheckedAt.plus(database.validForInDays, ChronoUnit.DAYS)); } - public void setDatabase(final DatasourceManifest datasourceManifest, final String[] fields) { + /** + * Set database attributes with given input + * + * @param datasourceManifest the datasource manifest + * @param fields the fields + */ + public void setDatabase(final DatasourceManifest datasourceManifest, final List fields) { this.database.setProvider(datasourceManifest.getProvider()); this.database.setMd5Hash(datasourceManifest.getMd5Hash()); this.database.setUpdatedAt(Instant.ofEpochMilli(datasourceManifest.getUpdatedAt())); - this.database.setValidForInDays(database.validForInDays); - this.database.setFields(Arrays.asList(fields)); + this.database.setValidForInDays(datasourceManifest.getValidForInDays()); + this.database.setFields(fields); + } + + /** + * Checks if the database fields are compatible with the given set of fields. + * + * If database fields are null, it is compatible with any input fields + * as it hasn't been generated before. + * + * @param fields The set of input fields to check for compatibility. + * @return true if the database fields are compatible with the given input fields, false otherwise. + */ + public boolean isCompatible(final List fields) { + if (database.fields == null) { + return true; + } + + if (fields.size() < database.fields.size()) { + return false; + } + + Set fieldsSet = new HashSet<>(fields); + for (String field : database.fields) { + if (fieldsSet.contains(field) == false) { + return false; + } + } + return true; } /** @@ -335,6 +383,8 @@ public void setDatabase(final DatasourceManifest datasourceManifest, final Strin */ @Getter @Setter + @ToString + @EqualsAndHashCode @AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Database implements ToXContent { @@ -427,6 +477,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa */ @Getter @Setter + @ToString + @EqualsAndHashCode @AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class UpdateStats implements ToXContent { @@ -517,8 +569,8 @@ public static class Builder { public static Datasource build(final PutDatasourceRequest request) { String id = request.getDatasourceName(); IntervalSchedule schedule = new IntervalSchedule( - Instant.now(), - (int) request.getUpdateIntervalInDays().days(), + Instant.now().truncatedTo(ChronoUnit.MILLIS), + (int) request.getUpdateInterval().days(), ChronoUnit.DAYS ); String endpoint = request.getEndpoint(); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java index ed94f6fe..6def5295 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java @@ -15,7 +15,7 @@ /** * Datasource job scheduler extension * - * This extension is responsible for scheduling Ip2Geo data update task + * This extension is responsible for scheduling GeoIp data update task * * See https://github.com/opensearch-project/job-scheduler/blob/main/README.md#getting-started */ diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index c408ff3b..a75984b4 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -8,40 +8,27 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; -import java.net.URL; -import java.time.Duration; +import java.io.IOException; import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; import lombok.extern.log4j.Log4j2; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; -import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; -import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutorHelper; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; import org.opensearch.jobscheduler.spi.utils.LockService; -import org.opensearch.threadpool.ThreadPool; /** * Datasource update task * - * This is a background task which is responsible for updating Ip2Geo datasource + * This is a background task which is responsible for updating GeoIp data */ @Log4j2 public class DatasourceRunner implements ScheduledJobRunner { @@ -66,10 +53,10 @@ public static DatasourceRunner getJobRunnerInstance() { } private ClusterService clusterService; - private ThreadPool threadPool; private Client client; - private TimeValue timeout; - private Integer indexingBulkSize; + private DatasourceUpdateService datasourceUpdateService; + private Ip2GeoExecutor ip2GeoExecutor; + private DatasourceFacade datasourceFacade; private boolean initialized; private DatasourceRunner() { @@ -79,17 +66,18 @@ private DatasourceRunner() { /** * Initialize timeout and indexingBulkSize from settings */ - public void initialize(final ClusterService clusterService, final ThreadPool threadPool, final Client client) { + public void initialize( + final ClusterService clusterService, + final Client client, + final DatasourceUpdateService datasourceUpdateService, + final Ip2GeoExecutor ip2GeoExecutor, + final DatasourceFacade datasourceFacade + ) { this.clusterService = clusterService; - this.threadPool = threadPool; this.client = client; - - this.timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(clusterService.getSettings()); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> this.timeout = newValue); - this.indexingBulkSize = Ip2GeoSettings.INDEXING_BULK_SIZE.get(clusterService.getSettings()); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(Ip2GeoSettings.INDEXING_BULK_SIZE, newValue -> this.indexingBulkSize = newValue); + this.datasourceUpdateService = datasourceUpdateService; + this.ip2GeoExecutor = ip2GeoExecutor; + this.datasourceFacade = datasourceFacade; this.initialized = true; } @@ -102,11 +90,11 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC log.info("Update job started for a datasource[{}]", jobParameter.getName()); if (jobParameter instanceof Datasource == false) { throw new IllegalStateException( - "job parameter is not instance of DatasourceUpdateJobParameter, type: " + jobParameter.getClass().getCanonicalName() + "job parameter is not instance of Datasource, type: " + jobParameter.getClass().getCanonicalName() ); } - Ip2GeoExecutorHelper.forDatasourceUpdate(threadPool).submit(updateDatasourceRunner(jobParameter, context)); + ip2GeoExecutor.forDatasourceUpdate().submit(updateDatasourceRunner(jobParameter, context)); } /** @@ -118,161 +106,56 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC * @param jobParameter job parameter * @param context context */ - private Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { + @VisibleForTesting + protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { final LockService lockService = context.getLockService(); return () -> { - if (jobParameter.getLockDurationSeconds() != null) { - lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { - if (lock == null) { - return; - } - try { - Datasource datasource = DatasourceHelper.getDatasource(client, jobParameter.getName(), timeout); - if (datasource == null) { - log.info("Datasource[{}] is already deleted", jobParameter.getName()); - return; - } - - try { - deleteUnusedIndices(datasource); - updateDatasource(datasource); - deleteUnusedIndices(datasource); - } catch (Exception e) { - log.error("Failed to update datasource for {}", datasource.getId(), e); - datasource.getUpdateStats().setLastFailedAt(Instant.now()); - DatasourceHelper.updateDatasource(client, datasource, timeout); - } - } finally { - lockService.release( - lock, - ActionListener.wrap(released -> {}, exception -> { log.error("Failed to release lock [{}]", lock); }) - ); - } - }, exception -> { log.error("Failed to acquire lock for job [{}]", jobParameter.getName()); })); - } - }; - - } - - /** - * Delete all indices except the one which are being used - * - * @param parameter - */ - private void deleteUnusedIndices(final Datasource parameter) { - try { - List deletedIndices = new ArrayList<>(); - for (String index : parameter.getIndices()) { - if (index.equals(parameter.currentIndexName())) { - continue; + lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { + if (lock == null) { + return; } - - if (!clusterService.state().metadata().hasIndex(index)) { - deletedIndices.add(index); - continue; - } - try { - if (client.admin() - .indices() - .prepareDelete(index) - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) - .execute() - .actionGet(timeout) - .isAcknowledged()) { - deletedIndices.add(index); - } else { - log.error("Failed to delete an index [{}]", index); - } - } catch (Exception e) { - log.error("Failed to delete an index [{}]", index, e); + updateDatasource(jobParameter); + } finally { + lockService.release( + lock, + ActionListener.wrap(released -> {}, exception -> { log.error("Failed to release lock [{}]", lock, exception); }) + ); } - } - if (!deletedIndices.isEmpty()) { - parameter.getIndices().removeAll(deletedIndices); - DatasourceHelper.updateDatasource(client, parameter, timeout); - } - } catch (Exception e) { - log.error("Failed to delete old indices for {}", parameter.getId(), e); - } + }, exception -> { log.error("Failed to acquire lock for job [{}]", jobParameter.getName(), exception); })); + }; } - /** - * Update GeoIP data internal - * - * @param jobParameter - * @throws Exception - */ - private void updateDatasource(final Datasource jobParameter) throws Exception { - if (!DatasourceState.AVAILABLE.equals(jobParameter.getState())) { - log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, jobParameter.getState()); - jobParameter.disable(); - jobParameter.getUpdateStats().setLastFailedAt(Instant.now()); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); + @VisibleForTesting + protected void updateDatasource(final ScheduledJobParameter jobParameter) throws IOException { + Datasource datasource = datasourceFacade.getDatasource(jobParameter.getName()); + /** + * If delete request comes while update task is waiting on a queue for other update tasks to complete, + * because update task for this datasource didn't acquire a lock yet, delete request is processed. + * When it is this datasource's turn to run, it will find that the datasource is deleted already. + * Therefore, we stop the update process when data source does not exist. + */ + if (datasource == null) { + log.info("Datasource[{}] does not exist", jobParameter.getName()); return; } - URL url = new URL(jobParameter.getEndpoint()); - DatasourceManifest manifest = DatasourceManifest.Builder.build(url); - - if (skipUpdate(jobParameter, manifest)) { - log.info("Skipping GeoIP database update. Update is not required for {}", jobParameter.getId()); - jobParameter.getUpdateStats().setLastSkippedAt(Instant.now()); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); + if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { + log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, datasource.getState()); + datasource.disable(); + datasource.getUpdateStats().setLastFailedAt(Instant.now()); + datasourceFacade.updateDatasource(datasource); return; } - Instant startTime = Instant.now(); - String indexName = jobParameter.indexNameFor(manifest); - jobParameter.getIndices().add(indexName); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - GeoIpDataHelper.createIndexIfNotExists(clusterService, client, indexName, timeout); - String[] fields; - try (CSVParser reader = GeoIpDataHelper.getDatabaseReader(manifest)) { - Iterator iter = reader.iterator(); - fields = iter.next().values(); - if (!jobParameter.getDatabase().getFields().equals(Arrays.asList(fields))) { - throw new OpenSearchException( - "fields does not match between old [{}] and new [{}]", - jobParameter.getDatabase().getFields().toString(), - Arrays.asList(fields).toString() - ); - } - GeoIpDataHelper.putGeoData(client, indexName, fields, iter, indexingBulkSize, timeout); - } - - Instant endTime = Instant.now(); - jobParameter.getDatabase().setProvider(manifest.getProvider()); - jobParameter.getDatabase().setMd5Hash(manifest.getMd5Hash()); - jobParameter.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); - jobParameter.getDatabase().setValidForInDays(manifest.getValidForInDays()); - jobParameter.getDatabase().setFields(Arrays.asList(fields)); - jobParameter.getUpdateStats().setLastSucceededAt(endTime); - jobParameter.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); - DatasourceHelper.updateDatasource(client, jobParameter, timeout); - log.info( - "GeoIP database creation succeeded for {} and took {} seconds", - jobParameter.getId(), - Duration.between(startTime, endTime) - ); - } - - /** - * Determine if update is needed or not - * - * Update is needed when all following conditions are met - * 1. MD5 hash value in datasource is different with MD5 hash value in manifest - * 2. updatedAt value in datasource is before updateAt value in manifest - * - * @param parameter - * @param manifest - * @return - */ - private boolean skipUpdate(final Datasource parameter, final DatasourceManifest manifest) { - if (manifest.getMd5Hash().equals(parameter.getDatabase().getMd5Hash())) { - return true; + try { + datasourceUpdateService.deleteUnusedIndices(datasource); + datasourceUpdateService.updateOrCreateGeoIpData(datasource); + datasourceUpdateService.deleteUnusedIndices(datasource); + } catch (Exception e) { + log.error("Failed to update datasource for {}", datasource.getId(), e); + datasource.getUpdateStats().setLastFailedAt(Instant.now()); + datasourceFacade.updateDatasource(datasource); } - - return parameter.getDatabase().getUpdatedAt().toEpochMilli() >= manifest.getUpdatedAt(); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java new file mode 100644 index 00000000..d3814805 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -0,0 +1,218 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import java.io.IOException; +import java.net.URL; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import lombok.extern.log4j.Log4j2; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.opensearch.OpenSearchException; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; + +@Log4j2 +public class DatasourceUpdateService { + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final Client client; + private final DatasourceFacade datasourceFacade; + private final GeoIpDataFacade geoIpDataFacade; + + public DatasourceUpdateService( + final ClusterService clusterService, + final Client client, + final DatasourceFacade datasourceFacade, + final GeoIpDataFacade geoIpDataFacade + ) { + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.client = client; + this.datasourceFacade = datasourceFacade; + this.geoIpDataFacade = geoIpDataFacade; + } + + /** + * Update GeoIp data + * + * @param datasource + * @throws Exception + */ + public void updateOrCreateGeoIpData(final Datasource datasource) throws Exception { + URL url = new URL(datasource.getEndpoint()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(url); + + if (shouldUpdate(datasource, manifest) == false) { + log.info("Skipping GeoIP database update. Update is not required for {}", datasource.getId()); + datasource.getUpdateStats().setLastSkippedAt(Instant.now()); + datasourceFacade.updateDatasource(datasource); + return; + } + + Instant startTime = Instant.now(); + String indexName = setupIndex(manifest, datasource); + String[] header; + List fieldsToStore; + try (CSVParser reader = geoIpDataFacade.getDatabaseReader(manifest)) { + CSVRecord headerLine = reader.iterator().next(); + header = validateHeader(headerLine).values(); + fieldsToStore = Arrays.asList(header).subList(1, header.length); + if (datasource.isCompatible(fieldsToStore) == false) { + throw new OpenSearchException( + "new fields [{}] does not contain all old fields [{}]", + fieldsToStore.toString(), + datasource.getDatabase().getFields().toString() + ); + } + geoIpDataFacade.putGeoIpData(indexName, header, reader.iterator(), clusterSettings.get(Ip2GeoSettings.INDEXING_BULK_SIZE)); + } + + Instant endTime = Instant.now(); + updateDatasourceAsSucceeded(datasource, manifest, fieldsToStore, startTime, endTime); + } + + /** + * Delete all indices except the one which are being used + * + * @param parameter + */ + public void deleteUnusedIndices(final Datasource parameter) { + try { + List indicesToDelete = parameter.getIndices() + .stream() + .filter(index -> index.equals(parameter.currentIndexName()) == false) + .collect(Collectors.toList()); + + List deletedIndices = deleteIndices(indicesToDelete); + + if (deletedIndices.isEmpty() == false) { + parameter.getIndices().removeAll(deletedIndices); + datasourceFacade.updateDatasource(parameter); + } + } catch (Exception e) { + log.error("Failed to delete old indices for {}", parameter.getId(), e); + } + } + + private List deleteIndices(final List indicesToDelete) { + List deletedIndices = new ArrayList<>(indicesToDelete.size()); + for (String index : indicesToDelete) { + if (clusterService.state().metadata().hasIndex(index) == false) { + deletedIndices.add(index); + continue; + } + + try { + if (geoIpDataFacade.deleteIp2GeoDataIndex(index).isAcknowledged()) { + deletedIndices.add(index); + } else { + log.error("Failed to delete an index [{}]", index); + } + } catch (Exception e) { + log.error("Failed to delete an index [{}]", index, e); + } + } + return deletedIndices; + } + + /** + * Validate header + * + * 1. header should not be null + * 2. the number of values in header should be more than one + * + * @param header the header + * @return CSVRecord the input header + */ + private CSVRecord validateHeader(CSVRecord header) { + if (header == null) { + throw new OpenSearchException("geoip database is empty"); + } + if (header.values().length < 2) { + throw new OpenSearchException("geoip database should have at least two fields"); + } + return header; + } + + /*** + * Update datasource as succeeded + * + * @param manifest the manifest + * @param datasource the datasource + * @return + * @throws IOException + */ + private void updateDatasourceAsSucceeded( + final Datasource datasource, + final DatasourceManifest manifest, + final List fields, + final Instant startTime, + final Instant endTime + ) throws IOException { + datasource.setDatabase(manifest, fields); + datasource.getUpdateStats().setLastSucceededAt(endTime); + datasource.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); + datasource.enable(); + datasource.setState(DatasourceState.AVAILABLE); + datasourceFacade.updateDatasource(datasource); + log.info("GeoIP database creation succeeded for {} and took {} seconds", datasource.getId(), Duration.between(startTime, endTime)); + } + + /*** + * Setup index to add a new geoip data + * + * @param manifest the manifest + * @param datasource the datasource + * @return + * @throws IOException + */ + private String setupIndex(final DatasourceManifest manifest, final Datasource datasource) throws IOException { + String indexName = datasource.indexNameFor(manifest); + datasource.getIndices().add(indexName); + datasourceFacade.updateDatasource(datasource); + geoIpDataFacade.createIndexIfNotExists(indexName); + return indexName; + } + + /** + * Determine if update is needed or not + * + * Update is needed when all following conditions are met + * 1. updatedAt value in datasource is equal or before updateAt value in manifest + * 2. MD5 hash value in datasource is different with MD5 hash value in manifest + * + * @param datasource + * @param manifest + * @return + */ + private boolean shouldUpdate(final Datasource datasource, final DatasourceManifest manifest) { + if (datasource.getDatabase().getUpdatedAt().toEpochMilli() > manifest.getUpdatedAt()) { + return false; + } + + if (manifest.getMd5Hash().equals(datasource.getDatabase().getMd5Hash())) { + return false; + } + return true; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java deleted file mode 100644 index 2bcc1ed5..00000000 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCache.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.geospatial.ip2geo.processor; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.function.Function; - -import org.opensearch.common.cache.Cache; -import org.opensearch.common.cache.CacheBuilder; -import org.opensearch.common.unit.TimeValue; - -/** - * The in-memory cache for the ip2geo data. There should only be 1 instance of this class. - */ -public class Ip2GeoCache { - private static final TimeValue CACHING_PERIOD = TimeValue.timeValueMinutes(1); - private final Cache> cache; - - /** - * Default constructor - * - * @param maxSize size of a cache - */ - public Ip2GeoCache(final long maxSize) { - if (maxSize < 0) { - throw new IllegalArgumentException("ip2geo max cache size must be 0 or greater"); - } - this.cache = CacheBuilder.>builder() - .setMaximumWeight(maxSize) - .setExpireAfterWrite(CACHING_PERIOD) - .build(); - } - - /** - * Put data in a cache if it is absent and return the data - * - * @param ip the first part of a key - * @param datasourceName the second part of a key - * @param retrieveFunction function to retrieve a data to be stored in a cache - * @return data in a cache - */ - public Map putIfAbsent( - final String ip, - final String datasourceName, - final Function> retrieveFunction - ) { - CacheKey cacheKey = new CacheKey(ip, datasourceName); - Map response = cache.get(cacheKey); - if (response == null) { - response = retrieveFunction.apply(ip); - response = response == null ? Collections.emptyMap() : response; - cache.put(cacheKey, response); - } - return response; - } - - /** - * Put data in a cache - * - * @param ip the first part of a key - * @param datasourceName the second part of a key - * @param data the data - */ - public void put(final String ip, final String datasourceName, final Map data) { - CacheKey cacheKey = new CacheKey(ip, datasourceName); - cache.put(cacheKey, data); - } - - protected Map get(final String ip, final String datasourceName) { - CacheKey cacheKey = new CacheKey(ip, datasourceName); - return cache.get(cacheKey); - } - - /** - * The key to use for the cache. Since this cache can span multiple ip2geo processors that all use different datasource, the datasource - * name is needed to be included in the cache key. For example, if we only used the IP address as the key the same IP may be in multiple - * datasource with different values. The datasource name scopes the IP to the correct datasource - */ - private static class CacheKey { - - private final String ip; - private final String datasourceName; - - private CacheKey(final String ip, final String datasourceName) { - this.ip = ip; - this.datasourceName = datasourceName; - } - - // generated - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CacheKey cacheKey = (CacheKey) o; - return Objects.equals(ip, cacheKey.ip) && Objects.equals(datasourceName, cacheKey.datasourceName); - } - - // generated - @Override - public int hashCode() { - return Objects.hash(ip, datasourceName); - } - } -} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 0d6ddd8c..a0f8974d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -21,17 +21,17 @@ import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; +import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.geospatial.ip2geo.common.DatasourceHelper; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataHelper; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.ingest.AbstractProcessor; @@ -52,12 +52,10 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final Set properties; private final boolean ignoreMissing; private final boolean firstOnly; - private final Ip2GeoCache cache; private final Client client; - private final ClusterService clusterService; - - private int maxBundleSize; - private int maxConcurrentSearches; + private final ClusterSettings clusterSettings; + private final DatasourceFacade datasourceFacade; + private final GeoIpDataFacade geoIpDataFacade; /** * Ip2Geo processor type @@ -74,9 +72,8 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param properties the properties * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array - * @param cache the Ip2Geo cache * @param client the client - * @param clusterService the cluster service + * @param clusterSettings the cluster settings */ public Ip2GeoProcessor( final String tag, @@ -87,9 +84,10 @@ public Ip2GeoProcessor( final Set properties, final boolean ignoreMissing, final boolean firstOnly, - final Ip2GeoCache cache, final Client client, - final ClusterService clusterService + final ClusterSettings clusterSettings, + final DatasourceFacade datasourceFacade, + final GeoIpDataFacade geoIpDataFacade ) { super(tag, description); this.field = field; @@ -98,15 +96,10 @@ public Ip2GeoProcessor( this.properties = properties; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; - this.cache = cache; this.client = client; - this.clusterService = clusterService; - - maxBundleSize = clusterService.getClusterSettings().get(Ip2GeoSettings.MAX_BUNDLE_SIZE); - clusterService.getClusterSettings().addSettingsUpdateConsumer(Ip2GeoSettings.MAX_BUNDLE_SIZE, newValue -> maxBundleSize = newValue); - maxConcurrentSearches = clusterService.getClusterSettings().get(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES, newValue -> maxConcurrentSearches = newValue); + this.clusterSettings = clusterSettings; + this.datasourceFacade = datasourceFacade; + this.geoIpDataFacade = geoIpDataFacade; } /** @@ -119,12 +112,9 @@ public Ip2GeoProcessor( public void execute(IngestDocument ingestDocument, BiConsumer handler) { Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); - if (ip == null && ignoreMissing) { + if (ip == null) { handler.accept(ingestDocument, null); return; - } else if (ip == null) { - handler.accept(null, new IllegalArgumentException("field [" + field + "] is null, cannot extract geo information.")); - return; } if (ip instanceof String) { @@ -147,49 +137,23 @@ public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException("Not implemented"); } - /** - * Handle single ip - * - * @param ingestDocument the document - * @param handler the handler - * @param ip the ip - */ - private void executeInternal( + @VisibleForTesting + protected void executeInternal( final IngestDocument ingestDocument, final BiConsumer handler, final String ip ) { - Map geoData = cache.get(ip, datasourceName); - if (geoData != null) { - if (!geoData.isEmpty()) { - ingestDocument.setFieldValue(targetField, filteredGeoData(geoData, ip)); - } - handler.accept(ingestDocument, null); - return; - } - - DatasourceHelper.getDatasource(client, datasourceName, new ActionListener<>() { + datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { - if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource does not exist")); + if (handleInvalidDatasource(ingestDocument, datasource, handler)) { return; } - if (datasource.isExpired()) { - ingestDocument.setFieldValue(targetField, DATA_EXPIRED); - handler.accept(ingestDocument, null); - return; - } - - GeoIpDataHelper.getGeoData(client, datasource.currentIndexName(), ip, new ActionListener<>() { + geoIpDataFacade.getGeoIpData(datasource.currentIndexName(), ip, new ActionListener<>() { @Override - public void onResponse(final Map stringObjectMap) { - cache.put(ip, datasourceName, stringObjectMap); - if (!stringObjectMap.isEmpty()) { - ingestDocument.setFieldValue(targetField, filteredGeoData(stringObjectMap, ip)); - } - handler.accept(ingestDocument, null); + public void onResponse(final Map ipToGeoData) { + handleSingleIp(ip, ipToGeoData, ingestDocument, handler); } @Override @@ -206,6 +170,34 @@ public void onFailure(final Exception e) { }); } + @VisibleForTesting + protected void handleSingleIp( + final String ip, + final Map ipToGeoData, + final IngestDocument ingestDocument, + final BiConsumer handler + ) { + if (ipToGeoData.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData, ip)); + } + handler.accept(ingestDocument, null); + } + + private Map filteredGeoData(final Map geoData, final String ip) { + Map filteredGeoData; + if (properties == null) { + return geoData; + } + + filteredGeoData = properties.stream() + .filter(p -> p.equals(PROPERTY_IP) == false) + .collect(Collectors.toMap(p -> p, p -> geoData.get(p))); + if (properties.contains(PROPERTY_IP)) { + filteredGeoData.put(PROPERTY_IP, ip); + } + return filteredGeoData; + } + /** * Handle multiple ips * @@ -213,7 +205,8 @@ public void onFailure(final Exception e) { * @param handler the handler * @param ips the ip list */ - private void executeInternal( + @VisibleForTesting + protected void executeInternal( final IngestDocument ingestDocument, final BiConsumer handler, final List ips @@ -223,72 +216,23 @@ private void executeInternal( if (ip instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - String ipAddr = (String) ip; - data.put(ipAddr, cache.get(ipAddr, datasourceName)); } List ipList = (List) ips; - DatasourceHelper.getDatasource(client, datasourceName, new ActionListener<>() { + datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { - if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource does not exist")); + if (handleInvalidDatasource(ingestDocument, datasource, handler)) { return; } - if (datasource.isExpired()) { - ingestDocument.setFieldValue(targetField, DATA_EXPIRED); - handler.accept(ingestDocument, null); - return; - } - GeoIpDataHelper.getGeoData( - client, + geoIpDataFacade.getGeoIpData( datasource.currentIndexName(), ipList.iterator(), - maxBundleSize, - maxConcurrentSearches, + clusterSettings.get(Ip2GeoSettings.MAX_BUNDLE_SIZE), + clusterSettings.get(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES), firstOnly, data, - new ActionListener<>() { - @Override - public void onResponse(final Object obj) { - for (Map.Entry> entry : data.entrySet()) { - cache.put(entry.getKey(), datasourceName, entry.getValue()); - } - - if (firstOnly) { - for (String ipAddr : ipList) { - Map geoData = data.get(ipAddr); - // GeoData for ipAddr won't be null - if (!geoData.isEmpty()) { - ingestDocument.setFieldValue(targetField, geoData); - handler.accept(ingestDocument, null); - return; - } - } - handler.accept(ingestDocument, null); - } else { - boolean match = false; - List> geoDataList = new ArrayList<>(ipList.size()); - for (String ipAddr : ipList) { - Map geoData = data.get(ipAddr); - // GeoData for ipAddr won't be null - geoDataList.add(geoData.isEmpty() ? null : geoData); - if (!geoData.isEmpty()) { - match = true; - } - } - if (match) { - ingestDocument.setFieldValue(targetField, geoDataList); - } - handler.accept(ingestDocument, null); - } - } - - @Override - public void onFailure(final Exception e) { - handler.accept(null, e); - } - } + listenerToAppendDataToDocument(data, ipList, ingestDocument, handler) ); } @@ -299,21 +243,70 @@ public void onFailure(final Exception e) { }); } - private Map filteredGeoData(final Map geoData, final String ip) { - Map filteredGeoData; - if (properties == null) { - filteredGeoData = geoData; - } else { - filteredGeoData = new HashMap<>(); - for (String property : this.properties) { - if (property.equals(PROPERTY_IP)) { - filteredGeoData.put(PROPERTY_IP, ip); + @VisibleForTesting + protected ActionListener>> listenerToAppendDataToDocument( + final Map> data, + final List ipList, + final IngestDocument ingestDocument, + final BiConsumer handler + ) { + return new ActionListener<>() { + @Override + public void onResponse(final Map> response) { + if (firstOnly) { + for (String ipAddr : ipList) { + Map geoData = data.get(ipAddr); + // GeoData for ipAddr won't be null + if (geoData.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, filteredGeoData(geoData, ipAddr)); + handler.accept(ingestDocument, null); + return; + } + } } else { - filteredGeoData.put(property, geoData.get(property)); + boolean match = false; + List> geoDataList = new ArrayList<>(ipList.size()); + for (String ipAddr : ipList) { + Map geoData = data.get(ipAddr); + // GeoData for ipAddr won't be null + geoDataList.add(geoData.isEmpty() ? null : filteredGeoData(geoData, ipAddr)); + if (geoData.isEmpty() == false) { + match = true; + } + } + if (match) { + ingestDocument.setFieldValue(targetField, geoDataList); + handler.accept(ingestDocument, null); + return; + } } + handler.accept(ingestDocument, null); + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); } + }; + } + + @VisibleForTesting + protected boolean handleInvalidDatasource( + final IngestDocument ingestDocument, + final Datasource datasource, + final BiConsumer handler + ) { + if (datasource == null) { + handler.accept(null, new IllegalStateException("datasource does not exist")); + return true; } - return filteredGeoData; + + if (datasource.isExpired()) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); + return true; + } + return false; } @Override @@ -325,26 +318,27 @@ public String getType() { * Ip2Geo processor factory */ public static final class Factory implements Processor.Factory { - private final Ip2GeoCache cache; private final Client client; private final IngestService ingestService; - private TimeValue timeout; + private final DatasourceFacade datasourceFacade; + private final GeoIpDataFacade geoIpDataFacade; /** * Default constructor * - * @param cache the cache * @param client the client * @param ingestService the ingest service */ - public Factory(final Ip2GeoCache cache, final Client client, final IngestService ingestService) { - this.cache = cache; + public Factory( + final Client client, + final IngestService ingestService, + final DatasourceFacade datasourceFacade, + final GeoIpDataFacade geoIpDataFacade + ) { this.client = client; this.ingestService = ingestService; - - timeout = Ip2GeoSettings.TIMEOUT_IN_SECONDS.get(client.settings()); - ClusterSettings clusterSettings = ingestService.getClusterService().getClusterSettings(); - clusterSettings.addSettingsUpdateConsumer(Ip2GeoSettings.TIMEOUT_IN_SECONDS, newValue -> timeout = newValue); + this.datasourceFacade = datasourceFacade; + this.geoIpDataFacade = geoIpDataFacade; } /** @@ -373,7 +367,7 @@ public Ip2GeoProcessor create( boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); // Skip validation for the call by cluster applier service - if (!Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME)) { + if (Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME) == false) { validate(processorTag, datasourceName, propertyNames); } @@ -386,20 +380,21 @@ public Ip2GeoProcessor create( propertyNames == null ? null : new HashSet<>(propertyNames), ignoreMissing, firstOnly, - cache, client, - ingestService.getClusterService() + ingestService.getClusterService().getClusterSettings(), + datasourceFacade, + geoIpDataFacade ); } private void validate(final String processorTag, final String datasourceName, final List propertyNames) throws IOException { - Datasource datasource = DatasourceHelper.getDatasource(client, datasourceName, timeout); + Datasource datasource = datasourceFacade.getDatasource(datasourceName); if (datasource == null) { throw newConfigurationException(TYPE, processorTag, "datasource", "datasource [" + datasourceName + "] doesn't exist"); } - if (!DatasourceState.AVAILABLE.equals(datasource.getState())) { + if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { throw newConfigurationException( TYPE, processorTag, @@ -416,7 +411,7 @@ private void validate(final String processorTag, final String datasourceName, fi final Set availableProperties = new HashSet<>(datasource.getDatabase().getFields()); availableProperties.add(PROPERTY_IP); for (String fieldName : propertyNames) { - if (!availableProperties.contains(fieldName)) { + if (availableProperties.contains(fieldName) == false) { throw newConfigurationException( TYPE, processorTag, diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 1809cef6..e468b6b1 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -39,10 +39,12 @@ import org.opensearch.geospatial.ip2geo.action.PutDatasourceAction; import org.opensearch.geospatial.ip2geo.action.PutDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutorHelper; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; -import org.opensearch.geospatial.ip2geo.processor.Ip2GeoCache; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; @@ -87,9 +89,10 @@ public Map getProcessors(Processor.Parameters paramet .put( Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory( - new Ip2GeoCache(Ip2GeoSettings.CACHE_SIZE.get(parameters.client.settings())), parameters.client, - parameters.ingestService + parameters.ingestService, + new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService().getClusterSettings()), + new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client) ) ) .immutableMap(); @@ -98,7 +101,7 @@ public Map getProcessors(Processor.Parameters paramet @Override public List> getExecutorBuilders(Settings settings) { List> executorBuilders = new ArrayList<>(); - executorBuilders.add(Ip2GeoExecutorHelper.executorBuilder(settings)); + executorBuilders.add(Ip2GeoExecutor.executorBuilder(settings)); return executorBuilders; } @@ -121,10 +124,23 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - // Initialize DatasourceUpdateRunner - DatasourceRunner.getJobRunnerInstance().initialize(clusterService, threadPool, client); - - return List.of(UploadStats.getInstance()); + GeoIpDataFacade geoIpDataFacade = new GeoIpDataFacade(clusterService, client); + DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService.getClusterSettings()); + DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService( + clusterService, + client, + datasourceFacade, + geoIpDataFacade + ); + Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); + /** + * We don't need to return datasource runner because it is used only by job scheduler and job scheduler + * does not use DI but it calls DatasourceExtension#getJobRunner to get DatasourceRunner instance. + */ + DatasourceRunner.getJobRunnerInstance() + .initialize(clusterService, client, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + + return List.of(UploadStats.getInstance(), datasourceUpdateService, datasourceFacade, ip2GeoExecutor, geoIpDataFacade); } @Override @@ -137,7 +153,7 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction(), new RestPutDatasourceHandler(settings, clusterSettings)); + return List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction(), new RestPutDatasourceHandler(clusterSettings)); } @Override diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java new file mode 100644 index 00000000..c1aec21e --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.junit.After; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionResponse; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Randomness; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.ingest.IngestService; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskListener; +import org.opensearch.test.client.NoOpNodeClient; +import org.opensearch.test.rest.RestActionTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public abstract class Ip2GeoTestCase extends RestActionTestCase { + @Mock + protected ClusterService clusterService; + @Mock + protected DatasourceUpdateService datasourceUpdateService; + @Mock + protected DatasourceFacade datasourceFacade; + @Mock + protected Ip2GeoExecutor ip2GeoExecutor; + @Mock + protected ExecutorService executorService; + @Mock + protected GeoIpDataFacade geoIpDataFacade; + @Mock + protected ClusterState clusterState; + @Mock + protected Metadata metadata; + @Mock + protected IngestService ingestService; + @Mock + protected ActionFilters actionFilters; + @Mock + protected ThreadPool threadPool; + @Mock + protected TransportService transportService; + protected NoOpNodeClient client; + protected VerifyingClient verifyingClient; + protected LockService lockService; + protected ClusterSettings clusterSettings; + protected Settings settings; + private AutoCloseable openMocks; + + @Before + public void prepareIp2GeoTestCase() { + openMocks = MockitoAnnotations.openMocks(this); + settings = Settings.EMPTY; + client = new NoOpNodeClient(this.getTestName()); + verifyingClient = spy(new VerifyingClient(this.getTestName())); + clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings())); + lockService = new LockService(client, clusterService); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.state()).thenReturn(clusterState); + when(clusterState.metadata()).thenReturn(metadata); + when(clusterState.routingTable()).thenReturn(RoutingTable.EMPTY_ROUTING_TABLE); + when(ip2GeoExecutor.forDatasourceUpdate()).thenReturn(executorService); + when(ingestService.getClusterService()).thenReturn(clusterService); + when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); + } + + @After + public void clean() throws Exception { + openMocks.close(); + client.close(); + verifyingClient.close(); + } + + public DatasourceState randomStateExcept(DatasourceState state) { + assertNotNull(state); + return Arrays.stream(DatasourceState.values()) + .sequential() + .filter(s -> !s.equals(state)) + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 2)); + } + + public String randomIpAddress() { + return String.format( + Locale.ROOT, + "%d.%d.%d.%d", + Randomness.get().nextInt(255), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255) + ); + } + + @SuppressForbidden(reason = "unit test") + public String sampleManifestUrl() throws Exception { + return Paths.get(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").toURI()).toUri().toURL().toExternalForm(); + } + + @SuppressForbidden(reason = "unit test") + public String sampleManifestUrlWithInvalidUrl() throws Exception { + return Paths.get(this.getClass().getClassLoader().getResource("ip2geo/manifest_invalid_url.json").toURI()) + .toUri() + .toURL() + .toExternalForm(); + } + + @SuppressForbidden(reason = "unit test") + public File sampleIp2GeoFile() { + return new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + } + + /** + * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) + * is merged in OpenSearch core + */ + public static class VerifyingClient extends NoOpNodeClient { + AtomicReference executeVerifier = new AtomicReference<>(); + AtomicReference executeLocallyVerifier = new AtomicReference<>(); + + public VerifyingClient(String testName) { + super(testName); + reset(); + } + + /** + * Clears any previously set verifier functions set by {@link #setExecuteVerifier(BiFunction)} and/or + * {@link #setExecuteLocallyVerifier(BiFunction)}. These functions are replaced with functions which will throw an + * {@link AssertionError} if called. + */ + public void reset() { + executeVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + executeLocallyVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + } + + /** + * Sets the function that will be called when {@link #doExecute(ActionType, ActionRequest, ActionListener)} is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #doExecute(ActionType, ActionRequest, ActionListener)} + */ + public void setExecuteVerifier( + BiFunction, Request, Response> verifier + ) { + executeVerifier.set(verifier); + } + + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + listener.onResponse((Response) executeVerifier.get().apply(action, request)); + } + + /** + * Sets the function that will be called when {@link #executeLocally(ActionType, ActionRequest, TaskListener)}is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #executeLocally(ActionType, ActionRequest, TaskListener)} + */ + public void setExecuteLocallyVerifier( + BiFunction, Request, Response> verifier + ) { + executeLocallyVerifier.set(verifier); + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + ActionListener listener + ) { + listener.onResponse((Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + TaskListener listener + ) { + listener.onResponse(null, (Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 3198f26a..383d832a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -11,25 +11,31 @@ import java.util.Locale; import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.Randomness; +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; -public class PutDatasourceRequestTests extends OpenSearchTestCase { +public class PutDatasourceRequestTests extends Ip2GeoTestCase { - public void testValidateInvalidUrl() { - PutDatasourceRequest request = new PutDatasourceRequest("test"); + public void testValidateWithInvalidUrl() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint("invalidUrl"); - request.setUpdateIntervalInDays(TimeValue.ZERO); + request.setUpdateInterval(TimeValue.timeValueDays(1)); ActionRequestValidationException exception = request.validate(); assertEquals(1, exception.validationErrors().size()); assertEquals("Invalid URL format is provided", exception.validationErrors().get(0)); } - public void testValidateInvalidManifestFile() { - PutDatasourceRequest request = new PutDatasourceRequest("test"); - request.setDatasourceName("test"); - request.setEndpoint("https://hi.com"); - request.setUpdateIntervalInDays(TimeValue.ZERO); + public void testValidateWithInvalidManifestFile() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String domain = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); + request.setUpdateInterval(TimeValue.timeValueDays(1)); ActionRequestValidationException exception = request.validate(); assertEquals(1, exception.validationErrors().size()); assertEquals( @@ -37,4 +43,74 @@ public void testValidateInvalidManifestFile() { exception.validationErrors().get(0) ); } + + public void testValidate() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + assertNull(request.validate()); + } + + public void testValidateWithZeroUpdateInterval() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(0)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertEquals( + String.format(Locale.ROOT, "Update interval should be equal to or larger than 1 day"), + exception.validationErrors().get(0) + ); + } + + public void testValidateWithLargeUpdateInterval() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(30)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("should be smaller")); + } + + public void testValidateWithInvalidUrlInsideManifest() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrlWithInvalidUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("Invalid URL format")); + } + + public void testStreamInOut() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String domain = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); + request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(30) + 1)); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + PutDatasourceRequest copiedRequest = new PutDatasourceRequest(input); + + // Verify + assertEquals(request, copiedRequest); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java new file mode 100644 index 00000000..38226c7f --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import org.junit.Before; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.tasks.Task; + +public class PutDatasourceTransportActionTests extends Ip2GeoTestCase { + private PutDatasourceTransportAction action; + + @Before + public void init() { + action = new PutDatasourceTransportAction( + transportService, + actionFilters, + verifyingClient, + threadPool, + datasourceFacade, + datasourceUpdateService + ); + } + + public void testDoExecute() throws Exception { + Task task = mock(Task.class); + PutDatasourceRequest request = new PutDatasourceRequest("test"); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + ActionListener listener = mock(ActionListener.class); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest indexRequest = (IndexRequest) actionRequest; + assertEquals(DatasourceExtension.JOB_INDEX_NAME, indexRequest.index()); + assertEquals(request.getDatasourceName(), indexRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); + assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); + return null; + }); + action.doExecute(task, request, listener); + verify(verifyingClient).index(any(IndexRequest.class), any(ActionListener.class)); + verify(listener).onResponse(new AcknowledgedResponse(true)); + } + + public void testIndexResponseListenerFailure() { + Datasource datasource = new Datasource(); + ActionListener listener = mock(ActionListener.class); + action.getIndexResponseListener(datasource, listener) + .onFailure( + new VersionConflictEngineException( + null, + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString() + ) + ); + verify(listener).onFailure(any(ResourceAlreadyExistsException.class)); + } + + public void testCreateDatasourceInvalidState() throws Exception { + Datasource datasource = new Datasource(); + datasource.setState(randomStateExcept(DatasourceState.CREATING)); + datasource.getUpdateStats().setLastFailedAt(null); + + // Run + action.createDatasource(datasource); + + // Verify + assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(datasourceFacade).updateDatasource(datasource); + } + + public void testCreateDatasourceWithException() throws Exception { + Datasource datasource = new Datasource(); + doThrow(new RuntimeException()).when(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + + // Run + action.createDatasource(datasource); + + // Verify + assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(datasourceFacade).updateDatasource(datasource); + } + + public void testCreateDatasource() throws Exception { + Datasource datasource = new Datasource(); + + // Run + action.createDatasource(datasource); + + // Verify + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + assertEquals(DatasourceState.CREATING, datasource.getState()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index 82cae4ad..b812cdfa 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -8,67 +8,78 @@ package org.opensearch.geospatial.ip2geo.action; +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; + import java.util.HashSet; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; import org.opensearch.test.rest.RestActionTestCase; +@SuppressForbidden(reason = "unit test") public class RestPutDatasourceHandlerTests extends RestActionTestCase { + private String path; private RestPutDatasourceHandler action; @Before public void setupAction() { - action = new RestPutDatasourceHandler(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings()))); + action = new RestPutDatasourceHandler(new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings()))); controller().registerHandler(action); + path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s"); } public void testPrepareRequest() { - String content = "{\"endpoint\":\"https://test.com\", \"update_interval\":1}"; - RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath("/_geoip/datasource/test") + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}"; + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath(String.format(path, datasourceName)) .withContent(new BytesArray(content), XContentType.JSON) .build(); + AtomicBoolean isExecuted = new AtomicBoolean(false); - verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof PutDatasourceRequest); PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); - assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateIntervalInDays()); - assertEquals("test", putDatasourceRequest.getDatasourceName()); + assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval()); + assertEquals(datasourceName, putDatasourceRequest.getDatasourceName()); + isExecuted.set(true); return null; }); - dispatchRequest(restRequest); + dispatchRequest(request); + assertTrue(isExecuted.get()); } public void testPrepareRequestDefaultValue() { - RestRequest restRequestWithEmptyContent = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath("/_geoip/datasource/test") + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath(String.format(path, datasourceName)) .withContent(new BytesArray("{}"), XContentType.JSON) .build(); - - RestRequest restRequestWithoutContent = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath("/_geoip/datasource/test") - .build(); - - verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + AtomicBoolean isExecuted = new AtomicBoolean(false); + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof PutDatasourceRequest); PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; assertEquals("https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", putDatasourceRequest.getEndpoint()); - assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateIntervalInDays()); - assertEquals("test", putDatasourceRequest.getDatasourceName()); + assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval()); + assertEquals(datasourceName, putDatasourceRequest.getDatasourceName()); + isExecuted.set(true); return null; }); - dispatchRequest(restRequestWithEmptyContent); - dispatchRequest(restRequestWithoutContent); + dispatchRequest(request); + assertTrue(isExecuted.get()); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java new file mode 100644 index 00000000..066ea27b --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -0,0 +1,130 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; + +import org.junit.Before; +import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; + +public class DatasourceFacadeTests extends Ip2GeoTestCase { + private DatasourceFacade datasourceFacade; + + @Before + public void init() { + datasourceFacade = new DatasourceFacade( + verifyingClient, + new ClusterSettings(Settings.EMPTY, new HashSet<>(Ip2GeoSettings.settings())) + ); + } + + public void testUpdateDatasource() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource( + datasourceName, + new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS), + "https://test.com" + ); + Instant previousTime = Instant.now().minusMillis(1); + datasource.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest request = (IndexRequest) actionRequest; + assertEquals(datasource.getId(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + return null; + }); + + datasourceFacade.updateDatasource(datasource); + assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); + } + + public void testGetDatasourceException() throws Exception { + Datasource datasource = setupClientForGetRequest(true, new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME)); + assertNull(datasourceFacade.getDatasource(datasource.getId())); + } + + public void testGetDatasourceExist() throws Exception { + Datasource datasource = setupClientForGetRequest(true, null); + assertEquals(datasource, datasourceFacade.getDatasource(datasource.getId())); + } + + public void testGetDatasourceNotExist() throws Exception { + Datasource datasource = setupClientForGetRequest(false, null); + assertNull(datasourceFacade.getDatasource(datasource.getId())); + } + + public void testGetDatasourceExistWithListener() { + Datasource datasource = setupClientForGetRequest(true, null); + ActionListener listener = mock(ActionListener.class); + datasourceFacade.getDatasource(datasource.getId(), listener); + verify(listener).onResponse(eq(datasource)); + } + + public void testGetDatasourceNotExistWithListener() { + Datasource datasource = setupClientForGetRequest(false, null); + ActionListener listener = mock(ActionListener.class); + datasourceFacade.getDatasource(datasource.getId(), listener); + verify(listener).onResponse(null); + } + + private Datasource setupClientForGetRequest(final boolean isExist, final RuntimeException exception) { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource( + datasourceName, + new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS), + "https://test.com" + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof GetRequest); + GetRequest request = (GetRequest) actionRequest; + assertEquals(datasource.getId(), request.id()); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + GetResponse response = mock(GetResponse.class); + when(response.isExists()).thenReturn(isExist); + try { + when(response.getSourceAsBytesRef()).thenReturn( + BytesReference.bytes(datasource.toXContent(JsonXContent.contentBuilder(), null)) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (exception != null) { + throw exception; + } + return response; + }); + return datasource; + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java deleted file mode 100644 index 02ff8d58..00000000 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceHelperTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.geospatial.ip2geo.common; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.test.rest.RestActionTestCase; - -public class DatasourceHelperTests extends RestActionTestCase { - - public void testUpdateDatasource() throws Exception { - Instant previousTime = Instant.now().minusMillis(1); - Datasource datasource = new Datasource( - "testId", - previousTime, - null, - false, - null, - null, - DatasourceState.PREPARING, - null, - null, - null - ); - - verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { - assertTrue(actionRequest instanceof IndexRequest); - IndexRequest request = (IndexRequest) actionRequest; - assertEquals(datasource.getId(), request.id()); - assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); - assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); - return null; - }); - - DatasourceHelper.updateDatasource(verifyingClient, datasource, TimeValue.timeValueSeconds(30)); - assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); - } - - public void testGetDatasourceException() throws Exception { - Datasource datasource = new Datasource( - "testId", - Instant.now(), - null, - false, - new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), - "https://test.com", - DatasourceState.PREPARING, - null, - null, - null - ); - - verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { - assertTrue(actionRequest instanceof GetRequest); - GetRequest request = (GetRequest) actionRequest; - assertEquals(datasource.getId(), request.id()); - assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); - throw new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME); - }); - - assertNull(DatasourceHelper.getDatasource(verifyingClient, datasource.getId(), TimeValue.timeValueSeconds(30))); - } -} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java new file mode 100644 index 00000000..4a61e612 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -0,0 +1,324 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.lucene.search.TotalHits; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.search.MultiSearchRequest; +import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.Randomness; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; + +@SuppressForbidden(reason = "unit test") +public class GeoIpDataFacadeTests extends Ip2GeoTestCase { + private static final String IP_RANGE_FIELD_NAME = "_cidr"; + private static final String DATA_FIELD_NAME = "_data"; + private GeoIpDataFacade noOpsGeoIpDataFacade; + private GeoIpDataFacade verifyingGeoIpDataFacade; + + @Before + public void init() { + noOpsGeoIpDataFacade = new GeoIpDataFacade(clusterService, client); + verifyingGeoIpDataFacade = new GeoIpDataFacade(clusterService, verifyingClient); + } + + public void testCreateIndexIfNotExistsWithExistingIndex() { + String index = GeospatialTestHelper.randomLowerCaseString(); + when(metadata.hasIndex(index)).thenReturn(true); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); + verifyingGeoIpDataFacade.createIndexIfNotExists(index); + } + + public void testCreateIndexIfNotExistsWithoutExistingIndex() { + String index = GeospatialTestHelper.randomLowerCaseString(); + when(metadata.hasIndex(index)).thenReturn(false); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof CreateIndexRequest); + CreateIndexRequest request = (CreateIndexRequest) actionRequest; + assertEquals(index, request.index()); + assertEquals("1", request.settings().get("index.number_of_shards")); + assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals( + "{\"dynamic\": false,\"properties\": {\"_cidr\": {\"type\": \"ip_range\",\"doc_values\": false}}}", + request.mappings() + ); + return null; + }); + verifyingGeoIpDataFacade.createIndexIfNotExists(index); + } + + public void testCreateDocument() { + String[] names = { "ip", "country", "city" }; + String[] values = { "1.0.0.0/25", "USA", "Seattle" }; + assertEquals( + "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", + noOpsGeoIpDataFacade.createDocument(names, values) + ); + } + + public void testGetDatabaseReader() throws Exception { + File zipFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.zip").getFile()); + DatasourceManifest manifest = new DatasourceManifest( + zipFile.toURI().toURL().toExternalForm(), + "sample_valid.csv", + "fake_md5", + 1l, + Instant.now().toEpochMilli(), + "tester" + ); + CSVParser parser = noOpsGeoIpDataFacade.getDatabaseReader(manifest); + String[] expectedHeader = { "network", "country_name" }; + assertArrayEquals(expectedHeader, parser.iterator().next().values()); + String[] expectedValues = { "1.0.0.0/24", "Australia" }; + assertArrayEquals(expectedValues, parser.iterator().next().values()); + } + + public void testGetDatabaseReaderNoFile() throws Exception { + File zipFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.zip").getFile()); + DatasourceManifest manifest = new DatasourceManifest( + zipFile.toURI().toURL().toExternalForm(), + "no_file.csv", + "fake_md5", + 1l, + Instant.now().toEpochMilli(), + "tester" + ); + OpenSearchException exception = expectThrows(OpenSearchException.class, () -> noOpsGeoIpDataFacade.getDatabaseReader(manifest)); + assertTrue(exception.getMessage().contains("does not exist")); + } + + public void testDeleteIp2GeoDataIndex() { + String index = String.format(Locale.ROOT, "%s.%s", IP2GEO_DATA_INDEX_NAME_PREFIX, GeospatialTestHelper.randomLowerCaseString()); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof DeleteIndexRequest); + DeleteIndexRequest request = (DeleteIndexRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(index, request.indices()[0]); + assertEquals(IndicesOptions.LENIENT_EXPAND_OPEN, request.indicesOptions()); + return null; + }); + verifyingGeoIpDataFacade.deleteIp2GeoDataIndex(index); + } + + public void testDeleteIp2GeoDataIndexWithNonIp2GeoDataIndex() { + String index = GeospatialTestHelper.randomLowerCaseString(); + Exception e = expectThrows(OpenSearchException.class, () -> verifyingGeoIpDataFacade.deleteIp2GeoDataIndex(index)); + assertTrue(e.getMessage().contains("not ip2geo data index")); + verify(verifyingClient, never()).index(any()); + } + + public void testPutGeoIpData() throws Exception { + String index = GeospatialTestHelper.randomLowerCaseString(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + if (actionRequest instanceof BulkRequest) { + BulkRequest request = (BulkRequest) actionRequest; + assertEquals(1, request.numberOfActions()); + assertEquals(WriteRequest.RefreshPolicy.WAIT_UNTIL, request.getRefreshPolicy()); + BulkResponse response = mock(BulkResponse.class); + when(response.hasFailures()).thenReturn(false); + return response; + } else if (actionRequest instanceof RefreshRequest) { + RefreshRequest request = (RefreshRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(index, request.indices()[0]); + return null; + } else if (actionRequest instanceof ForceMergeRequest) { + ForceMergeRequest request = (ForceMergeRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(index, request.indices()[0]); + assertEquals(1, request.maxNumSegments()); + return null; + } else { + throw new RuntimeException("invalid request is called"); + } + }); + try (CSVParser csvParser = CSVParser.parse(sampleIp2GeoFile(), StandardCharsets.UTF_8, CSVFormat.RFC4180)) { + Iterator iterator = csvParser.iterator(); + String[] fields = iterator.next().values(); + verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, 1); + } + } + + public void testGetSingleGeoIpData() { + String indexName = GeospatialTestHelper.randomLowerCaseString(); + String ip = randomIpAddress(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assert actionRequest instanceof SearchRequest; + SearchRequest request = (SearchRequest) actionRequest; + assertEquals("_local", request.preference()); + assertEquals(1, request.source().size()); + assertEquals(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip), request.source().query()); + + String data = String.format( + Locale.ROOT, + "{\"%s\":\"1.0.0.1/16\",\"%s\":{\"city\":\"seattle\"}}", + IP_RANGE_FIELD_NAME, + DATA_FIELD_NAME + ); + SearchHit searchHit = new SearchHit(1); + searchHit.sourceRef(BytesReference.fromByteBuffer(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)))); + SearchHit[] searchHitArray = { searchHit }; + SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); + return response; + }); + ActionListener> listener = mock(ActionListener.class); + verifyingGeoIpDataFacade.getGeoIpData(indexName, ip, listener); + ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(listener).onResponse(captor.capture()); + assertEquals("seattle", captor.getValue().get("city")); + } + + public void testGetMultipleGeoIpDataNoSearchRequired() { + String indexName = GeospatialTestHelper.randomLowerCaseString(); + String ip1 = randomIpAddress(); + String ip2 = randomIpAddress(); + Iterator ipIterator = Arrays.asList(ip1, ip2).iterator(); + int maxBundleSize = 1; + int maxConcurrentSearches = 1; + boolean firstOnly = true; + Map> geoData = new HashMap<>(); + geoData.put(ip1, Map.of("city", "Seattle")); + geoData.put(ip2, Map.of("city", "Hawaii")); + ActionListener>> actionListener = mock(ActionListener.class); + + // Run + verifyingGeoIpDataFacade.getGeoIpData( + indexName, + ipIterator, + maxBundleSize, + maxConcurrentSearches, + firstOnly, + geoData, + actionListener + ); + + // Verify + verify(actionListener).onResponse(geoData); + } + + public void testGetMultipleGeoIpData() { + String indexName = GeospatialTestHelper.randomLowerCaseString(); + int dataSize = Randomness.get().nextInt(10) + 1; + List ips = new ArrayList<>(); + for (int i = 0; i < dataSize; i++) { + ips.add(randomIpAddress()); + } + int maxBundleSize = Randomness.get().nextInt(11) + 1; + int maxConcurrentSearches = 1; + boolean firstOnly = false; + Map> geoData = new HashMap<>(); + ActionListener>> actionListener = mock(ActionListener.class); + + List cities = new ArrayList<>(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assert actionRequest instanceof MultiSearchRequest; + MultiSearchRequest request = (MultiSearchRequest) actionRequest; + assertEquals(maxConcurrentSearches, request.maxConcurrentSearchRequests()); + assertTrue(request.requests().size() == maxBundleSize || request.requests().size() == dataSize % maxBundleSize); + for (SearchRequest searchRequest : request.requests()) { + assertEquals("_local", searchRequest.preference()); + assertEquals(1, searchRequest.source().size()); + } + + MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.requests().size()]; + for (int i = 0; i < request.requests().size(); i++) { + String city = GeospatialTestHelper.randomLowerCaseString(); + cities.add(city); + String data = String.format( + Locale.ROOT, + "{\"%s\":\"1.0.0.1/16\",\"%s\":{\"city\":\"%s\"}}", + IP_RANGE_FIELD_NAME, + DATA_FIELD_NAME, + city + ); + SearchHit searchHit = new SearchHit(1); + searchHit.sourceRef(BytesReference.fromByteBuffer(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)))); + SearchHit[] searchHitArray = { searchHit }; + SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn(searchHits); + MultiSearchResponse.Item item = mock(MultiSearchResponse.Item.class); + when(item.isFailure()).thenReturn(false); + when(item.getResponse()).thenReturn(searchResponse); + items[i] = item; + } + MultiSearchResponse response = mock(MultiSearchResponse.class); + when(response.getResponses()).thenReturn(items); + return response; + }); + + // Run + verifyingGeoIpDataFacade.getGeoIpData( + indexName, + ips.iterator(), + maxBundleSize, + maxConcurrentSearches, + firstOnly, + geoData, + actionListener + ); + + // Verify + verify(verifyingClient, times((dataSize + maxBundleSize - 1) / maxBundleSize)).execute( + any(ActionType.class), + any(ActionRequest.class), + any(ActionListener.class) + ); + for (int i = 0; i < dataSize; i++) { + assertEquals(cities.get(i), geoData.get(ips.get(i)).get("city")); + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java deleted file mode 100644 index 6b65026d..00000000 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataHelperTests.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.geospatial.ip2geo.common; - -import org.opensearch.test.OpenSearchTestCase; - -public class GeoIpDataHelperTests extends OpenSearchTestCase { - public void testCreateDocument() { - String[] names = { "ip", "country", "city" }; - String[] values = { "1.0.0.0/25", "USA", "Seattle" }; - assertEquals( - "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", - GeoIpDataHelper.createDocument(names, values) - ); - } -} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java new file mode 100644 index 00000000..3632d9e9 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import static org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension.JOB_INDEX_NAME; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import org.opensearch.common.Randomness; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; + +public class DatasourceExtensionTests extends Ip2GeoTestCase { + public void testBasic() throws Exception { + DatasourceExtension extension = new DatasourceExtension(); + assertEquals("scheduler_geospatial_ip2geo_datasource", extension.getJobType()); + assertEquals(JOB_INDEX_NAME, extension.getJobIndex()); + assertEquals(DatasourceRunner.getJobRunnerInstance(), extension.getJobRunner()); + } + + public void testParser() throws Exception { + DatasourceExtension extension = new DatasourceExtension(); + String id = GeospatialTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + String endpoint = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource(id, schedule, endpoint); + + Datasource anotherDatasource = (Datasource) extension.getJobParser() + .parse( + createParser(datasource.toXContent(XContentFactory.jsonBuilder(), null)), + GeospatialTestHelper.randomLowerCaseString(), + new JobDocVersion(Randomness.get().nextLong(), Randomness.get().nextLong(), Randomness.get().nextLong()) + ); + + assertTrue(datasource.equals(anotherDatasource)); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java new file mode 100644 index 00000000..0e33e232 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.internal.verification.VerificationModeFactory.times; +import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; + +import java.time.Instant; + +import org.junit.Before; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; + +public class DatasourceRunnerTests extends Ip2GeoTestCase { + @Before + public void init() { + DatasourceRunner.getJobRunnerInstance() + .initialize(clusterService, client, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + } + + public void testRunJobInvalidClass() { + JobExecutionContext jobExecutionContext = mock(JobExecutionContext.class); + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + expectThrows(IllegalStateException.class, () -> DatasourceRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); + } + + public void testRunJob() { + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = randomLowerCaseString(); + String jobId = randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); + Datasource datasource = new Datasource(); + + // Run + DatasourceRunner.getJobRunnerInstance().runJob(datasource, jobExecutionContext); + + // Verify + verify(executorService).submit(any(Runnable.class)); + } + + public void testUpdateDatasourceNull() throws Exception { + Datasource datasource = new Datasource(); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + + // Verify + verify(datasourceUpdateService, never()).deleteUnusedIndices(any()); + } + + public void testUpdateDatasourceInvalidState() throws Exception { + Datasource datasource = new Datasource(); + datasource.enable(); + datasource.getUpdateStats().setLastFailedAt(null); + datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); + when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + + // Verify + assertFalse(datasource.isEnabled()); + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(datasourceFacade).updateDatasource(datasource); + } + + public void testUpdateDatasource() throws Exception { + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.setId(randomLowerCaseString()); + when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + + // Verify + verify(datasourceUpdateService, times(2)).deleteUnusedIndices(datasource); + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + } + + public void testUpdateDatasourceExceptionHandling() throws Exception { + Datasource datasource = new Datasource(); + datasource.setId(randomLowerCaseString()); + datasource.getUpdateStats().setLastFailedAt(null); + when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + doThrow(new RuntimeException("test failure")).when(datasourceUpdateService).deleteUnusedIndices(any()); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + + // Verify + assertNotNull(datasource.getUpdateStats().getLastFailedAt()); + verify(datasourceFacade).updateDatasource(datasource); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 0faaa3e2..b4f76ef6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -15,15 +15,41 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.Locale; import org.opensearch.common.Randomness; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.test.OpenSearchTestCase; public class DatasourceTests extends OpenSearchTestCase { + + public void testParser() throws Exception { + String id = GeospatialTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + String endpoint = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource(id, schedule, endpoint); + datasource.enable(); + datasource.getDatabase().setFields(Arrays.asList("field1", "field2")); + datasource.getDatabase().setProvider("test_provider"); + datasource.getDatabase().setUpdatedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setValidForInDays(1l); + datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); + datasource.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + datasource.getUpdateStats().setLastSkippedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + datasource.getUpdateStats().setLastFailedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + + Datasource anotherDatasource = Datasource.PARSER.parse( + createParser(datasource.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + assertTrue(datasource.equals(anotherDatasource)); + } + public void testCurrentIndexName() { String id = GeospatialTestHelper.randomLowerCaseString(); Instant now = Instant.now(); @@ -56,9 +82,33 @@ public void testGetIndexNameFor() { public void testGetJitter() { Datasource datasource = new Datasource(); - datasource.setSchedule(new IntervalSchedule(Instant.now(), Randomness.get().nextInt(31), ChronoUnit.DAYS)); - long intervalInMinutes = datasource.getSchedule().getInterval() * 60 * 24; + datasource.setSchedule(new IntervalSchedule(Instant.now(), Randomness.get().ints(1, 31).findFirst().getAsInt(), ChronoUnit.DAYS)); + long intervalInMinutes = datasource.getSchedule().getInterval() * 60l * 24l; double sixMinutes = 6; assertTrue(datasource.getJitter() * intervalInMinutes <= sixMinutes); } + + public void testIsExpired() { + Datasource datasource = new Datasource(); + // never expire if validForInDays is null + assertFalse(datasource.isExpired()); + + datasource.getDatabase().setValidForInDays(1l); + + // if last skipped date is null, use only last succeeded date to determine + datasource.getUpdateStats().setLastSucceededAt(Instant.now().minus(25, ChronoUnit.HOURS)); + assertTrue(datasource.isExpired()); + + // use the latest date between last skipped date and last succeeded date to determine + datasource.getUpdateStats().setLastSkippedAt(Instant.now()); + assertFalse(datasource.isExpired()); + datasource.getUpdateStats().setLastSkippedAt(Instant.now().minus(25, ChronoUnit.HOURS)); + datasource.getUpdateStats().setLastSucceededAt(Instant.now()); + assertFalse(datasource.isExpired()); + } + + public void testLockDurationSeconds() { + Datasource datasource = new Datasource(); + assertNotNull(datasource.getLockDurationSeconds()); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java new file mode 100644 index 00000000..3c9ec15d --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -0,0 +1,152 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Arrays; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.junit.Before; +import org.opensearch.OpenSearchException; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; + +@SuppressForbidden(reason = "unit test") +public class DatasourceUpdateServiceTests extends Ip2GeoTestCase { + private DatasourceUpdateService datasourceUpdateService; + + @Before + public void init() { + datasourceUpdateService = new DatasourceUpdateService(clusterService, client, datasourceFacade, geoIpDataFacade); + } + + public void testUpdateDatasourceSkip() throws Exception { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); + + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getUpdateStats().setLastSkippedAt(null); + datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); + datasource.getDatabase().setMd5Hash(manifest.getMd5Hash()); + datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); + + // Run + datasourceUpdateService.updateOrCreateGeoIpData(datasource); + + // Verify + assertNotNull(datasource.getUpdateStats().getLastSkippedAt()); + verify(datasourceFacade).updateDatasource(datasource); + } + + public void testUpdateDatasourceInvalidFile() throws Exception { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); + + File sampleFile = new File( + this.getClass().getClassLoader().getResource("ip2geo/sample_invalid_less_than_two_fields.csv").getFile() + ); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); + datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setFields(Arrays.asList("country_name")); + datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); + + // Run + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource)); + } + + public void testUpdateDatasourceIncompatibleFields() throws Exception { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); + datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setFields(Arrays.asList("country_name", "additional_field")); + datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); + + // Run + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource)); + } + + public void testUpdateDatasource() throws Exception { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); + datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setFields(Arrays.asList("country_name")); + datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); + datasource.getUpdateStats().setLastSucceededAt(null); + datasource.getUpdateStats().setLastProcessingTimeInMillis(null); + + // Run + datasourceUpdateService.updateOrCreateGeoIpData(datasource); + + // Verify + assertEquals(manifest.getProvider(), datasource.getDatabase().getProvider()); + assertEquals(manifest.getMd5Hash(), datasource.getDatabase().getMd5Hash()); + assertEquals(Instant.ofEpochMilli(manifest.getUpdatedAt()), datasource.getDatabase().getUpdatedAt()); + assertEquals(manifest.getValidForInDays(), datasource.getDatabase().getValidForInDays()); + assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); + assertNotNull(datasource.getUpdateStats().getLastProcessingTimeInMillis()); + verify(datasourceFacade, times(2)).updateDatasource(datasource); + } + + public void testDeleteUnusedIndices() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String indexPrefix = String.format(".ip2geo-data.%s.", datasourceName); + Instant now = Instant.now(); + String currentIndex = indexPrefix + now.toEpochMilli(); + String oldIndex = indexPrefix + now.minusMillis(1).toEpochMilli(); + String lingeringIndex = indexPrefix + now.minusMillis(2).toEpochMilli(); + Datasource datasource = new Datasource(); + datasource.setId(datasourceName); + datasource.getIndices().add(currentIndex); + datasource.getIndices().add(oldIndex); + datasource.getIndices().add(lingeringIndex); + datasource.getDatabase().setUpdatedAt(now); + + when(metadata.hasIndex(currentIndex)).thenReturn(true); + when(metadata.hasIndex(oldIndex)).thenReturn(true); + when(metadata.hasIndex(lingeringIndex)).thenReturn(false); + when(geoIpDataFacade.deleteIp2GeoDataIndex(any())).thenReturn(new AcknowledgedResponse(true)); + + datasourceUpdateService.deleteUnusedIndices(datasource); + + assertEquals(1, datasource.getIndices().size()); + assertEquals(currentIndex, datasource.getIndices().get(0)); + verify(datasourceFacade).updateDatasource(datasource); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java deleted file mode 100644 index ad340f47..00000000 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoCacheTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.geospatial.ip2geo.processor; - -import java.util.HashMap; -import java.util.Map; - -import org.opensearch.OpenSearchException; -import org.opensearch.test.OpenSearchTestCase; - -public class Ip2GeoCacheTests extends OpenSearchTestCase { - public void testCachesAndEvictsResults() { - Ip2GeoCache cache = new Ip2GeoCache(1); - String datasource = "datasource"; - Map response1 = new HashMap<>(); - Map response2 = new HashMap<>(); - assertNotSame(response1, response2); - - // add a key - Map cachedResponse = cache.putIfAbsent("127.0.0.1", datasource, key -> response1); - assertSame(cachedResponse, response1); - assertSame(cachedResponse, cache.putIfAbsent("127.0.0.1", datasource, key -> response2)); - assertSame(cachedResponse, cache.get("127.0.0.1", datasource)); - - // evict old key by adding another value - cachedResponse = cache.putIfAbsent("127.0.0.2", datasource, key -> response2); - assertSame(cachedResponse, response2); - assertSame(cachedResponse, cache.putIfAbsent("127.0.0.2", datasource, ip -> response2)); - assertSame(cachedResponse, cache.get("127.0.0.2", datasource)); - - assertNotSame(response1, cache.get("127.0.0.1", datasource)); - } - - public void testThrowsFunctionsException() { - Ip2GeoCache cache = new Ip2GeoCache(1); - expectThrows( - OpenSearchException.class, - () -> cache.putIfAbsent("127.0.0.1", "datasource", ip -> { throw new OpenSearchException("bad"); }) - ); - } - - public void testNoExceptionForNullValue() { - Ip2GeoCache cache = new Ip2GeoCache(1); - Map response = cache.putIfAbsent("127.0.0.1", "datasource", ip -> null); - assertTrue(response.isEmpty()); - } - - public void testInvalidInit() { - IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Ip2GeoCache(-1)); - assertEquals("ip2geo max cache size must be 0 or greater", ex.getMessage()); - } -} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java new file mode 100644 index 00000000..bd6857ed --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -0,0 +1,346 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.processor; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionListener; +import org.opensearch.common.Randomness; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.ingest.IngestDocument; + +public class Ip2GeoProcessorTests extends Ip2GeoTestCase { + private static final String DEFAULT_TARGET_FIELD = "ip2geo"; + private static final String CONFIG_DATASOURCE_KEY = "datasource"; + private static final String CONFIG_FIELD_KEY = "field"; + private static final List SUPPORTED_FIELDS = Arrays.asList("city", "country"); + private Ip2GeoProcessor.Factory factory; + + @Before + public void init() { + factory = new Ip2GeoProcessor.Factory(client, ingestService, datasourceFacade, geoIpDataFacade); + } + + public void testCreateWithNoDatasource() { + Map config = new HashMap<>(); + config.put("field", "ip"); + config.put(CONFIG_DATASOURCE_KEY, "no_datasource"); + OpenSearchException exception = expectThrows( + OpenSearchException.class, + () -> factory.create( + Collections.emptyMap(), + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString(), + config + ) + ); + assertTrue(exception.getDetailedMessage().contains("doesn't exist")); + } + + public void testCreateWithInvalidDatasourceState() { + Datasource datasource = new Datasource(); + datasource.setId(GeospatialTestHelper.randomLowerCaseString()); + datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); + OpenSearchException exception = expectThrows(OpenSearchException.class, () -> createProcessor(datasource, Collections.emptyMap())); + assertTrue(exception.getDetailedMessage().contains("available state")); + } + + public void testCreateWithInvalidProperties() { + Map config = new HashMap<>(); + config.put("properties", Arrays.asList("ip", "invalid_property")); + OpenSearchException exception = expectThrows( + OpenSearchException.class, + () -> createProcessor(GeospatialTestHelper.randomLowerCaseString(), config) + ); + assertTrue(exception.getDetailedMessage().contains("property")); + } + + public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Map config = new HashMap<>(); + config.put("ignore_missing", true); + Ip2GeoProcessor processor = createProcessor(datasourceName, config); + IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); + BiConsumer handler = (doc, e) -> { + assertEquals(document, doc); + assertNull(e); + }; + processor.execute(document, handler); + } + + public void testExecuteWithNoIp() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Map config = new HashMap<>(); + Ip2GeoProcessor processor = createProcessor(datasourceName, config); + IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); + BiConsumer handler = (doc, e) -> {}; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(document, handler)); + assertTrue(exception.getMessage().contains("not present")); + } + + public void testExecuteWithNonStringValue() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + Map source = new HashMap<>(); + source.put("ip", Randomness.get().nextInt()); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + BiConsumer handler = (doc, e) -> {}; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(document, handler)); + assertTrue(exception.getMessage().contains("string")); + } + + public void testExecuteWithNullDatasource() throws Exception { + BiConsumer handler = (doc, e) -> { + assertNull(doc); + assertTrue(e.getMessage().contains("datasource does not exist")); + }; + getActionListener(Collections.emptyMap(), handler).onResponse(null); + } + + public void testExecuteWithExpiredDatasource() throws Exception { + Datasource datasource = mock(Datasource.class); + when(datasource.isExpired()).thenReturn(true); + BiConsumer handler = (doc, e) -> { + assertEquals("ip2geo_data_expired", doc.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); + assertNull(e); + }; + getActionListener(Collections.emptyMap(), handler).onResponse(datasource); + } + + public void testExecute() throws Exception { + Map ip2geoData = new HashMap<>(); + for (String field : SUPPORTED_FIELDS) { + ip2geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); + } + + Datasource datasource = mock(Datasource.class); + when(datasource.isExpired()).thenReturn(false); + when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + BiConsumer handler = (doc, e) -> { + assertEquals( + ip2geoData.get(SUPPORTED_FIELDS.get(0)), + doc.getFieldValue(DEFAULT_TARGET_FIELD + "." + SUPPORTED_FIELDS.get(0), String.class) + ); + for (int i = 1; i < SUPPORTED_FIELDS.size(); i++) { + assertNull(doc.getFieldValue(DEFAULT_TARGET_FIELD + "." + SUPPORTED_FIELDS.get(i), String.class, true)); + } + assertNull(e); + }; + Map config = Map.of("properties", Arrays.asList(SUPPORTED_FIELDS.get(0))); + getActionListener(config, handler).onResponse(datasource); + + ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(geoIpDataFacade).getGeoIpData(anyString(), anyString(), captor.capture()); + captor.getValue().onResponse(ip2geoData); + } + + private ActionListener getActionListener( + final Map config, + final BiConsumer handler + ) throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, config); + + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + processor.execute(document, handler); + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); + return captor.getValue(); + } + + public void testExecuteNotImplemented() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + IngestDocument document = new IngestDocument(Collections.emptyMap(), Collections.emptyMap()); + Exception e = expectThrows(IllegalStateException.class, () -> processor.execute(document)); + assertTrue(e.getMessage().contains("Not implemented")); + } + + public void testGenerateDataToAppendWithFirstOnlyOption() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor( + datasourceName, + Map.of("first_only", true, "properties", Arrays.asList(SUPPORTED_FIELDS.get(0))) + ); + List ips = new ArrayList<>(); + Map> data = new HashMap<>(); + for (int i = 0; i < 3; i++) { + String ip = randomIpAddress(); + ips.add(ip); + Map geoData = new HashMap<>(); + for (String field : SUPPORTED_FIELDS) { + geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); + } + data.put(ip, i == 0 ? Collections.emptyMap() : geoData); + } + IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); + + // Verify + verify(handler).accept(document, null); + assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).size()); + assertEquals( + data.get(ips.get(1)).get(SUPPORTED_FIELDS.get(0)), + document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get(SUPPORTED_FIELDS.get(0)) + ); + assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get(SUPPORTED_FIELDS.get(1))); + } + + public void testGenerateDataToAppendWithOutFirstOnlyOption() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor( + datasourceName, + Map.of("first_only", false, "properties", Arrays.asList(SUPPORTED_FIELDS.get(0))) + ); + List ips = new ArrayList<>(); + Map> data = new HashMap<>(); + for (int i = 0; i < 3; i++) { + String ip = randomIpAddress(); + ips.add(ip); + Map geoData = new HashMap<>(); + for (String field : SUPPORTED_FIELDS) { + geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); + } + data.put(ip, i == 0 ? Collections.emptyMap() : geoData); + } + IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); + + // Verify + verify(handler).accept(document, null); + assertEquals(ips.size(), document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); + for (int i = 0; i < ips.size(); i++) { + if (data.get(ips.get(i)).isEmpty()) { + assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).get(i)); + } else { + Map documentData = (Map) document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).get(i); + assertEquals(1, documentData.size()); + assertEquals(data.get(ips.get(i)).get(SUPPORTED_FIELDS.get(0)), documentData.get(SUPPORTED_FIELDS.get(0))); + assertNull(documentData.get(SUPPORTED_FIELDS.get(1))); + } + } + } + + public void testGenerateDataToAppendWithNoData() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("first_only", Randomness.get().nextInt() % 2 == 0)); + List ips = new ArrayList<>(); + Map> data = new HashMap<>(); + for (int i = 0; i < 3; i++) { + String ip = randomIpAddress(); + ips.add(ip); + data.put(ip, Collections.emptyMap()); + } + IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); + BiConsumer handler = mock(BiConsumer.class); + processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); + verify(handler).accept(document, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class)); + assertTrue(e.getMessage().contains("not present")); + } + + public void testExecuteInternalNonStringIp() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + List ips = Arrays.asList(randomIpAddress(), 1); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + BiConsumer handler = mock(BiConsumer.class); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.executeInternal(document, handler, ips)); + assertTrue(e.getMessage().contains("should only contain strings")); + } + + public void testExecuteInternal() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + BiConsumer handler = mock(BiConsumer.class); + processor.executeInternal(document, handler, ips); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); + Datasource datasource = mock(Datasource.class); + when(datasource.isExpired()).thenReturn(false); + when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + captor.getValue().onResponse(datasource); + verify(geoIpDataFacade).getGeoIpData( + anyString(), + any(Iterator.class), + anyInt(), + anyInt(), + anyBoolean(), + anyMap(), + any(ActionListener.class) + ); + } + + private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { + Datasource datasource = new Datasource(); + datasource.setId(datasourceName); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getDatabase().setFields(SUPPORTED_FIELDS); + return createProcessor(datasource, config); + } + + private Ip2GeoProcessor createProcessor(final Datasource datasource, final Map config) throws Exception { + when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + Map baseConfig = new HashMap<>(); + baseConfig.put(CONFIG_FIELD_KEY, "ip"); + baseConfig.put(CONFIG_DATASOURCE_KEY, datasource.getName()); + baseConfig.putAll(config); + + return factory.create( + Collections.emptyMap(), + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString(), + baseConfig + ); + } +} diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index d7ce5594..6d9430bc 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -7,50 +7,148 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import org.junit.After; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.geospatial.stats.upload.RestUploadStatsAction; +import org.opensearch.geospatial.stats.upload.UploadStats; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.ingest.IngestService; import org.opensearch.ingest.Processor; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IngestPlugin; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; public class GeospatialPluginTests extends OpenSearchTestCase { private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings())); private final List SUPPORTED_REST_HANDLERS = List.of( new RestUploadGeoJSONAction(), new RestUploadStatsAction(), - new RestPutDatasourceHandler(Settings.EMPTY, clusterSettings) + new RestPutDatasourceHandler(clusterSettings) ); - private final Client client; - private final ClusterService clusterService; - private final IngestService ingestService; - - public GeospatialPluginTests() { - client = mock(Client.class); - when(client.settings()).thenReturn(Settings.EMPTY); - clusterService = mock(ClusterService.class); + + private final Set SUPPORTED_SYSTEM_INDEX_PATTERN = Set.of(IP2GEO_DATA_INDEX_NAME_PREFIX); + + private final Set SUPPORTED_COMPONENTS = Set.of( + UploadStats.class, + DatasourceUpdateService.class, + DatasourceFacade.class, + Ip2GeoExecutor.class, + GeoIpDataFacade.class + ); + + @Mock + private Client client; + @Mock + private ClusterService clusterService; + @Mock + private IngestService ingestService; + @Mock + private ThreadPool threadPool; + @Mock + private ResourceWatcherService resourceWatcherService; + @Mock + private ScriptService scriptService; + @Mock + private NamedXContentRegistry xContentRegistry; + @Mock + private Environment environment; + @Mock + private NamedWriteableRegistry namedWriteableRegistry; + @Mock + private IndexNameExpressionResolver indexNameExpressionResolver; + @Mock + private Supplier repositoriesServiceSupplier; + private NodeEnvironment nodeEnvironment; + private Settings settings; + private AutoCloseable openMocks; + + @Before + public void init() { + openMocks = MockitoAnnotations.openMocks(this); + settings = Settings.EMPTY; + when(client.settings()).thenReturn(settings); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - ingestService = mock(IngestService.class); + when(clusterService.getSettings()).thenReturn(settings); when(ingestService.getClusterService()).thenReturn(clusterService); + nodeEnvironment = null; + } + + @After + public void close() throws Exception { + openMocks.close(); + } + + public void testSystemIndexDescriptors() { + GeospatialPlugin plugin = new GeospatialPlugin(); + Set registeredSystemIndexPatterns = new HashSet<>(); + for (SystemIndexDescriptor descriptor : plugin.getSystemIndexDescriptors(Settings.EMPTY)) { + registeredSystemIndexPatterns.add(descriptor.getIndexPattern()); + } + assertEquals(SUPPORTED_SYSTEM_INDEX_PATTERN, registeredSystemIndexPatterns); + + } + + public void testExecutorBuilders() { + GeospatialPlugin plugin = new GeospatialPlugin(); + assertEquals(1, plugin.getExecutorBuilders(Settings.EMPTY).size()); + } + + public void testCreateComponents() { + GeospatialPlugin plugin = new GeospatialPlugin(); + Set registeredComponents = new HashSet<>(); + Collection components = plugin.createComponents( + client, + clusterService, + threadPool, + resourceWatcherService, + scriptService, + xContentRegistry, + environment, + nodeEnvironment, + namedWriteableRegistry, + indexNameExpressionResolver, + repositoriesServiceSupplier + ); + for (Object component : components) { + registeredComponents.add(component.getClass()); + } + assertEquals(SUPPORTED_COMPONENTS, registeredComponents); } public void testIsAnIngestPlugin() { diff --git a/src/test/resources/ip2geo/manifest.json b/src/test/resources/ip2geo/manifest.json new file mode 100644 index 00000000..652bc9d8 --- /dev/null +++ b/src/test/resources/ip2geo/manifest.json @@ -0,0 +1,8 @@ +{ + "url": "https://test.com/db.zip", + "db_name": "sample_valid.csv", + "md5_hash": "safasdfaskkkesadfasdf", + "valid_for_in_days": 30, + "updated_at": 3134012341236, + "provider": "sample_provider" +} \ No newline at end of file diff --git a/src/test/resources/ip2geo/manifest_invalid_url.json b/src/test/resources/ip2geo/manifest_invalid_url.json new file mode 100644 index 00000000..77d68aaf --- /dev/null +++ b/src/test/resources/ip2geo/manifest_invalid_url.json @@ -0,0 +1,8 @@ +{ + "url": "invalid://test.com/db.zip", + "db_name": "sample_valid.csv", + "md5_hash": "safasdfaskkkesadfasdf", + "valid_for_in_days": 30, + "updated_at": 3134012341236, + "provider": "sample_provider" +} \ No newline at end of file diff --git a/src/test/resources/ip2geo/manifest_template.json b/src/test/resources/ip2geo/manifest_template.json new file mode 100644 index 00000000..4c273fa4 --- /dev/null +++ b/src/test/resources/ip2geo/manifest_template.json @@ -0,0 +1,8 @@ +{ + "url": "URL", + "db_name": "sample_valid.csv", + "md5_hash": "safasdfaskkkesadfasdf", + "valid_for_in_days": 30, + "updated_at": 3134012341236, + "provider": "maxmind" +} \ No newline at end of file diff --git a/src/test/resources/ip2geo/sample_invalid_less_than_two_fields.csv b/src/test/resources/ip2geo/sample_invalid_less_than_two_fields.csv new file mode 100644 index 00000000..08670061 --- /dev/null +++ b/src/test/resources/ip2geo/sample_invalid_less_than_two_fields.csv @@ -0,0 +1,2 @@ +network +1.0.0.0/24 \ No newline at end of file diff --git a/src/test/resources/ip2geo/sample_valid.csv b/src/test/resources/ip2geo/sample_valid.csv new file mode 100644 index 00000000..a6d08935 --- /dev/null +++ b/src/test/resources/ip2geo/sample_valid.csv @@ -0,0 +1,3 @@ +network,country_name +1.0.0.0/24,Australia +10.0.0.0/24,USA \ No newline at end of file diff --git a/src/test/resources/ip2geo/sample_valid.zip b/src/test/resources/ip2geo/sample_valid.zip new file mode 100644 index 0000000000000000000000000000000000000000..0bdeeadbf1f9d2c9c7d542cde8694556b4e055fa GIT binary patch literal 250 zcmWIWW@Zs#-~d9W>KS1SP@oB<1sD_E%D&A`a=gOPy&XbJVQ4^ZKW*d7jfhtF5`J=e+kB-={`bj2Rqjm4#)G zUwSlY(UCc4To?kp**Vz%&wil+)C00Bz?+dtgc;!uWI2#KU|>ljh()Ta0=!w-K>8Sg LFb+t!fjA5RP5V9S literal 0 HcmV?d00001 From ee519a24734b9186fc070ef937ebd6382902e64a Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 3 May 2023 11:12:49 -0700 Subject: [PATCH 13/61] Sync from main (#280) * Update gradle version to 7.6 (#265) Signed-off-by: Vijayan Balasubramanian * Exclude lombok generated code from jacoco coverage report (#268) Signed-off-by: Heemin Kim * Make jacoco report to be generated faster in local (#267) Signed-off-by: Heemin Kim * Update dependency org.json:json to v20230227 (#273) Co-authored-by: mend-for-github.aaakk.us.kg[bot] <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> * Baseline owners and maintainers (#275) Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian Signed-off-by: Heemin Kim Co-authored-by: Vijayan Balasubramanian Co-authored-by: mend-for-github.aaakk.us.kg[bot] <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a19cc0e..109fc6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Make jacoco report to be generated faster in local ([#267](https://github.com/opensearch-project/geospatial/pull/267)) +* Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance ### Refactoring From cc41bf5f488d8c5c2df18ac48d5e250e9eb5f07e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 3 May 2023 14:25:41 -0700 Subject: [PATCH 14/61] Add datasource name validation (#281) Signed-off-by: Heemin Kim --- .../ip2geo/action/PutDatasourceRequest.java | 43 ++++++++++++++ .../action/PutDatasourceRequestTests.java | 57 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 82e513f0..83aa3acc 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -9,6 +9,7 @@ package org.opensearch.geospatial.ip2geo.action; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -19,8 +20,10 @@ import lombok.Setter; import lombok.extern.log4j.Log4j2; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; @@ -38,6 +41,7 @@ public class PutDatasourceRequest extends AcknowledgedRequest { private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + private static final int MAX_DATASOURCE_NAME_BYTES = 255; /** * @param datasourceName the datasource name * @return the datasource name @@ -95,11 +99,50 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { ActionRequestValidationException errors = new ActionRequestValidationException(); + validateDatasourceName(errors); validateEndpoint(errors); validateUpdateInterval(errors); return errors.validationErrors().isEmpty() ? null : errors; } + private void validateDatasourceName(final ActionRequestValidationException errors) { + if (!Strings.validFileName(datasourceName)) { + errors.addValidationError("Datasource name must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); + return; + } + if (datasourceName.isEmpty()) { + errors.addValidationError("Datasource name must not be empty"); + return; + } + if (datasourceName.contains("#")) { + errors.addValidationError("Datasource name must not contain '#'"); + return; + } + if (datasourceName.contains(":")) { + errors.addValidationError("Datasource name must not contain ':'"); + return; + } + if (datasourceName.charAt(0) == '_' || datasourceName.charAt(0) == '-' || datasourceName.charAt(0) == '+') { + errors.addValidationError("Datasource name must not start with '_', '-', or '+'"); + return; + } + int byteCount = 0; + try { + byteCount = datasourceName.getBytes("UTF-8").length; + } catch (UnsupportedEncodingException e) { + // UTF-8 should always be supported, but rethrow this if it is not for some reason + throw new OpenSearchException("Unable to determine length of datasource name", e); + } + if (byteCount > MAX_DATASOURCE_NAME_BYTES) { + errors.addValidationError("Datasource name is too long, (" + byteCount + " > " + MAX_DATASOURCE_NAME_BYTES + ")"); + return; + } + if (datasourceName.equals(".") || datasourceName.equals("..")) { + errors.addValidationError("Datasource name must not be '.' or '..'"); + return; + } + } + /** * Conduct following validation on endpoint * 1. endpoint format complies with RFC-2396 diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 383d832a..ca2cda81 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -8,10 +8,13 @@ package org.opensearch.geospatial.ip2geo.action; +import java.util.Arrays; import java.util.Locale; +import java.util.Map; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.Randomness; +import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; @@ -97,6 +100,60 @@ public void testValidateWithInvalidUrlInsideManifest() throws Exception { assertTrue(exception.validationErrors().get(0).contains("Invalid URL format")); } + public void testValidateDatasourceNames() throws Exception { + String validDatasourceName = GeospatialTestHelper.randomLowerCaseString(); + String domain = GeospatialTestHelper.randomLowerCaseString(); + PutDatasourceRequest request = new PutDatasourceRequest(validDatasourceName); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(30) + 1)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertNull(exception); + + String fileNameChar = validDatasourceName + Strings.INVALID_FILENAME_CHARS.stream() + .skip(Randomness.get().nextInt(Strings.INVALID_FILENAME_CHARS.size() - 1)) + .findFirst(); + String startsWith = Arrays.asList("_", "-", "+").get(Randomness.get().nextInt(3)) + validDatasourceName; + String empty = ""; + String hash = validDatasourceName + "#"; + String colon = validDatasourceName + ":"; + StringBuilder longName = new StringBuilder(); + while (longName.length() < 256) { + longName.append(GeospatialTestHelper.randomLowerCaseString()); + } + String point = Arrays.asList(".", "..").get(Randomness.get().nextInt(2)); + Map nameToError = Map.of( + fileNameChar, + "not contain the following characters", + empty, + "must not be empty", + hash, + "must not contain '#'", + colon, + "must not contain ':'", + startsWith, + "must not start with", + longName.toString(), + "name is too long", + point, + "must not be '.' or '..'" + ); + + for (Map.Entry entry : nameToError.entrySet()) { + request.setDatasourceName(entry.getKey()); + + // Run + exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains(entry.getValue())); + } + } + public void testStreamInOut() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String domain = GeospatialTestHelper.randomLowerCaseString(); From c31ded4c2673d9dc46a8add23307336866776a6f Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 3 May 2023 14:54:03 -0700 Subject: [PATCH 15/61] Refactoring of code (#282) 1. Change variable name from datasourceName to name 2. Change variable name from id to name 3. Added helper methods in test code Signed-off-by: Heemin Kim --- .../ip2geo/action/PutDatasourceRequest.java | 28 +++++------ .../action/PutDatasourceTransportAction.java | 8 +-- .../ip2geo/common/DatasourceFacade.java | 16 +++--- .../ip2geo/jobscheduler/Datasource.java | 34 ++++++------- .../ip2geo/jobscheduler/DatasourceRunner.java | 2 +- .../jobscheduler/DatasourceUpdateService.java | 10 ++-- .../geospatial/ip2geo/Ip2GeoTestCase.java | 39 +++++++++++++++ .../action/PutDatasourceRequestTests.java | 2 +- .../PutDatasourceTransportActionTests.java | 2 +- .../action/RestPutDatasourceHandlerTests.java | 4 +- .../ip2geo/common/DatasourceFacadeTests.java | 50 +++++++++++-------- .../jobscheduler/DatasourceRunnerTests.java | 10 ++-- .../ip2geo/jobscheduler/DatasourceTests.java | 4 +- .../DatasourceUpdateServiceTests.java | 2 +- .../processor/Ip2GeoProcessorTests.java | 6 +-- 15 files changed, 133 insertions(+), 84 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 83aa3acc..3d977a92 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -43,10 +43,10 @@ public class PutDatasourceRequest extends AcknowledgedRequest " + MAX_DATASOURCE_NAME_BYTES + ")"); return; } - if (datasourceName.equals(".") || datasourceName.equals("..")) { + if (name.equals(".") || name.equals("..")) { errors.addValidationError("Datasource name must not be '.' or '..'"); return; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index c1ca0737..43e0f76c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -73,7 +73,7 @@ protected void doExecute(final Task task, final PutDatasourceRequest request, fi try { Datasource datasource = Datasource.Builder.build(request); IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) - .id(datasource.getId()) + .id(datasource.getName()) .source(datasource.toXContent(JsonXContent.contentBuilder(), null)) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .opType(DocWriteRequest.OpType.CREATE); @@ -100,7 +100,7 @@ public void onResponse(final IndexResponse indexResponse) { @Override public void onFailure(final Exception e) { if (e instanceof VersionConflictEngineException) { - listener.onFailure(new ResourceAlreadyExistsException("datasource [{}] already exists", datasource.getId())); + listener.onFailure(new ResourceAlreadyExistsException("datasource [{}] already exists", datasource.getName())); } else { listener.onFailure(e); } @@ -119,7 +119,7 @@ protected void createDatasource(final Datasource datasource) { try { datasourceUpdateService.updateOrCreateGeoIpData(datasource); } catch (Exception e) { - log.error("Failed to create datasource for {}", datasource.getId(), e); + log.error("Failed to create datasource for {}", datasource.getName(), e); markDatasourceAsCreateFailed(datasource); } } @@ -130,7 +130,7 @@ private void markDatasourceAsCreateFailed(final Datasource datasource) { try { datasourceFacade.updateDatasource(datasource); } catch (Exception e) { - log.error("Failed to mark datasource state as CREATE_FAILED for {}", datasource.getId(), e); + log.error("Failed to mark datasource state as CREATE_FAILED for {}", datasource.getName(), e); } } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index b16e5478..4d964273 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -53,7 +53,7 @@ public DatasourceFacade(final Client client, final ClusterSettings clusterSettin public IndexResponse updateDatasource(final Datasource datasource) throws IOException { datasource.setLastUpdateTime(Instant.now()); IndexRequestBuilder requestBuilder = client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME); - requestBuilder.setId(datasource.getId()); + requestBuilder.setId(datasource.getName()); requestBuilder.setOpType(DocWriteRequest.OpType.INDEX); requestBuilder.setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); return client.index(requestBuilder.request()).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); @@ -61,17 +61,17 @@ public IndexResponse updateDatasource(final Datasource datasource) throws IOExce /** * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param id the name of a datasource + * @param name the name of a datasource * @return datasource * @throws IOException exception */ - public Datasource getDatasource(final String id) throws IOException { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); + public Datasource getDatasource(final String name) throws IOException { + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); GetResponse response; try { response = client.get(request).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); if (response.isExists() == false) { - log.error("Datasource[{}] does not exist in an index[{}]", id, DatasourceExtension.JOB_INDEX_NAME); + log.error("Datasource[{}] does not exist in an index[{}]", name, DatasourceExtension.JOB_INDEX_NAME); return null; } } catch (IndexNotFoundException e) { @@ -89,11 +89,11 @@ public Datasource getDatasource(final String id) throws IOException { /** * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} - * @param id the name of a datasource + * @param name the name of a datasource * @param actionListener the action listener */ - public void getDatasource(final String id, final ActionListener actionListener) { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, id); + public void getDatasource(final String name, final ActionListener actionListener) { + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); client.get(request, new ActionListener() { @Override public void onResponse(final GetResponse response) { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index dfc26d71..fc7d2e58 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -58,13 +58,13 @@ public class Datasource implements ScheduledJobParameter { /** * Default fields for job scheduling */ - private static final ParseField ID_FIELD = new ParseField("id"); - private static final ParseField ENABLED_FILED = new ParseField("update_enabled"); + private static final ParseField NAME_FIELD = new ParseField("name"); + private static final ParseField ENABLED_FIELD = new ParseField("update_enabled"); private static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time"); private static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field"); private static final ParseField SCHEDULE_FIELD = new ParseField("schedule"); - private static final ParseField ENABLED_TIME_FILED = new ParseField("enabled_time"); - private static final ParseField ENABLED_TIME_FILED_READABLE = new ParseField("enabled_time_field"); + private static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time"); + private static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field"); /** * Additional fields for datasource @@ -80,10 +80,10 @@ public class Datasource implements ScheduledJobParameter { */ /** - * @param id Id of a datasource - * @return Id of a datasource + * @param name name of a datasource + * @return name of a datasource */ - private String id; + private String name; /** * @param lastUpdateTime Last update time of a datasource * @return Last update time of a datasource @@ -169,10 +169,10 @@ public class Datasource implements ScheduledJobParameter { } ); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_UPDATE_TIME_FIELD); - PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ENABLED_TIME_FILED); - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FILED); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ENABLED_TIME_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SCHEDULE_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), ENDPOINT_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); @@ -205,7 +205,7 @@ public Datasource(final String id, final IntervalSchedule schedule, final String @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); - builder.field(ID_FIELD.getPreferredName(), id); + builder.field(NAME_FIELD.getPreferredName(), name); builder.timeField( LAST_UPDATE_TIME_FIELD.getPreferredName(), LAST_UPDATE_TIME_FIELD_READABLE.getPreferredName(), @@ -213,12 +213,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa ); if (enabledTime != null) { builder.timeField( - ENABLED_TIME_FILED.getPreferredName(), - ENABLED_TIME_FILED_READABLE.getPreferredName(), + ENABLED_TIME_FIELD.getPreferredName(), + ENABLED_TIME_FIELD_READABLE.getPreferredName(), enabledTime.toEpochMilli() ); } - builder.field(ENABLED_FILED.getPreferredName(), isEnabled); + builder.field(ENABLED_FIELD.getPreferredName(), isEnabled); builder.field(SCHEDULE_FIELD.getPreferredName(), schedule); builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); builder.field(STATE_FIELD.getPreferredName(), state.name()); @@ -231,7 +231,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa @Override public String getName() { - return id; + return name; } @Override @@ -313,7 +313,7 @@ public String indexNameFor(final DatasourceManifest manifest) { } private String indexNameFor(final long suffix) { - return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, suffix); + return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, name, suffix); } /** @@ -567,7 +567,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa */ public static class Builder { public static Datasource build(final PutDatasourceRequest request) { - String id = request.getDatasourceName(); + String id = request.getName(); IntervalSchedule schedule = new IntervalSchedule( Instant.now().truncatedTo(ChronoUnit.MILLIS), (int) request.getUpdateInterval().days(), diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index a75984b4..bc78fd0b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -153,7 +153,7 @@ protected void updateDatasource(final ScheduledJobParameter jobParameter) throws datasourceUpdateService.updateOrCreateGeoIpData(datasource); datasourceUpdateService.deleteUnusedIndices(datasource); } catch (Exception e) { - log.error("Failed to update datasource for {}", datasource.getId(), e); + log.error("Failed to update datasource for {}", datasource.getName(), e); datasource.getUpdateStats().setLastFailedAt(Instant.now()); datasourceFacade.updateDatasource(datasource); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index d3814805..4fe3274c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -63,7 +63,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource) throws Exceptio DatasourceManifest manifest = DatasourceManifest.Builder.build(url); if (shouldUpdate(datasource, manifest) == false) { - log.info("Skipping GeoIP database update. Update is not required for {}", datasource.getId()); + log.info("Skipping GeoIP database update. Update is not required for {}", datasource.getName()); datasource.getUpdateStats().setLastSkippedAt(Instant.now()); datasourceFacade.updateDatasource(datasource); return; @@ -110,7 +110,7 @@ public void deleteUnusedIndices(final Datasource parameter) { datasourceFacade.updateDatasource(parameter); } } catch (Exception e) { - log.error("Failed to delete old indices for {}", parameter.getId(), e); + log.error("Failed to delete old indices for {}", parameter.getName(), e); } } @@ -175,7 +175,11 @@ private void updateDatasourceAsSucceeded( datasource.enable(); datasource.setState(DatasourceState.AVAILABLE); datasourceFacade.updateDatasource(datasource); - log.info("GeoIP database creation succeeded for {} and took {} seconds", datasource.getId(), Duration.between(startTime, endTime)); + log.info( + "GeoIP database creation succeeded for {} and took {} seconds", + datasource.getName(), + Duration.between(startTime, endTime) + ); } /*** diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index c1aec21e..11ea1861 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -13,6 +13,8 @@ import java.io.File; import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashSet; import java.util.Locale; @@ -39,13 +41,16 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.ingest.IngestService; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.utils.LockService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; @@ -120,6 +125,13 @@ public DatasourceState randomStateExcept(DatasourceState state) { .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 2)); } + public DatasourceState randomState() { + return Arrays.stream(DatasourceState.values()) + .sequential() + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 1)); + } + public String randomIpAddress() { return String.format( Locale.ROOT, @@ -149,6 +161,33 @@ public File sampleIp2GeoFile() { return new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); } + public Datasource randomDatasource() { + Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + Datasource datasource = new Datasource(); + datasource.setName(GeospatialTestHelper.randomLowerCaseString()); + datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(30) + 1, ChronoUnit.DAYS)); + datasource.setState(randomState()); + datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); + datasource.setEndpoint(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase() + .setFields(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); + datasource.getDatabase().setProvider(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setUpdatedAt(now); + datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setValidForInDays(Randomness.get().nextInt(30) + 1l); + datasource.getUpdateStats().setLastSkippedAt(now); + datasource.getUpdateStats().setLastSucceededAt(now); + datasource.getUpdateStats().setLastFailedAt(now); + datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); + datasource.setLastUpdateTime(now); + if (Randomness.get().nextInt() % 2 == 0) { + datasource.enable(); + } else { + datasource.disable(); + } + return datasource; + } + /** * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) * is merged in OpenSearch core diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index ca2cda81..634d9581 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -143,7 +143,7 @@ public void testValidateDatasourceNames() throws Exception { ); for (Map.Entry entry : nameToError.entrySet()) { - request.setDatasourceName(entry.getKey()); + request.setName(entry.getKey()); // Run exception = request.validate(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 38226c7f..ff542418 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -54,7 +54,7 @@ public void testDoExecute() throws Exception { assertTrue(actionRequest instanceof IndexRequest); IndexRequest indexRequest = (IndexRequest) actionRequest; assertEquals(DatasourceExtension.JOB_INDEX_NAME, indexRequest.index()); - assertEquals(request.getDatasourceName(), indexRequest.id()); + assertEquals(request.getName(), indexRequest.id()); assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); return null; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index b812cdfa..521f3420 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -53,7 +53,7 @@ public void testPrepareRequest() { PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval()); - assertEquals(datasourceName, putDatasourceRequest.getDatasourceName()); + assertEquals(datasourceName, putDatasourceRequest.getName()); isExecuted.set(true); return null; }); @@ -74,7 +74,7 @@ public void testPrepareRequestDefaultValue() { PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; assertEquals("https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", putDatasourceRequest.getEndpoint()); assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval()); - assertEquals(datasourceName, putDatasourceRequest.getDatasourceName()); + assertEquals(datasourceName, putDatasourceRequest.getName()); isExecuted.set(true); return null; }); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 066ea27b..4a34dcb3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -59,7 +59,7 @@ public void testUpdateDatasource() throws Exception { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof IndexRequest); IndexRequest request = (IndexRequest) actionRequest; - assertEquals(datasource.getId(), request.id()); + assertEquals(datasource.getName(), request.id()); assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); return null; @@ -71,55 +71,42 @@ public void testUpdateDatasource() throws Exception { public void testGetDatasourceException() throws Exception { Datasource datasource = setupClientForGetRequest(true, new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME)); - assertNull(datasourceFacade.getDatasource(datasource.getId())); + assertNull(datasourceFacade.getDatasource(datasource.getName())); } public void testGetDatasourceExist() throws Exception { Datasource datasource = setupClientForGetRequest(true, null); - assertEquals(datasource, datasourceFacade.getDatasource(datasource.getId())); + assertEquals(datasource, datasourceFacade.getDatasource(datasource.getName())); } public void testGetDatasourceNotExist() throws Exception { Datasource datasource = setupClientForGetRequest(false, null); - assertNull(datasourceFacade.getDatasource(datasource.getId())); + assertNull(datasourceFacade.getDatasource(datasource.getName())); } public void testGetDatasourceExistWithListener() { Datasource datasource = setupClientForGetRequest(true, null); ActionListener listener = mock(ActionListener.class); - datasourceFacade.getDatasource(datasource.getId(), listener); + datasourceFacade.getDatasource(datasource.getName(), listener); verify(listener).onResponse(eq(datasource)); } public void testGetDatasourceNotExistWithListener() { Datasource datasource = setupClientForGetRequest(false, null); ActionListener listener = mock(ActionListener.class); - datasourceFacade.getDatasource(datasource.getId(), listener); + datasourceFacade.getDatasource(datasource.getName(), listener); verify(listener).onResponse(null); } private Datasource setupClientForGetRequest(final boolean isExist, final RuntimeException exception) { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Datasource datasource = new Datasource( - datasourceName, - new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS), - "https://test.com" - ); + Datasource datasource = randomDatasource(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof GetRequest); GetRequest request = (GetRequest) actionRequest; - assertEquals(datasource.getId(), request.id()); + assertEquals(datasource.getName(), request.id()); assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); - GetResponse response = mock(GetResponse.class); - when(response.isExists()).thenReturn(isExist); - try { - when(response.getSourceAsBytesRef()).thenReturn( - BytesReference.bytes(datasource.toXContent(JsonXContent.contentBuilder(), null)) - ); - } catch (IOException e) { - throw new RuntimeException(e); - } + GetResponse response = getMockedGetResponse(isExist ? datasource : null); if (exception != null) { throw exception; } @@ -127,4 +114,23 @@ private Datasource setupClientForGetRequest(final boolean isExist, final Runtime }); return datasource; } + + private GetResponse getMockedGetResponse(Datasource datasource) { + GetResponse response = mock(GetResponse.class); + when(response.isExists()).thenReturn(datasource != null); + when(response.getSourceAsBytesRef()).thenReturn(toBytesReference(datasource)); + return response; + } + + private BytesReference toBytesReference(Datasource datasource) { + if (datasource == null) { + return null; + } + + try { + return BytesReference.bytes(datasource.toXContent(JsonXContent.contentBuilder(), null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 0e33e232..34e4ab3b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -68,7 +68,7 @@ public void testUpdateDatasourceInvalidState() throws Exception { datasource.enable(); datasource.getUpdateStats().setLastFailedAt(null); datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); - when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); // Run DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); @@ -82,8 +82,8 @@ public void testUpdateDatasourceInvalidState() throws Exception { public void testUpdateDatasource() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); - datasource.setId(randomLowerCaseString()); - when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + datasource.setName(randomLowerCaseString()); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); // Run DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); @@ -95,9 +95,9 @@ public void testUpdateDatasource() throws Exception { public void testUpdateDatasourceExceptionHandling() throws Exception { Datasource datasource = new Datasource(); - datasource.setId(randomLowerCaseString()); + datasource.setName(randomLowerCaseString()); datasource.getUpdateStats().setLastFailedAt(null); - when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); doThrow(new RuntimeException("test failure")).when(datasourceUpdateService).deleteUnusedIndices(any()); // Run diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index b4f76ef6..9c7e30fc 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -54,7 +54,7 @@ public void testCurrentIndexName() { String id = GeospatialTestHelper.randomLowerCaseString(); Instant now = Instant.now(); Datasource datasource = new Datasource(); - datasource.setId(id); + datasource.setName(id); datasource.getDatabase().setProvider("provider"); datasource.getDatabase().setMd5Hash("md5Hash"); datasource.getDatabase().setUpdatedAt(now); @@ -73,7 +73,7 @@ public void testGetIndexNameFor() { String id = GeospatialTestHelper.randomLowerCaseString(); Datasource datasource = new Datasource(); - datasource.setId(id); + datasource.setName(id); assertEquals( String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, updatedAt), datasource.indexNameFor(manifest) diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 3c9ec15d..517c45a0 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -132,7 +132,7 @@ public void testDeleteUnusedIndices() throws Exception { String oldIndex = indexPrefix + now.minusMillis(1).toEpochMilli(); String lingeringIndex = indexPrefix + now.minusMillis(2).toEpochMilli(); Datasource datasource = new Datasource(); - datasource.setId(datasourceName); + datasource.setName(datasourceName); datasource.getIndices().add(currentIndex); datasource.getIndices().add(oldIndex); datasource.getIndices().add(lingeringIndex); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index bd6857ed..6028b029 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -68,7 +68,7 @@ public void testCreateWithNoDatasource() { public void testCreateWithInvalidDatasourceState() { Datasource datasource = new Datasource(); - datasource.setId(GeospatialTestHelper.randomLowerCaseString()); + datasource.setName(GeospatialTestHelper.randomLowerCaseString()); datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); OpenSearchException exception = expectThrows(OpenSearchException.class, () -> createProcessor(datasource, Collections.emptyMap())); assertTrue(exception.getDetailedMessage().contains("available state")); @@ -323,14 +323,14 @@ public void testExecuteInternal() throws Exception { private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { Datasource datasource = new Datasource(); - datasource.setId(datasourceName); + datasource.setName(datasourceName); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setFields(SUPPORTED_FIELDS); return createProcessor(datasource, config); } private Ip2GeoProcessor createProcessor(final Datasource datasource, final Map config) throws Exception { - when(datasourceFacade.getDatasource(datasource.getId())).thenReturn(datasource); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); Map baseConfig = new HashMap<>(); baseConfig.put(CONFIG_FIELD_KEY, "ip"); baseConfig.put(CONFIG_DATASOURCE_KEY, datasource.getName()); From 2bc5bb07931e2a0f9c2a478849dc614e0983ceed Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 11:14:01 -0700 Subject: [PATCH 16/61] Change field name from md5 to sha256 (#285) Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceManifest.java | 14 ++++++------- .../ip2geo/jobscheduler/Datasource.java | 20 +++++++++---------- .../jobscheduler/DatasourceUpdateService.java | 4 ++-- .../geospatial/ip2geo/Ip2GeoTestCase.java | 2 +- .../ip2geo/common/GeoIpDataFacadeTests.java | 4 ++-- .../ip2geo/jobscheduler/DatasourceTests.java | 4 ++-- .../DatasourceUpdateServiceTests.java | 10 +++++----- src/test/resources/ip2geo/manifest.json | 2 +- .../ip2geo/manifest_invalid_url.json | 2 +- .../resources/ip2geo/manifest_template.json | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java index 0c65382f..957740a8 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -40,7 +40,7 @@ public class DatasourceManifest { private static final ParseField URL_FIELD = new ParseField("url"); private static final ParseField DB_NAME_FIELD = new ParseField("db_name"); - private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); private static final ParseField PROVIDER_FIELD = new ParseField("provider"); @@ -56,10 +56,10 @@ public class DatasourceManifest { */ private String dbName; /** - * @param md5Hash MD5 hash value of a database file - * @return MD5 hash value of a database file + * @param sha256Hash SHA256 hash value of a database file + * @return SHA256 hash value of a database file */ - private String md5Hash; + private String sha256Hash; /** * @param validForInDays A duration in which the database file is valid to use * @return A duration in which the database file is valid to use @@ -85,17 +85,17 @@ public class DatasourceManifest { args -> { String url = (String) args[0]; String dbName = (String) args[1]; - String md5hash = (String) args[2]; + String sha256Hash = (String) args[2]; Long validForInDays = (Long) args[3]; Long updatedAt = (Long) args[4]; String provider = (String) args[5]; - return new DatasourceManifest(url, dbName, md5hash, validForInDays, updatedAt, provider); + return new DatasourceManifest(url, dbName, sha256Hash, validForInDays, updatedAt, provider); } ); static { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), URL_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DB_NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index fc7d2e58..2f93cff5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -345,7 +345,7 @@ public boolean isExpired() { */ public void setDatabase(final DatasourceManifest datasourceManifest, final List fields) { this.database.setProvider(datasourceManifest.getProvider()); - this.database.setMd5Hash(datasourceManifest.getMd5Hash()); + this.database.setSha256Hash(datasourceManifest.getSha256Hash()); this.database.setUpdatedAt(Instant.ofEpochMilli(datasourceManifest.getUpdatedAt())); this.database.setValidForInDays(datasourceManifest.getValidForInDays()); this.database.setFields(fields); @@ -389,7 +389,7 @@ public boolean isCompatible(final List fields) { @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Database implements ToXContent { private static final ParseField PROVIDER_FIELD = new ParseField("provider"); - private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at_field"); private static final ParseField FIELDS_FIELD = new ParseField("fields"); @@ -401,10 +401,10 @@ public static class Database implements ToXContent { */ private String provider; /** - * @param md5Hash MD5 hash value of a database file - * @return MD5 hash value of a database file + * @param sha256Hash SHA256 hash value of a database file + * @return SHA256 hash value of a database file */ - private String md5Hash; + private String sha256Hash; /** * @param updatedAt A date when the database was updated * @return A date when the database was updated @@ -426,16 +426,16 @@ public static class Database implements ToXContent { true, args -> { String provider = (String) args[0]; - String md5Hash = (String) args[1]; + String sha256Hash = (String) args[1]; Instant updatedAt = args[2] == null ? null : Instant.ofEpochMilli((Long) args[2]); Long validForInDays = (Long) args[3]; List fields = (List) args[4]; - return new Database(provider, md5Hash, updatedAt, validForInDays, fields); + return new Database(provider, sha256Hash, updatedAt, validForInDays, fields); } ); static { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); @@ -447,8 +447,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (provider != null) { builder.field(PROVIDER_FIELD.getPreferredName(), provider); } - if (md5Hash != null) { - builder.field(MD5_HASH_FIELD.getPreferredName(), md5Hash); + if (sha256Hash != null) { + builder.field(SHA256_HASH_FIELD.getPreferredName(), sha256Hash); } if (updatedAt != null) { builder.timeField( diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 4fe3274c..07ea55c7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -203,7 +203,7 @@ private String setupIndex(final DatasourceManifest manifest, final Datasource da * * Update is needed when all following conditions are met * 1. updatedAt value in datasource is equal or before updateAt value in manifest - * 2. MD5 hash value in datasource is different with MD5 hash value in manifest + * 2. SHA256 hash value in datasource is different with SHA256 hash value in manifest * * @param datasource * @param manifest @@ -214,7 +214,7 @@ private boolean shouldUpdate(final Datasource datasource, final DatasourceManife return false; } - if (manifest.getMd5Hash().equals(datasource.getDatabase().getMd5Hash())) { + if (manifest.getSha256Hash().equals(datasource.getDatabase().getSha256Hash())) { return false; } return true; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 11ea1861..1051693a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -173,7 +173,7 @@ public Datasource randomDatasource() { .setFields(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase().setProvider(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setUpdatedAt(now); - datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setValidForInDays(Randomness.get().nextInt(30) + 1l); datasource.getUpdateStats().setLastSkippedAt(now); datasource.getUpdateStats().setLastSucceededAt(now); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 4a61e612..786fc6f9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -111,7 +111,7 @@ public void testGetDatabaseReader() throws Exception { DatasourceManifest manifest = new DatasourceManifest( zipFile.toURI().toURL().toExternalForm(), "sample_valid.csv", - "fake_md5", + "fake_sha256", 1l, Instant.now().toEpochMilli(), "tester" @@ -128,7 +128,7 @@ public void testGetDatabaseReaderNoFile() throws Exception { DatasourceManifest manifest = new DatasourceManifest( zipFile.toURI().toURL().toExternalForm(), "no_file.csv", - "fake_md5", + "fake_sha256", 1l, Instant.now().toEpochMilli(), "tester" diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 9c7e30fc..d1a19c0c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -36,7 +36,7 @@ public void testParser() throws Exception { datasource.getDatabase().setFields(Arrays.asList("field1", "field2")); datasource.getDatabase().setProvider("test_provider"); datasource.getDatabase().setUpdatedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); - datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setValidForInDays(1l); datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); datasource.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); @@ -56,7 +56,7 @@ public void testCurrentIndexName() { Datasource datasource = new Datasource(); datasource.setName(id); datasource.getDatabase().setProvider("provider"); - datasource.getDatabase().setMd5Hash("md5Hash"); + datasource.getDatabase().setSha256Hash("sha256Hash"); datasource.getDatabase().setUpdatedAt(now); datasource.getDatabase().setValidForInDays(10l); datasource.getDatabase().setFields(new ArrayList<>()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 517c45a0..8d78f44b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -46,7 +46,7 @@ public void testUpdateDatasourceSkip() throws Exception { datasource.setState(DatasourceState.AVAILABLE); datasource.getUpdateStats().setLastSkippedAt(null); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash()); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash()); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); // Run @@ -69,7 +69,7 @@ public void testUpdateDatasourceInvalidFile() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); @@ -87,7 +87,7 @@ public void testUpdateDatasourceIncompatibleFields() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name", "additional_field")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); @@ -105,7 +105,7 @@ public void testUpdateDatasource() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); datasource.getUpdateStats().setLastSucceededAt(null); @@ -116,7 +116,7 @@ public void testUpdateDatasource() throws Exception { // Verify assertEquals(manifest.getProvider(), datasource.getDatabase().getProvider()); - assertEquals(manifest.getMd5Hash(), datasource.getDatabase().getMd5Hash()); + assertEquals(manifest.getSha256Hash(), datasource.getDatabase().getSha256Hash()); assertEquals(Instant.ofEpochMilli(manifest.getUpdatedAt()), datasource.getDatabase().getUpdatedAt()); assertEquals(manifest.getValidForInDays(), datasource.getDatabase().getValidForInDays()); assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); diff --git a/src/test/resources/ip2geo/manifest.json b/src/test/resources/ip2geo/manifest.json index 652bc9d8..4986fbd8 100644 --- a/src/test/resources/ip2geo/manifest.json +++ b/src/test/resources/ip2geo/manifest.json @@ -1,7 +1,7 @@ { "url": "https://test.com/db.zip", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "sample_provider" diff --git a/src/test/resources/ip2geo/manifest_invalid_url.json b/src/test/resources/ip2geo/manifest_invalid_url.json index 77d68aaf..4e806f49 100644 --- a/src/test/resources/ip2geo/manifest_invalid_url.json +++ b/src/test/resources/ip2geo/manifest_invalid_url.json @@ -1,7 +1,7 @@ { "url": "invalid://test.com/db.zip", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "sample_provider" diff --git a/src/test/resources/ip2geo/manifest_template.json b/src/test/resources/ip2geo/manifest_template.json index 4c273fa4..92ceb590 100644 --- a/src/test/resources/ip2geo/manifest_template.json +++ b/src/test/resources/ip2geo/manifest_template.json @@ -1,7 +1,7 @@ { "url": "URL", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "maxmind" From df6de87cdd72ac94b05918f6036947538f5c008e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 12:08:29 -0700 Subject: [PATCH 17/61] Implement get datasource api (#279) Signed-off-by: Heemin Kim --- .../ip2geo/action/GetDatasourceAction.java | 29 +++++ .../ip2geo/action/GetDatasourceRequest.java | 69 ++++++++++++ .../ip2geo/action/GetDatasourceResponse.java | 91 ++++++++++++++++ .../action/GetDatasourceTransportAction.java | 76 +++++++++++++ .../action/RestGetDatasourceHandler.java | 49 +++++++++ .../action/RestPutDatasourceHandler.java | 6 +- .../ip2geo/common/DatasourceFacade.java | 89 +++++++++++++++ .../ip2geo/jobscheduler/Datasource.java | 96 ++++++++++++++--- .../geospatial/plugin/GeospatialPlugin.java | 13 ++- .../geospatial/ip2geo/Ip2GeoTestCase.java | 21 ++-- .../action/GetDatasourceRequestTests.java | 46 ++++++++ .../action/GetDatasourceResponseTests.java | 99 +++++++++++++++++ .../GetDatasourceTransportActionTests.java | 88 +++++++++++++++ .../action/PutDatasourceRequestTests.java | 2 +- .../action/RestGetDatasourceHandlerTests.java | 81 ++++++++++++++ .../ip2geo/common/DatasourceFacadeTests.java | 101 ++++++++++++++++-- .../DatasourceExtensionTests.java | 3 +- .../ip2geo/jobscheduler/DatasourceTests.java | 8 +- .../plugin/GeospatialPluginTests.java | 4 +- 19 files changed, 929 insertions(+), 42 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java new file mode 100644 index 00000000..09f90934 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import org.opensearch.action.ActionType; + +/** + * Ip2Geo datasource get action + */ +public class GetDatasourceAction extends ActionType { + /** + * Get datasource action instance + */ + public static final GetDatasourceAction INSTANCE = new GetDatasourceAction(); + /** + * Name of a get datasource action + */ + public static final String NAME = "cluster:admin/geospatial/datasource/get"; + + private GetDatasourceAction() { + super(NAME, GetDatasourceResponse::new); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java new file mode 100644 index 00000000..a3de669d --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; + +import lombok.Getter; +import lombok.Setter; + +import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +/** + * Ip2Geo datasource get request + */ +@Getter +@Setter +public class GetDatasourceRequest extends ActionRequest { + /** + * @param names the datasource names + * @return the datasource names + */ + private String[] names; + + /** + * Constructs a new get datasource request with a list of datasources. + * + * If the list of datasources is empty or it contains a single element "_all", all registered datasources + * are returned. + * + * @param names list of datasource names + */ + public GetDatasourceRequest(final String[] names) { + this.names = names; + } + + /** + * Constructor with stream input + * @param in the stream input + * @throws IOException IOException + */ + public GetDatasourceRequest(final StreamInput in) throws IOException { + super(in); + this.names = in.readStringArray(); + } + + @Override + public ActionRequestValidationException validate() { + if (names == null) { + throw new OpenSearchException("names should not be null"); + } + return null; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(names); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java new file mode 100644 index 00000000..e4915fa8 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; + +import org.opensearch.action.ActionResponse; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; + +/** + * Ip2Geo datasource get request + */ +@Getter +@Setter +@EqualsAndHashCode +public class GetDatasourceResponse extends ActionResponse implements ToXContentObject { + private static final ParseField FIELD_NAME_DATASOURCES = new ParseField("datasources"); + private static final ParseField FIELD_NAME_NAME = new ParseField("name"); + private static final ParseField FIELD_NAME_STATE = new ParseField("state"); + private static final ParseField FIELD_NAME_ENDPOINT = new ParseField("endpoint"); + private static final ParseField FIELD_NAME_UPDATE_INTERVAL = new ParseField("update_interval_in_days"); + private static final ParseField FIELD_NAME_NEXT_UPDATE_AT = new ParseField("next_update_at_in_epoch_millis"); + private static final ParseField FIELD_NAME_NEXT_UPDATE_AT_READABLE = new ParseField("next_update_at"); + private static final ParseField FIELD_NAME_DATABASE = new ParseField("database"); + private static final ParseField FIELD_NAME_UPDATE_STATS = new ParseField("update_stats"); + private List datasources; + + /** + * Default constructor + * + * @param datasources List of datasources + */ + public GetDatasourceResponse(final List datasources) { + this.datasources = datasources; + } + + /** + * Constructor with StreamInput + * + * @param in the stream input + */ + public GetDatasourceResponse(final StreamInput in) throws IOException { + datasources = in.readList(Datasource::new); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeList(datasources); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.startArray(FIELD_NAME_DATASOURCES.getPreferredName()); + for (Datasource datasource : datasources) { + builder.startObject(); + builder.field(FIELD_NAME_NAME.getPreferredName(), datasource.getName()); + builder.field(FIELD_NAME_STATE.getPreferredName(), datasource.getState()); + builder.field(FIELD_NAME_ENDPOINT.getPreferredName(), datasource.getEndpoint()); + builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), datasource.getSchedule().getInterval()); + builder.timeField( + FIELD_NAME_NEXT_UPDATE_AT.getPreferredName(), + FIELD_NAME_NEXT_UPDATE_AT_READABLE.getPreferredName(), + datasource.getSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() + ); + builder.field(FIELD_NAME_DATABASE.getPreferredName(), datasource.getDatabase()); + builder.field(FIELD_NAME_UPDATE_STATS.getPreferredName(), datasource.getUpdateStats()); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + return builder; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java new file mode 100644 index 00000000..4bc69949 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.util.List; + +import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +/** + * Transport action to get datasource + */ +public class GetDatasourceTransportAction extends HandledTransportAction { + private final DatasourceFacade datasourceFacade; + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param datasourceFacade the datasource facade + */ + @Inject + public GetDatasourceTransportAction( + final TransportService transportService, + final ActionFilters actionFilters, + final DatasourceFacade datasourceFacade + ) { + super(GetDatasourceAction.NAME, transportService, actionFilters, GetDatasourceRequest::new); + this.datasourceFacade = datasourceFacade; + } + + @Override + protected void doExecute(final Task task, final GetDatasourceRequest request, final ActionListener listener) { + if (shouldGetAllDatasource(request)) { + // We don't expect too many data sources. Therefore, querying all data sources without pagination should be fine. + datasourceFacade.getAllDatasources(newActionListener(listener)); + } else { + datasourceFacade.getDatasources(request.getNames(), newActionListener(listener)); + } + } + + private boolean shouldGetAllDatasource(final GetDatasourceRequest request) { + if (request.getNames() == null) { + throw new OpenSearchException("names in a request should not be null"); + } + + return request.getNames().length == 0 || (request.getNames().length == 1 && "_all".equals(request.getNames()[0])); + } + + private ActionListener> newActionListener(final ActionListener listener) { + return new ActionListener<>() { + @Override + public void onResponse(final List datasources) { + listener.onResponse(new GetDatasourceResponse(datasources)); + } + + @Override + public void onFailure(final Exception e) { + listener.onFailure(e); + } + }; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java new file mode 100644 index 00000000..0dc50530 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.rest.RestRequest.Method.GET; + +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.Strings; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +/** + * Rest handler for Ip2Geo datasource get request + */ +public class RestGetDatasourceHandler extends BaseRestHandler { + private static final String ACTION_NAME = "ip2geo_datasource_get"; + + @Override + public String getName() { + return ACTION_NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + final String[] names = request.paramAsStringArray("name", Strings.EMPTY_ARRAY); + final GetDatasourceRequest getDatasourceRequest = new GetDatasourceRequest(names); + + return channel -> client.executeLocally(GetDatasourceAction.INSTANCE, getDatasourceRequest, new RestToXContentListener<>(channel)); + } + + @Override + public List routes() { + return List.of( + new Route(GET, String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource")), + new Route(GET, String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/{name}")) + ); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java index 3ccffa06..d35b7751 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java @@ -39,7 +39,7 @@ * */ public class RestPutDatasourceHandler extends BaseRestHandler { - private static final String ACTION_NAME = "ip2geo_datasource"; + private static final String ACTION_NAME = "ip2geo_datasource_put"; private final ClusterSettings clusterSettings; public RestPutDatasourceHandler(final ClusterSettings clusterSettings) { @@ -53,7 +53,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("id")); + final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("name")); if (request.hasContentOrSourceParam()) { try (XContentParser parser = request.contentOrSourceParamParser()) { PutDatasourceRequest.PARSER.parse(parser, putDatasourceRequest, null); @@ -70,7 +70,7 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No @Override public List routes() { - String path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/{id}"); + String path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/{name}"); return List.of(new Route(PUT, path)); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 4d964273..25d2f5af 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -10,16 +10,25 @@ import java.io.IOException; import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; +import org.opensearch.action.get.MultiGetItemResponse; +import org.opensearch.action.get.MultiGetResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; +import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; @@ -30,12 +39,15 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; /** * Facade class for datasource */ @Log4j2 public class DatasourceFacade { + private static final Integer MAX_SIZE = 1000; private final Client client; private final ClusterSettings clusterSettings; @@ -120,4 +132,81 @@ public void onFailure(final Exception e) { } }); } + + /** + * Get datasources from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param names the array of datasource names + * @param actionListener the action listener + */ + public void getDatasources(final String[] names, final ActionListener> actionListener) { + client.prepareMultiGet() + .add(DatasourceExtension.JOB_INDEX_NAME, names) + .execute(createGetDataSourceQueryActionLister(MultiGetResponse.class, actionListener)); + } + + /** + * Get all datasources up to {@code MAX_SIZE} from an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param actionListener the action listener + */ + public void getAllDatasources(final ActionListener> actionListener) { + client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(MAX_SIZE) + .execute(createGetDataSourceQueryActionLister(SearchResponse.class, actionListener)); + } + + private ActionListener createGetDataSourceQueryActionLister( + final Class response, + final ActionListener> actionListener + ) { + return new ActionListener() { + @Override + public void onResponse(final T response) { + try { + List bytesReferences = toBytesReferences(response); + List datasources = bytesReferences.stream() + .map(bytesRef -> toDatasource(bytesRef)) + .collect(Collectors.toList()); + actionListener.onResponse(datasources); + } catch (Exception e) { + actionListener.onFailure(e); + } + } + + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); + } + }; + } + + private List toBytesReferences(final Object response) { + if (response instanceof SearchResponse) { + SearchResponse searchResponse = (SearchResponse) response; + return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getSourceRef).collect(Collectors.toList()); + } else if (response instanceof MultiGetResponse) { + MultiGetResponse multiGetResponse = (MultiGetResponse) response; + return Arrays.stream(multiGetResponse.getResponses()) + .map(MultiGetItemResponse::getResponse) + .filter(Objects::nonNull) + .filter(GetResponse::isExists) + .map(GetResponse::getSourceAsBytesRef) + .collect(Collectors.toList()); + } else { + throw new OpenSearchException("No supported instance type[{}] is provided", response.getClass()); + } + } + + private Datasource toDatasource(final BytesReference bytesReference) { + try { + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + bytesReference + ); + return Datasource.PARSER.parse(parser, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 2f93cff5..7c7eae55 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -25,6 +25,9 @@ import lombok.Setter; import lombok.ToString; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; @@ -45,7 +48,7 @@ @ToString @EqualsAndHashCode @AllArgsConstructor -public class Datasource implements ScheduledJobParameter { +public class Datasource implements Writeable, ScheduledJobParameter { /** * Prefix of indices having Ip2Geo data */ @@ -142,7 +145,7 @@ public class Datasource implements ScheduledJobParameter { "datasource_metadata", true, args -> { - String id = (String) args[0]; + String name = (String) args[0]; Instant lastUpdateTime = Instant.ofEpochMilli((long) args[1]); Instant enabledTime = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); boolean isEnabled = (boolean) args[3]; @@ -153,7 +156,7 @@ public class Datasource implements ScheduledJobParameter { Database database = (Database) args[8]; UpdateStats updateStats = (UpdateStats) args[9]; Datasource parameter = new Datasource( - id, + name, lastUpdateTime, enabledTime, isEnabled, @@ -187,9 +190,9 @@ public Datasource() { this(null, null, null); } - public Datasource(final String id, final IntervalSchedule schedule, final String endpoint) { + public Datasource(final String name, final IntervalSchedule schedule, final String endpoint) { this( - id, + name, Instant.now().truncatedTo(ChronoUnit.MILLIS), null, false, @@ -202,6 +205,33 @@ public Datasource(final String id, final IntervalSchedule schedule, final String ); } + public Datasource(final StreamInput in) throws IOException { + name = in.readString(); + lastUpdateTime = toInstant(in.readVLong()); + enabledTime = toInstant(in.readOptionalVLong()); + isEnabled = in.readBoolean(); + schedule = new IntervalSchedule(in); + endpoint = in.readString(); + state = DatasourceState.valueOf(in.readString()); + indices = in.readStringList(); + database = new Database(in); + updateStats = new UpdateStats(in); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(name); + out.writeVLong(lastUpdateTime.toEpochMilli()); + out.writeOptionalVLong(enabledTime == null ? null : enabledTime.toEpochMilli()); + out.writeBoolean(isEnabled); + schedule.writeTo(out); + out.writeString(endpoint); + out.writeString(state.name()); + out.writeStringCollection(indices); + database.writeTo(out); + updateStats.writeTo(out); + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -378,6 +408,10 @@ public boolean isCompatible(final List fields) { return true; } + private static Instant toInstant(final Long epochMilli) { + return epochMilli == null ? null : Instant.ofEpochMilli(epochMilli); + } + /** * Database of a datasource */ @@ -387,11 +421,11 @@ public boolean isCompatible(final List fields) { @EqualsAndHashCode @AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class Database implements ToXContent { + public static class Database implements Writeable, ToXContent { private static final ParseField PROVIDER_FIELD = new ParseField("provider"); private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); - private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); - private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at_field"); + private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at_in_epoch_millis"); + private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at"); private static final ParseField FIELDS_FIELD = new ParseField("fields"); private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); @@ -441,6 +475,23 @@ public static class Database implements ToXContent { PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); } + public Database(final StreamInput in) throws IOException { + provider = in.readOptionalString(); + sha256Hash = in.readOptionalString(); + updatedAt = toInstant(in.readOptionalVLong()); + validForInDays = in.readOptionalVLong(); + fields = in.readOptionalStringList(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeOptionalString(provider); + out.writeOptionalString(sha256Hash); + out.writeOptionalVLong(updatedAt.toEpochMilli()); + out.writeOptionalVLong(validForInDays); + out.writeOptionalStringCollection(fields); + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -481,14 +532,14 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa @EqualsAndHashCode @AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class UpdateStats implements ToXContent { - private static final ParseField LAST_SUCCEEDED_AT_FIELD = new ParseField("last_succeeded_at"); - private static final ParseField LAST_SUCCEEDED_AT_FIELD_READABLE = new ParseField("last_succeeded_at_field"); + public static class UpdateStats implements Writeable, ToXContent { + private static final ParseField LAST_SUCCEEDED_AT_FIELD = new ParseField("last_succeeded_at_in_epoch_millis"); + private static final ParseField LAST_SUCCEEDED_AT_FIELD_READABLE = new ParseField("last_succeeded_at"); private static final ParseField LAST_PROCESSING_TIME_IN_MILLIS_FIELD = new ParseField("last_processing_time_in_millis"); - private static final ParseField LAST_FAILED_AT_FIELD = new ParseField("last_failed_at"); - private static final ParseField LAST_FAILED_AT_FIELD_READABLE = new ParseField("last_failed_at_field"); - private static final ParseField LAST_SKIPPED_AT = new ParseField("last_skipped_at"); - private static final ParseField LAST_SKIPPED_AT_READABLE = new ParseField("last_skipped_at_field"); + private static final ParseField LAST_FAILED_AT_FIELD = new ParseField("last_failed_at_in_epoch_millis"); + private static final ParseField LAST_FAILED_AT_FIELD_READABLE = new ParseField("last_failed_at"); + private static final ParseField LAST_SKIPPED_AT = new ParseField("last_skipped_at_in_epoch_millis"); + private static final ParseField LAST_SKIPPED_AT_READABLE = new ParseField("last_skipped_at"); /** * @param lastSucceededAt The last time when GeoIP data update was succeeded @@ -530,6 +581,21 @@ public static class UpdateStats implements ToXContent { PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LAST_SKIPPED_AT); } + public UpdateStats(final StreamInput in) throws IOException { + lastSucceededAt = toInstant(in.readOptionalVLong()); + lastProcessingTimeInMillis = in.readOptionalVLong(); + lastFailedAt = toInstant(in.readOptionalVLong()); + lastSkippedAt = toInstant(in.readOptionalVLong()); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeOptionalVLong(lastSucceededAt.toEpochMilli()); + out.writeOptionalVLong(lastProcessingTimeInMillis); + out.writeOptionalVLong(lastFailedAt.toEpochMilli()); + out.writeOptionalVLong(lastSkippedAt.toEpochMilli()); + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index e468b6b1..085fab98 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -36,8 +36,11 @@ import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapper; import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldTypeParser; import org.opensearch.geospatial.index.query.xyshape.XYShapeQueryBuilder; +import org.opensearch.geospatial.ip2geo.action.GetDatasourceAction; +import org.opensearch.geospatial.ip2geo.action.GetDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.action.PutDatasourceAction; import org.opensearch.geospatial.ip2geo.action.PutDatasourceTransportAction; +import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; @@ -153,7 +156,12 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction(), new RestPutDatasourceHandler(clusterSettings)); + return List.of( + new RestUploadStatsAction(), + new RestUploadGeoJSONAction(), + new RestPutDatasourceHandler(clusterSettings), + new RestGetDatasourceHandler() + ); } @Override @@ -161,7 +169,8 @@ public List getRestHandlers( return List.of( new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class), new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class), - new ActionHandler<>(PutDatasourceAction.INSTANCE, PutDatasourceTransportAction.class) + new ActionHandler<>(PutDatasourceAction.INSTANCE, PutDatasourceTransportAction.class), + new ActionHandler<>(GetDatasourceAction.INSTANCE, GetDatasourceTransportAction.class) ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 1051693a..48451a9c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -116,7 +116,7 @@ public void clean() throws Exception { verifyingClient.close(); } - public DatasourceState randomStateExcept(DatasourceState state) { + protected DatasourceState randomStateExcept(DatasourceState state) { assertNotNull(state); return Arrays.stream(DatasourceState.values()) .sequential() @@ -125,14 +125,14 @@ public DatasourceState randomStateExcept(DatasourceState state) { .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 2)); } - public DatasourceState randomState() { + protected DatasourceState randomState() { return Arrays.stream(DatasourceState.values()) .sequential() .collect(Collectors.toList()) .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 1)); } - public String randomIpAddress() { + protected String randomIpAddress() { return String.format( Locale.ROOT, "%d.%d.%d.%d", @@ -144,12 +144,12 @@ public String randomIpAddress() { } @SuppressForbidden(reason = "unit test") - public String sampleManifestUrl() throws Exception { + protected String sampleManifestUrl() throws Exception { return Paths.get(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").toURI()).toUri().toURL().toExternalForm(); } @SuppressForbidden(reason = "unit test") - public String sampleManifestUrlWithInvalidUrl() throws Exception { + protected String sampleManifestUrlWithInvalidUrl() throws Exception { return Paths.get(this.getClass().getClassLoader().getResource("ip2geo/manifest_invalid_url.json").toURI()) .toUri() .toURL() @@ -157,11 +157,16 @@ public String sampleManifestUrlWithInvalidUrl() throws Exception { } @SuppressForbidden(reason = "unit test") - public File sampleIp2GeoFile() { + protected File sampleIp2GeoFile() { return new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); } - public Datasource randomDatasource() { + protected long randomPositiveLong() { + long value = Randomness.get().nextLong(); + return value < 0 ? -value : value; + } + + protected Datasource randomDatasource() { Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); @@ -178,7 +183,7 @@ public Datasource randomDatasource() { datasource.getUpdateStats().setLastSkippedAt(now); datasource.getUpdateStats().setLastSucceededAt(now); datasource.getUpdateStats().setLastFailedAt(now); - datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); + datasource.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); datasource.setLastUpdateTime(now); if (Randomness.get().nextInt() % 2 == 0) { datasource.enable(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java new file mode 100644 index 00000000..41dc2d97 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; + +public class GetDatasourceRequestTests extends Ip2GeoTestCase { + public void testStreamInOut_whenEmptyNames_thenSucceed() throws Exception { + String[] names = new String[0]; + GetDatasourceRequest request = new GetDatasourceRequest(names); + assertNull(request.validate()); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + GetDatasourceRequest copiedRequest = new GetDatasourceRequest(input); + + // Verify + assertArrayEquals(request.getNames(), copiedRequest.getNames()); + } + + public void testStreamInOut_whenNames_thenSucceed() throws Exception { + String[] names = { GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString() }; + GetDatasourceRequest request = new GetDatasourceRequest(names); + assertNull(request.validate()); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + GetDatasourceRequest copiedRequest = new GetDatasourceRequest(input); + + // Verify + assertArrayEquals(request.getNames(), copiedRequest.getNames()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java new file mode 100644 index 00000000..e07feaaa --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import org.opensearch.common.Strings; +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; + +public class GetDatasourceResponseTests extends Ip2GeoTestCase { + + public void testStreamInOut_whenValidInput_thenSucceed() throws Exception { + List datasourceList = Arrays.asList(randomDatasource(), randomDatasource()); + GetDatasourceResponse response = new GetDatasourceResponse(datasourceList); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + response.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + GetDatasourceResponse copiedResponse = new GetDatasourceResponse(input); + + // Verify + assertArrayEquals(response.getDatasources().toArray(), copiedResponse.getDatasources().toArray()); + } + + public void testToXContent_whenValidInput_thenSucceed() throws Exception { + List datasourceList = Arrays.asList(randomDatasource(), randomDatasource()); + GetDatasourceResponse response = new GetDatasourceResponse(datasourceList); + String json = Strings.toString(response.toXContent(JsonXContent.contentBuilder(), null)); + for (Datasource datasource : datasourceList) { + assertTrue(json.contains(String.format(Locale.ROOT, "\"name\":\"%s\"", datasource.getName()))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"state\":\"%s\"", datasource.getState()))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"endpoint\":\"%s\"", datasource.getEndpoint()))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"update_interval_in_days\":%d", datasource.getSchedule().getInterval()))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"next_update_at_in_epoch_millis\""))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"provider\":\"%s\"", datasource.getDatabase().getProvider()))); + assertTrue(json.contains(String.format(Locale.ROOT, "\"sha256_hash\":\"%s\"", datasource.getDatabase().getSha256Hash()))); + assertTrue( + json.contains( + String.format(Locale.ROOT, "\"updated_at_in_epoch_millis\":%d", datasource.getDatabase().getUpdatedAt().toEpochMilli()) + ) + ); + assertTrue(json.contains(String.format(Locale.ROOT, "\"valid_for_in_days\":%d", datasource.getDatabase().getValidForInDays()))); + for (String field : datasource.getDatabase().getFields()) { + assertTrue(json.contains(field)); + } + assertTrue( + json.contains( + String.format( + Locale.ROOT, + "\"last_succeeded_at_in_epoch_millis\":%d", + datasource.getUpdateStats().getLastSucceededAt().toEpochMilli() + ) + ) + ); + assertTrue( + json.contains( + String.format( + Locale.ROOT, + "\"last_processing_time_in_millis\":%d", + datasource.getUpdateStats().getLastProcessingTimeInMillis() + ) + ) + ); + assertTrue( + json.contains( + String.format( + Locale.ROOT, + "\"last_failed_at_in_epoch_millis\":%d", + datasource.getUpdateStats().getLastFailedAt().toEpochMilli() + ) + ) + ); + assertTrue( + json.contains( + String.format( + Locale.ROOT, + "\"last_skipped_at_in_epoch_millis\":%d", + datasource.getUpdateStats().getLastSkippedAt().toEpochMilli() + ) + ) + ); + + } + } + +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java new file mode 100644 index 00000000..bf8de2eb --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.action.ActionListener; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.tasks.Task; + +public class GetDatasourceTransportActionTests extends Ip2GeoTestCase { + private GetDatasourceTransportAction action; + + @Before + public void init() { + action = new GetDatasourceTransportAction(transportService, actionFilters, datasourceFacade); + } + + public void testDoExecute_whenAll_thenSucceed() throws Exception { + Task task = mock(Task.class); + GetDatasourceRequest request = new GetDatasourceRequest(new String[] { "_all" }); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getAllDatasources(captor.capture()); + + // Run + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + captor.getValue().onResponse(datasources); + + // Verify + verify(listener).onResponse(new GetDatasourceResponse(datasources)); + + // Run + RuntimeException exception = new RuntimeException(); + captor.getValue().onFailure(exception); + + // Verify + verify(listener).onFailure(exception); + } + + public void testDoExecute_whenNames_thenSucceed() { + Task task = mock(Task.class); + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + String[] datasourceNames = datasources.stream().map(Datasource::getName).toArray(String[]::new); + + GetDatasourceRequest request = new GetDatasourceRequest(datasourceNames); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getDatasources(eq(datasourceNames), captor.capture()); + + // Run + captor.getValue().onResponse(datasources); + + // Verify + verify(listener).onResponse(new GetDatasourceResponse(datasources)); + + // Run + RuntimeException exception = new RuntimeException(); + captor.getValue().onFailure(exception); + + // Verify + verify(listener).onFailure(exception); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 634d9581..01d3025c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -105,7 +105,7 @@ public void testValidateDatasourceNames() throws Exception { String domain = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(validDatasourceName); request.setEndpoint(sampleManifestUrl()); - request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(30) + 1)); + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(10) + 1)); // Run ActionRequestValidationException exception = request.validate(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java new file mode 100644 index 00000000..5d4bef42 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; + +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; + +public class RestGetDatasourceHandlerTests extends RestActionTestCase { + private String PATH_FOR_ALL = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource"); + private String path; + private RestGetDatasourceHandler action; + + @Before + public void setupAction() { + action = new RestGetDatasourceHandler(); + controller().registerHandler(action); + path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s"); + } + + public void testPrepareRequest_whenNames_thenSucceed() { + String dsName1 = GeospatialTestHelper.randomLowerCaseString(); + String dsName2 = GeospatialTestHelper.randomLowerCaseString(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath(String.format(Locale.ROOT, path, StringUtils.joinWith(",", dsName1, dsName2))) + .build(); + + AtomicBoolean isExecuted = new AtomicBoolean(false); + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof GetDatasourceRequest); + GetDatasourceRequest getDatasourceRequest = (GetDatasourceRequest) actionRequest; + assertArrayEquals(new String[] { dsName1, dsName2 }, getDatasourceRequest.getNames()); + isExecuted.set(true); + return null; + }); + + // Run + dispatchRequest(request); + + // Verify + assertTrue(isExecuted.get()); + } + + public void testPrepareRequest_whenAll_thenSucceed() { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath(PATH_FOR_ALL) + .build(); + + AtomicBoolean isExecuted = new AtomicBoolean(false); + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof GetDatasourceRequest); + GetDatasourceRequest getDatasourceRequest = (GetDatasourceRequest) actionRequest; + assertArrayEquals(new String[] {}, getDatasourceRequest.getNames()); + isExecuted.set(true); + return null; + }); + + // Run + dispatchRequest(request); + + // Verify + assertTrue(isExecuted.get()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 4a34dcb3..aacd940a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -16,14 +16,24 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.HashSet; +import java.util.List; +import org.apache.lucene.search.TotalHits; import org.junit.Before; +import org.mockito.ArgumentCaptor; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; +import org.opensearch.action.get.MultiGetItemResponse; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -33,7 +43,10 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; public class DatasourceFacadeTests extends Ip2GeoTestCase { private DatasourceFacade datasourceFacade; @@ -46,7 +59,7 @@ public void init() { ); } - public void testUpdateDatasource() throws Exception { + public void testUpdateDatasource_whenValidInput_thenSucceed() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Datasource datasource = new Datasource( datasourceName, @@ -69,29 +82,29 @@ public void testUpdateDatasource() throws Exception { assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); } - public void testGetDatasourceException() throws Exception { + public void testGetDatasource_whenException_thenNull() throws Exception { Datasource datasource = setupClientForGetRequest(true, new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME)); assertNull(datasourceFacade.getDatasource(datasource.getName())); } - public void testGetDatasourceExist() throws Exception { + public void testGetDatasource_whenExist_thenReturnDatasource() throws Exception { Datasource datasource = setupClientForGetRequest(true, null); assertEquals(datasource, datasourceFacade.getDatasource(datasource.getName())); } - public void testGetDatasourceNotExist() throws Exception { + public void testGetDatasource_whenNotExist_thenNull() throws Exception { Datasource datasource = setupClientForGetRequest(false, null); assertNull(datasourceFacade.getDatasource(datasource.getName())); } - public void testGetDatasourceExistWithListener() { + public void testGetDatasource_whenExistWithListener_thenListenerIsCalledWithDatasource() { Datasource datasource = setupClientForGetRequest(true, null); ActionListener listener = mock(ActionListener.class); datasourceFacade.getDatasource(datasource.getName(), listener); verify(listener).onResponse(eq(datasource)); } - public void testGetDatasourceNotExistWithListener() { + public void testGetDatasource_whenExistWithListener_thenListenerIsCalledWithNull() { Datasource datasource = setupClientForGetRequest(false, null); ActionListener listener = mock(ActionListener.class); datasourceFacade.getDatasource(datasource.getName(), listener); @@ -115,6 +128,76 @@ private Datasource setupClientForGetRequest(final boolean isExist, final Runtime return datasource; } + public void testGetDatasources_whenValidInput_thenSucceed() { + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + String[] names = datasources.stream().map(Datasource::getName).toArray(String[]::new); + ActionListener> listener = mock(ActionListener.class); + MultiGetItemResponse[] multiGetItemResponses = datasources.stream().map(datasource -> { + GetResponse getResponse = getMockedGetResponse(datasource); + MultiGetItemResponse multiGetItemResponse = mock(MultiGetItemResponse.class); + when(multiGetItemResponse.getResponse()).thenReturn(getResponse); + return multiGetItemResponse; + }).toArray(MultiGetItemResponse[]::new); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof MultiGetRequest); + MultiGetRequest request = (MultiGetRequest) actionRequest; + assertEquals(2, request.getItems().size()); + for (MultiGetRequest.Item item : request.getItems()) { + assertEquals(DatasourceExtension.JOB_INDEX_NAME, item.index()); + assertTrue(datasources.stream().filter(datasource -> datasource.getName().equals(item.id())).findAny().isPresent()); + } + + MultiGetResponse response = mock(MultiGetResponse.class); + when(response.getResponses()).thenReturn(multiGetItemResponses); + return response; + }); + + // Run + datasourceFacade.getDatasources(names, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(listener).onResponse(captor.capture()); + assertEquals(datasources, captor.getValue()); + + } + + public void testGetAllDatasources_whenValidInput_thenSucceed() { + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + ActionListener> listener = mock(ActionListener.class); + SearchHits searchHits = getMockedSearchHits(datasources); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof SearchRequest); + SearchRequest request = (SearchRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.indices()[0]); + assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); + assertEquals(1000, request.source().size()); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); + return response; + }); + + // Run + datasourceFacade.getAllDatasources(listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(listener).onResponse(captor.capture()); + assertEquals(datasources, captor.getValue()); + } + + private SearchHits getMockedSearchHits(List datasources) { + SearchHit[] searchHitArray = datasources.stream().map(this::toBytesReference).map(this::toSearchHit).toArray(SearchHit[]::new); + + return new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + } + private GetResponse getMockedGetResponse(Datasource datasource) { GetResponse response = mock(GetResponse.class); when(response.isExists()).thenReturn(datasource != null); @@ -133,4 +216,10 @@ private BytesReference toBytesReference(Datasource datasource) { throw new RuntimeException(e); } } + + private SearchHit toSearchHit(BytesReference bytesReference) { + SearchHit searchHit = new SearchHit(Randomness.get().nextInt()); + searchHit.sourceRef(bytesReference); + return searchHit; + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java index 3632d9e9..18fc8628 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java @@ -13,7 +13,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; -import org.opensearch.common.Randomness; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -39,7 +38,7 @@ public void testParser() throws Exception { .parse( createParser(datasource.toXContent(XContentFactory.jsonBuilder(), null)), GeospatialTestHelper.randomLowerCaseString(), - new JobDocVersion(Randomness.get().nextLong(), Randomness.get().nextLong(), Randomness.get().nextLong()) + new JobDocVersion(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) ); assertTrue(datasource.equals(anotherDatasource)); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index d1a19c0c..1f547bf9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -21,11 +21,11 @@ import org.opensearch.common.Randomness; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.test.OpenSearchTestCase; -public class DatasourceTests extends OpenSearchTestCase { +public class DatasourceTests extends Ip2GeoTestCase { public void testParser() throws Exception { String id = GeospatialTestHelper.randomLowerCaseString(); @@ -38,7 +38,7 @@ public void testParser() throws Exception { datasource.getDatabase().setUpdatedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setValidForInDays(1l); - datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); + datasource.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); datasource.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); datasource.getUpdateStats().setLastSkippedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); datasource.getUpdateStats().setLastFailedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); @@ -67,7 +67,7 @@ public void testCurrentIndexName() { } public void testGetIndexNameFor() { - long updatedAt = Randomness.get().nextLong(); + long updatedAt = randomPositiveLong(); DatasourceManifest manifest = mock(DatasourceManifest.class); when(manifest.getUpdatedAt()).thenReturn(updatedAt); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 6d9430bc..3917e7c7 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -32,6 +32,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; +import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; @@ -59,7 +60,8 @@ public class GeospatialPluginTests extends OpenSearchTestCase { private final List SUPPORTED_REST_HANDLERS = List.of( new RestUploadGeoJSONAction(), new RestUploadStatsAction(), - new RestPutDatasourceHandler(clusterSettings) + new RestPutDatasourceHandler(clusterSettings), + new RestGetDatasourceHandler() ); private final Set SUPPORTED_SYSTEM_INDEX_PATTERN = Set.of(IP2GEO_DATA_INDEX_NAME_PREFIX); From 6d5ffa5b9206d482dc293775d4f431601383aab4 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 15:07:03 -0700 Subject: [PATCH 18/61] Update index option (#284) 1. Make geodata index as hidden 2. Make geodata index as read only allow delete after creation is done 3. Refresh datasource index immediately after update Signed-off-by: Heemin Kim --- .../geospatial/ip2geo/common/DatasourceFacade.java | 2 ++ .../geospatial/ip2geo/common/GeoIpDataFacade.java | 12 ++++++++++++ .../ip2geo/common/DatasourceFacadeTests.java | 2 ++ .../ip2geo/common/GeoIpDataFacadeTests.java | 11 ++++++++++- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 25d2f5af..c3be7fef 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -27,6 +27,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; @@ -67,6 +68,7 @@ public IndexResponse updateDatasource(final Datasource datasource) throws IOExce IndexRequestBuilder requestBuilder = client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME); requestBuilder.setId(datasource.getName()); requestBuilder.setOpType(DocWriteRequest.OpType.INDEX); + requestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); requestBuilder.setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); return client.index(requestBuilder.request()).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index c1fb3a70..92650020 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -66,6 +66,11 @@ public class GeoIpDataFacade { private static final String DATA_FIELD_NAME = "_data"; private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); + private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); + private static final Tuple INDEX_SETTING_READ_ONLY_ALLOW_DELETE = new Tuple<>( + "index.blocks.read_only_allow_delete", + true + ); private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final Client client; @@ -88,6 +93,7 @@ public void createIndexIfNotExists(final String indexName) { final Map indexSettings = new HashMap<>(); indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); + indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); } @@ -345,6 +351,12 @@ public void putGeoIpData(final String indexName, final String[] fields, final It } client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + client.admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Map.of(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2())) + .execute() + .actionGet(timeout); } public AcknowledgedResponse deleteIp2GeoDataIndex(final String index) { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index aacd940a..9f114751 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -33,6 +33,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; @@ -75,6 +76,7 @@ public void testUpdateDatasource_whenValidInput_thenSucceed() throws Exception { assertEquals(datasource.getName(), request.id()); assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); return null; }); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 786fc6f9..dd54bf88 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -42,6 +42,7 @@ import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.search.MultiSearchRequest; @@ -86,8 +87,10 @@ public void testCreateIndexIfNotExistsWithoutExistingIndex() { assertTrue(actionRequest instanceof CreateIndexRequest); CreateIndexRequest request = (CreateIndexRequest) actionRequest; assertEquals(index, request.index()); - assertEquals("1", request.settings().get("index.number_of_shards")); + assertEquals(1, (int) request.settings().getAsInt("index.number_of_shards", 2)); assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals(true, request.settings().getAsBoolean("index.hidden", false)); + assertEquals( "{\"dynamic\": false,\"properties\": {\"_cidr\": {\"type\": \"ip_range\",\"doc_values\": false}}}", request.mappings() @@ -178,6 +181,12 @@ public void testPutGeoIpData() throws Exception { assertEquals(index, request.indices()[0]); assertEquals(1, request.maxNumSegments()); return null; + } else if (actionRequest instanceof UpdateSettingsRequest) { + UpdateSettingsRequest request = (UpdateSettingsRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(index, request.indices()[0]); + assertEquals(true, request.settings().getAsBoolean("index.blocks.read_only_allow_delete", false)); + return null; } else { throw new RuntimeException("invalid request is called"); } From 82e3430c201891480cd6bbf46aa42170a6261c37 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 16:18:27 -0700 Subject: [PATCH 19/61] Make some fields in manifest file as mandatory (#289) Signed-off-by: Heemin Kim --- .../ip2geo/action/PutDatasourceRequest.java | 4 ++-- .../ip2geo/common/DatasourceManifest.java | 10 ++++----- .../action/PutDatasourceRequestTests.java | 21 ++++++++----------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 3d977a92..ddb60ae4 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -176,7 +176,7 @@ private void validateManifestFile(final URL url, final ActionRequestValidationEx manifest = DatasourceManifest.Builder.build(url); } catch (Exception e) { log.info("Error occurred while reading a file from {}", url, e); - errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s", url)); + errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s: %s", url, e.getMessage())); return; } @@ -188,7 +188,7 @@ private void validateManifestFile(final URL url, final ActionRequestValidationEx return; } - if (updateInterval.days() >= manifest.getValidForInDays()) { + if (manifest.getValidForInDays() != null && updateInterval.days() >= manifest.getValidForInDays()) { errors.addValidationError( String.format( Locale.ROOT, diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java index 957740a8..9a43ca6f 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -93,12 +93,12 @@ public class DatasourceManifest { } ); static { - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), URL_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DB_NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), URL_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DB_NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), SHA256_HASH_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); - PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), UPDATED_AT_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), PROVIDER_FIELD); } /** diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 01d3025c..ec8d2c9b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -23,7 +23,7 @@ public class PutDatasourceRequestTests extends Ip2GeoTestCase { - public void testValidateWithInvalidUrl() { + public void testValidate_whenInvalidUrl_thenFails() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint("invalidUrl"); @@ -33,7 +33,7 @@ public void testValidateWithInvalidUrl() { assertEquals("Invalid URL format is provided", exception.validationErrors().get(0)); } - public void testValidateWithInvalidManifestFile() { + public void testValidate_whenInvalidManifestFile_thenFails() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String domain = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); @@ -41,13 +41,10 @@ public void testValidateWithInvalidManifestFile() { request.setUpdateInterval(TimeValue.timeValueDays(1)); ActionRequestValidationException exception = request.validate(); assertEquals(1, exception.validationErrors().size()); - assertEquals( - String.format(Locale.ROOT, "Error occurred while reading a file from %s", request.getEndpoint()), - exception.validationErrors().get(0) - ); + assertTrue(exception.validationErrors().get(0).contains("Error occurred while reading a file")); } - public void testValidate() throws Exception { + public void testValidate_whenValidInput_thenSucceed() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(sampleManifestUrl()); @@ -55,7 +52,7 @@ public void testValidate() throws Exception { assertNull(request.validate()); } - public void testValidateWithZeroUpdateInterval() throws Exception { + public void testValidate_whenZeroUpdateInterval_thenFails() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(sampleManifestUrl()); @@ -72,7 +69,7 @@ public void testValidateWithZeroUpdateInterval() throws Exception { ); } - public void testValidateWithLargeUpdateInterval() throws Exception { + public void testValidate_whenLargeUpdateInterval_thenFail() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(sampleManifestUrl()); @@ -86,7 +83,7 @@ public void testValidateWithLargeUpdateInterval() throws Exception { assertTrue(exception.validationErrors().get(0).contains("should be smaller")); } - public void testValidateWithInvalidUrlInsideManifest() throws Exception { + public void testValidate_whenInvalidUrlInsideManifest_thenFail() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(sampleManifestUrlWithInvalidUrl()); @@ -100,7 +97,7 @@ public void testValidateWithInvalidUrlInsideManifest() throws Exception { assertTrue(exception.validationErrors().get(0).contains("Invalid URL format")); } - public void testValidateDatasourceNames() throws Exception { + public void testValidate_whenInvalidDatasourceNames_thenFails() throws Exception { String validDatasourceName = GeospatialTestHelper.randomLowerCaseString(); String domain = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(validDatasourceName); @@ -154,7 +151,7 @@ public void testValidateDatasourceNames() throws Exception { } } - public void testStreamInOut() throws Exception { + public void testStreamInOut_whenValidInput_thenSucceed() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String domain = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); From 932e1e164c6a0d0e271509bd0bc4799fbffa2699 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 16:31:05 -0700 Subject: [PATCH 20/61] Create datasource index explicitly (#283) Signed-off-by: Heemin Kim --- .../action/PutDatasourceTransportAction.java | 24 ++++-- .../ip2geo/common/DatasourceFacade.java | 68 +++++++++++++++- .../ip2geo/common/GeoIpDataFacade.java | 2 +- .../ip2geo/jobscheduler/Datasource.java | 1 - .../jobscheduler/DatasourceUpdateService.java | 3 +- .../geospatial/plugin/GeospatialPlugin.java | 4 +- .../resources/mappings/ip2geo_datasource.json | 80 +++++++++++++++++-- src/main/resources/mappings/ip2geo_geoip.json | 9 +++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 6 +- .../action/PutDatasourceRequestTests.java | 2 +- .../PutDatasourceTransportActionTests.java | 15 +++- .../ip2geo/common/DatasourceFacadeTests.java | 74 +++++++++++++++-- .../DatasourceExtensionTests.java | 2 +- 13 files changed, 259 insertions(+), 31 deletions(-) create mode 100644 src/main/resources/mappings/ip2geo_geoip.json diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index 43e0f76c..f1e0fda5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -8,6 +8,7 @@ package org.opensearch.geospatial.ip2geo.action; +import java.io.IOException; import java.time.Instant; import lombok.extern.log4j.Log4j2; @@ -15,6 +16,7 @@ import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.ActionFilters; @@ -71,18 +73,26 @@ public PutDatasourceTransportAction( @Override protected void doExecute(final Task task, final PutDatasourceRequest request, final ActionListener listener) { try { - Datasource datasource = Datasource.Builder.build(request); - IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) - .id(datasource.getName()) - .source(datasource.toXContent(JsonXContent.contentBuilder(), null)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .opType(DocWriteRequest.OpType.CREATE); - client.index(indexRequest, getIndexResponseListener(datasource, listener)); + StepListener createIndexStep = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(createIndexStep); + createIndexStep.whenComplete(v -> putDatasource(request, listener), exception -> listener.onFailure(exception)); } catch (Exception e) { listener.onFailure(e); } } + @VisibleForTesting + protected void putDatasource(final PutDatasourceRequest request, final ActionListener listener) + throws IOException { + Datasource datasource = Datasource.Builder.build(request); + IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) + .id(datasource.getName()) + .source(datasource.toXContent(JsonXContent.contentBuilder(), null)) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .opType(DocWriteRequest.OpType.CREATE); + client.index(indexRequest, getIndexResponseListener(datasource, listener)); + } + @VisibleForTesting protected ActionListener getIndexResponseListener( final Datasource datasource, diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index c3be7fef..99b98e57 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -8,18 +8,28 @@ package org.opensearch.geospatial.ip2geo.common; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; import org.opensearch.OpenSearchException; +import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; @@ -29,7 +39,9 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; @@ -49,12 +61,64 @@ @Log4j2 public class DatasourceFacade { private static final Integer MAX_SIZE = 1000; + private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); + private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); + private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); private final Client client; + private final ClusterService clusterService; private final ClusterSettings clusterSettings; - public DatasourceFacade(final Client client, final ClusterSettings clusterSettings) { + public DatasourceFacade(final Client client, final ClusterService clusterService) { this.client = client; - this.clusterSettings = clusterSettings; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + } + + /** + * Create a datasource index of single shard with auto expand replicas to all nodes + * + * We want the index to expand to all replica so that datasource query request can be executed locally + * for faster ingestion time. + */ + public void createIndexIfNotExists(final StepListener stepListener) { + if (clusterService.state().metadata().hasIndex(DatasourceExtension.JOB_INDEX_NAME) == true) { + stepListener.onResponse(null); + return; + } + final Map indexSettings = new HashMap<>(); + indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); + indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); + indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(DatasourceExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(indexSettings); + client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(final CreateIndexResponse createIndexResponse) { + stepListener.onResponse(null); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + log.info("index[{}] already exist", DatasourceExtension.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + stepListener.onFailure(e); + } + }); + } + + private String getIndexMapping() { + try { + try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 92650020..dd498d1b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -115,7 +115,7 @@ public void createIndexIfNotExists(final String indexName) { */ private String getIndexMapping() { try { - try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { + try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_geoip.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 7c7eae55..d56f5184 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -182,7 +182,6 @@ public class Datasource implements Writeable, ScheduledJobParameter { PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), Database.PARSER, DATABASE_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), UpdateStats.PARSER, UPDATE_STATS_FIELD); - } @VisibleForTesting diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 07ea55c7..0f27993d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -210,7 +210,8 @@ private String setupIndex(final DatasourceManifest manifest, final Datasource da * @return */ private boolean shouldUpdate(final Datasource datasource, final DatasourceManifest manifest) { - if (datasource.getDatabase().getUpdatedAt().toEpochMilli() > manifest.getUpdatedAt()) { + if (datasource.getDatabase().getUpdatedAt() != null + && datasource.getDatabase().getUpdatedAt().toEpochMilli() > manifest.getUpdatedAt()) { return false; } diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 085fab98..37104c69 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -94,7 +94,7 @@ public Map getProcessors(Processor.Parameters paramet new Ip2GeoProcessor.Factory( parameters.client, parameters.ingestService, - new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService().getClusterSettings()), + new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService()), new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client) ) ) @@ -128,7 +128,7 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { GeoIpDataFacade geoIpDataFacade = new GeoIpDataFacade(clusterService, client); - DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService.getClusterSettings()); + DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService); DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService( clusterService, client, diff --git a/src/main/resources/mappings/ip2geo_datasource.json b/src/main/resources/mappings/ip2geo_datasource.json index 3179ef0d..3f3d5aa1 100644 --- a/src/main/resources/mappings/ip2geo_datasource.json +++ b/src/main/resources/mappings/ip2geo_datasource.json @@ -1,9 +1,77 @@ { - "dynamic": false, - "properties": { - "_cidr": { - "type": "ip_range", - "doc_values": false + "properties" : { + "database" : { + "properties" : { + "fields" : { + "type" : "text" + }, + "sha256_hash" : { + "type" : "text" + }, + "provider" : { + "type" : "text" + }, + "updated_at_in_epoch_millis" : { + "type" : "long" + }, + "valid_for_in_days" : { + "type" : "long" + } + } + }, + "enabled_time" : { + "type" : "long" + }, + "endpoint" : { + "type" : "text" + }, + "name" : { + "type" : "text" + }, + "indices" : { + "type" : "text" + }, + "last_update_time" : { + "type" : "long" + }, + "schedule" : { + "properties" : { + "interval" : { + "properties" : { + "period" : { + "type" : "long" + }, + "start_time" : { + "type" : "long" + }, + "unit" : { + "type" : "text" + } + } + } + } + }, + "state" : { + "type" : "text" + }, + "update_enabled" : { + "type" : "boolean" + }, + "update_stats" : { + "properties" : { + "last_failed_at_in_epoch_millis" : { + "type" : "long" + }, + "last_processing_time_in_millis" : { + "type" : "long" + }, + "last_skipped_at_in_epoch_millis" : { + "type" : "long" + }, + "last_succeeded_at_in_epoch_millis" : { + "type" : "long" + } + } } } -} +} \ No newline at end of file diff --git a/src/main/resources/mappings/ip2geo_geoip.json b/src/main/resources/mappings/ip2geo_geoip.json new file mode 100644 index 00000000..3179ef0d --- /dev/null +++ b/src/main/resources/mappings/ip2geo_geoip.json @@ -0,0 +1,9 @@ +{ + "dynamic": false, + "properties": { + "_cidr": { + "type": "ip_range", + "doc_values": false + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 48451a9c..7163c7cd 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -233,7 +233,11 @@ public void doE Request request, ActionListener listener ) { - listener.onResponse((Response) executeVerifier.get().apply(action, request)); + try { + listener.onResponse((Response) executeVerifier.get().apply(action, request)); + } catch (Exception e) { + listener.onFailure(e); + } } /** diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index ec8d2c9b..2334cf0e 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -156,7 +156,7 @@ public void testStreamInOut_whenValidInput_thenSucceed() throws Exception { String domain = GeospatialTestHelper.randomLowerCaseString(); PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); - request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(30) + 1)); + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(29) + 1)); // Run BytesStreamOutput output = new BytesStreamOutput(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index ff542418..62613581 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -14,9 +14,11 @@ import static org.mockito.Mockito.verify; import org.junit.Before; +import org.mockito.ArgumentCaptor; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -44,7 +46,7 @@ public void init() { ); } - public void testDoExecute() throws Exception { + public void testDoExecute_whenValidInput_thenSucceed() throws Exception { Task task = mock(Task.class); PutDatasourceRequest request = new PutDatasourceRequest("test"); request.setEndpoint(sampleManifestUrl()); @@ -59,7 +61,18 @@ public void testDoExecute() throws Exception { assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); return null; }); + + // Run action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor captor = ArgumentCaptor.forClass(StepListener.class); + verify(datasourceFacade).createIndexIfNotExists(captor.capture()); + + // Run + captor.getValue().onResponse(null); + + // Verify verify(verifyingClient).index(any(IndexRequest.class), any(ActionListener.class)); verify(listener).onResponse(new AcknowledgedResponse(true)); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 9f114751..bac7f7dd 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -17,14 +17,16 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import org.apache.lucene.search.TotalHits; import org.junit.Before; import org.mockito.ArgumentCaptor; +import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; @@ -36,8 +38,6 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -54,10 +54,70 @@ public class DatasourceFacadeTests extends Ip2GeoTestCase { @Before public void init() { - datasourceFacade = new DatasourceFacade( - verifyingClient, - new ClusterSettings(Settings.EMPTY, new HashSet<>(Ip2GeoSettings.settings())) + datasourceFacade = new DatasourceFacade(verifyingClient, clusterService); + } + + public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCalled() { + when(metadata.hasIndex(DatasourceExtension.JOB_INDEX_NAME)).thenReturn(true); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); + + // Run + StepListener stepListener = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsCalled() { + when(metadata.hasIndex(DatasourceExtension.JOB_INDEX_NAME)).thenReturn(false); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof CreateIndexRequest); + CreateIndexRequest request = (CreateIndexRequest) actionRequest; + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals("1", request.settings().get("index.number_of_shards")); + assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals("true", request.settings().get("index.hidden")); + assertNotNull(request.mappings()); + return null; + }); + + // Run + StepListener stepListener = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenIndexCreatedAlready_thenExceptionIsIgnored() { + when(metadata.hasIndex(DatasourceExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier( + (actionResponse, actionRequest) -> { throw new ResourceAlreadyExistsException(DatasourceExtension.JOB_INDEX_NAME); } ); + + // Run + StepListener stepListener = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testCreateIndexIfNotExists_whenExceptionIsThrown_thenExceptionIsThrown() { + when(metadata.hasIndex(DatasourceExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException(); }); + + // Run + StepListener stepListener = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(stepListener); + + // Verify stepListener is called + expectThrows(RuntimeException.class, () -> stepListener.result()); } public void testUpdateDatasource_whenValidInput_thenSucceed() throws Exception { @@ -106,7 +166,7 @@ public void testGetDatasource_whenExistWithListener_thenListenerIsCalledWithData verify(listener).onResponse(eq(datasource)); } - public void testGetDatasource_whenExistWithListener_thenListenerIsCalledWithNull() { + public void testGetDatasource_whenNotExistWithListener_thenListenerIsCalledWithNull() { Datasource datasource = setupClientForGetRequest(false, null); ActionListener listener = mock(ActionListener.class); datasourceFacade.getDatasource(datasource.getName(), listener); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java index 18fc8628..75dea954 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java @@ -20,7 +20,7 @@ import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; public class DatasourceExtensionTests extends Ip2GeoTestCase { - public void testBasic() throws Exception { + public void testBasic() { DatasourceExtension extension = new DatasourceExtension(); assertEquals("scheduler_geospatial_ip2geo_datasource", extension.getJobType()); assertEquals(JOB_INDEX_NAME, extension.getJobIndex()); From fde10e18067baffddc129e71a0140bf22f101fa2 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 5 May 2023 10:49:20 -0700 Subject: [PATCH 21/61] Add wrapper class of job scheduler lock service (#290) Signed-off-by: Heemin Kim --- .../ip2geo/common/Ip2GeoLockService.java | 98 +++++++++++++++++++ .../ip2geo/common/Ip2GeoLockServiceTests.java | 46 +++++++++ 2 files changed, 144 insertions(+) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java new file mode 100644 index 00000000..cd41251a --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension.JOB_INDEX_NAME; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.opensearch.action.ActionListener; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.utils.LockService; + +/** + * A wrapper of job scheduler's lock service for datasource + */ +public class Ip2GeoLockService { + private final ClusterService clusterService; + private final Client client; + private final LockService lockService; + + /** + * Constructor + * + * @param clusterService the cluster service + * @param client the client + */ + @Inject + public Ip2GeoLockService(final ClusterService clusterService, final Client client) { + this.clusterService = clusterService; + this.client = client; + this.lockService = new LockService(client, clusterService); + } + + /** + * Wrapper method of LockService#acquireLockWithId + * + * Datasource use its name as doc id in job scheduler. Therefore, we can use datasource name to acquire + * a lock on a datasource. + * + * @param datasourceName datasourceName to acquire lock on + * @param lockDurationSeconds the lock duration in seconds + * @param listener the listener + */ + public void acquireLock(final String datasourceName, final Long lockDurationSeconds, final ActionListener listener) { + lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, datasourceName, listener); + } + + /** + * Wrapper method of LockService#release + * + * @param lockModel the lock model + * @param listener the listener + */ + public void releaseLock(final LockModel lockModel, final ActionListener listener) { + lockService.release(lockModel, listener); + } + + /** + * Synchronous method of LockService#renewLock + * + * @param lockModel lock to renew + * @param timeout timeout in milliseconds precise + * @return renewed lock if renew succeed and null otherwise + */ + public LockModel renewLock(final LockModel lockModel, final TimeValue timeout) { + AtomicReference lockReference = new AtomicReference(); + CountDownLatch countDownLatch = new CountDownLatch(1); + lockService.renewLock(lockModel, new ActionListener<>() { + @Override + public void onResponse(final LockModel lockModel) { + lockReference.set(lockModel); + countDownLatch.countDown(); + } + + @Override + public void onFailure(final Exception e) { + lockReference.set(null); + countDownLatch.countDown(); + } + }); + + try { + countDownLatch.await(timeout.getMillis(), TimeUnit.MILLISECONDS); + return lockReference.get(); + } catch (InterruptedException e) { + return null; + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java new file mode 100644 index 00000000..a746f314 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.mockito.Mockito.mock; + +import java.time.Instant; + +import org.junit.Before; +import org.opensearch.action.ActionListener; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; + +public class Ip2GeoLockServiceTests extends Ip2GeoTestCase { + private Ip2GeoLockService ip2GeoLockService; + + @Before + public void init() { + ip2GeoLockService = new Ip2GeoLockService(clusterService, client); + } + + public void testAcquireLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + ip2GeoLockService.acquireLock(GeospatialTestHelper.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); + } + + public void testReleaseLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + ip2GeoLockService.releaseLock(null, mock(ActionListener.class)); + } + + public void testRenewLock_whenCalled_thenNotBlocked() { + long timeoutInMillis = 10000; + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertNull(ip2GeoLockService.renewLock(null, TimeValue.timeValueMillis(timeoutInMillis))); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } +} From d3f39b0496dba44e3555b6dd7827fc86fadcf49e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 5 May 2023 17:30:01 -0700 Subject: [PATCH 22/61] Remove all unused client attributes (#293) Signed-off-by: Heemin Kim --- .../action/PutDatasourceTransportAction.java | 19 ++----- .../ip2geo/common/DatasourceFacade.java | 33 +++++++++--- .../ip2geo/jobscheduler/DatasourceRunner.java | 4 -- .../jobscheduler/DatasourceUpdateService.java | 4 -- .../ip2geo/processor/Ip2GeoProcessor.java | 20 ++----- .../geospatial/plugin/GeospatialPlugin.java | 11 +--- .../PutDatasourceTransportActionTests.java | 52 ++++++++----------- .../ip2geo/common/DatasourceFacadeTests.java | 22 ++++++++ .../jobscheduler/DatasourceRunnerTests.java | 3 +- .../DatasourceUpdateServiceTests.java | 2 +- .../processor/Ip2GeoProcessorTests.java | 2 +- 11 files changed, 84 insertions(+), 88 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index f1e0fda5..40882492 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -15,22 +15,16 @@ import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; -import org.opensearch.action.DocWriteRequest; import org.opensearch.action.StepListener; -import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; -import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.tasks.Task; @@ -42,7 +36,6 @@ */ @Log4j2 public class PutDatasourceTransportAction extends HandledTransportAction { - private final Client client; private final ThreadPool threadPool; private final DatasourceFacade datasourceFacade; private final DatasourceUpdateService datasourceUpdateService; @@ -51,20 +44,19 @@ public class PutDatasourceTransportAction extends HandledTransportAction listener) throws IOException { Datasource datasource = Datasource.Builder.build(request); - IndexRequest indexRequest = new IndexRequest().index(DatasourceExtension.JOB_INDEX_NAME) - .id(datasource.getName()) - .source(datasource.toXContent(JsonXContent.contentBuilder(), null)) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .opType(DocWriteRequest.OpType.CREATE); - client.index(indexRequest, getIndexResponseListener(datasource, listener)); + datasourceFacade.putDatasource(datasource, getIndexResponseListener(datasource, listener)); } @VisibleForTesting diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 99b98e57..71d8122b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -21,6 +21,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import javax.swing.*; + import lombok.extern.log4j.Log4j2; import org.opensearch.OpenSearchException; @@ -34,7 +36,6 @@ import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; import org.opensearch.action.get.MultiGetResponse; -import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; @@ -129,12 +130,30 @@ private String getIndexMapping() { */ public IndexResponse updateDatasource(final Datasource datasource) throws IOException { datasource.setLastUpdateTime(Instant.now()); - IndexRequestBuilder requestBuilder = client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME); - requestBuilder.setId(datasource.getName()); - requestBuilder.setOpType(DocWriteRequest.OpType.INDEX); - requestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - requestBuilder.setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); - return client.index(requestBuilder.request()).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + return client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) + .setId(datasource.getName()) + .setOpType(DocWriteRequest.OpType.INDEX) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + } + + /** + * Put datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * + * @param datasource the datasource + * @param listener the listener + * @throws IOException exception + */ + public void putDatasource(final Datasource datasource, final ActionListener listener) throws IOException { + datasource.setLastUpdateTime(Instant.now()); + client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) + .setId(datasource.getName()) + .setOpType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .execute(listener); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index bc78fd0b..3e902c61 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -14,7 +14,6 @@ import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; -import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -53,7 +52,6 @@ public static DatasourceRunner getJobRunnerInstance() { } private ClusterService clusterService; - private Client client; private DatasourceUpdateService datasourceUpdateService; private Ip2GeoExecutor ip2GeoExecutor; private DatasourceFacade datasourceFacade; @@ -68,13 +66,11 @@ private DatasourceRunner() { */ public void initialize( final ClusterService clusterService, - final Client client, final DatasourceUpdateService datasourceUpdateService, final Ip2GeoExecutor ip2GeoExecutor, final DatasourceFacade datasourceFacade ) { this.clusterService = clusterService; - this.client = client; this.datasourceUpdateService = datasourceUpdateService; this.ip2GeoExecutor = ip2GeoExecutor; this.datasourceFacade = datasourceFacade; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 0f27993d..98423a7a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -22,7 +22,6 @@ import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.opensearch.OpenSearchException; -import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -35,19 +34,16 @@ public class DatasourceUpdateService { private final ClusterService clusterService; private final ClusterSettings clusterSettings; - private final Client client; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; public DatasourceUpdateService( final ClusterService clusterService, - final Client client, final DatasourceFacade datasourceFacade, final GeoIpDataFacade geoIpDataFacade ) { this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); - this.client = client; this.datasourceFacade = datasourceFacade; this.geoIpDataFacade = geoIpDataFacade; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index a0f8974d..ab810fb1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -26,7 +26,6 @@ import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; -import org.opensearch.client.Client; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -52,7 +51,6 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final Set properties; private final boolean ignoreMissing; private final boolean firstOnly; - private final Client client; private final ClusterSettings clusterSettings; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; @@ -72,8 +70,9 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param properties the properties * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array - * @param client the client * @param clusterSettings the cluster settings + * @param datasourceFacade the datasource facade + * @param geoIpDataFacade the geoip data facade */ public Ip2GeoProcessor( final String tag, @@ -84,7 +83,6 @@ public Ip2GeoProcessor( final Set properties, final boolean ignoreMissing, final boolean firstOnly, - final Client client, final ClusterSettings clusterSettings, final DatasourceFacade datasourceFacade, final GeoIpDataFacade geoIpDataFacade @@ -96,7 +94,6 @@ public Ip2GeoProcessor( this.properties = properties; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; - this.client = client; this.clusterSettings = clusterSettings; this.datasourceFacade = datasourceFacade; this.geoIpDataFacade = geoIpDataFacade; @@ -318,7 +315,6 @@ public String getType() { * Ip2Geo processor factory */ public static final class Factory implements Processor.Factory { - private final Client client; private final IngestService ingestService; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; @@ -326,16 +322,11 @@ public static final class Factory implements Processor.Factory { /** * Default constructor * - * @param client the client * @param ingestService the ingest service + * @param datasourceFacade the datasource facade + * @param geoIpDataFacade the geoip data facade */ - public Factory( - final Client client, - final IngestService ingestService, - final DatasourceFacade datasourceFacade, - final GeoIpDataFacade geoIpDataFacade - ) { - this.client = client; + public Factory(final IngestService ingestService, final DatasourceFacade datasourceFacade, final GeoIpDataFacade geoIpDataFacade) { this.ingestService = ingestService; this.datasourceFacade = datasourceFacade; this.geoIpDataFacade = geoIpDataFacade; @@ -380,7 +371,6 @@ public Ip2GeoProcessor create( propertyNames == null ? null : new HashSet<>(propertyNames), ignoreMissing, firstOnly, - client, ingestService.getClusterService().getClusterSettings(), datasourceFacade, geoIpDataFacade diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 37104c69..e52939b0 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -92,7 +92,6 @@ public Map getProcessors(Processor.Parameters paramet .put( Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory( - parameters.client, parameters.ingestService, new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService()), new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client) @@ -129,19 +128,13 @@ public Collection createComponents( ) { GeoIpDataFacade geoIpDataFacade = new GeoIpDataFacade(clusterService, client); DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService); - DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService( - clusterService, - client, - datasourceFacade, - geoIpDataFacade - ); + DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); /** * We don't need to return datasource runner because it is used only by job scheduler and job scheduler * does not use DI but it calls DatasourceExtension#getJobRunner to get DatasourceRunner instance. */ - DatasourceRunner.getJobRunnerInstance() - .initialize(clusterService, client, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + DatasourceRunner.getJobRunnerInstance().initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); return List.of(UploadStats.getInstance(), datasourceUpdateService, datasourceFacade, ip2GeoExecutor, geoIpDataFacade); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 62613581..3d255cd2 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -13,21 +13,19 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import lombok.SneakyThrows; + import org.junit.Before; import org.mockito.ArgumentCaptor; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; -import org.opensearch.action.DocWriteRequest; import org.opensearch.action.StepListener; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.unit.TimeValue; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.tasks.Task; @@ -36,31 +34,16 @@ public class PutDatasourceTransportActionTests extends Ip2GeoTestCase { @Before public void init() { - action = new PutDatasourceTransportAction( - transportService, - actionFilters, - verifyingClient, - threadPool, - datasourceFacade, - datasourceUpdateService - ); + action = new PutDatasourceTransportAction(transportService, actionFilters, threadPool, datasourceFacade, datasourceUpdateService); } - public void testDoExecute_whenValidInput_thenSucceed() throws Exception { + @SneakyThrows + public void testDoExecute_whenValidInput_thenSucceed() { Task task = mock(Task.class); - PutDatasourceRequest request = new PutDatasourceRequest("test"); + PutDatasourceRequest request = new PutDatasourceRequest(GeospatialTestHelper.randomLowerCaseString()); request.setEndpoint(sampleManifestUrl()); request.setUpdateInterval(TimeValue.timeValueDays(1)); ActionListener listener = mock(ActionListener.class); - verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { - assertTrue(actionRequest instanceof IndexRequest); - IndexRequest indexRequest = (IndexRequest) actionRequest; - assertEquals(DatasourceExtension.JOB_INDEX_NAME, indexRequest.index()); - assertEquals(request.getName(), indexRequest.id()); - assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); - assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); - return null; - }); // Run action.doExecute(task, request, listener); @@ -71,13 +54,21 @@ public void testDoExecute_whenValidInput_thenSucceed() throws Exception { // Run captor.getValue().onResponse(null); - // Verify - verify(verifyingClient).index(any(IndexRequest.class), any(ActionListener.class)); + ArgumentCaptor datasourceCaptor = ArgumentCaptor.forClass(Datasource.class); + ArgumentCaptor actionListenerCaptor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).putDatasource(datasourceCaptor.capture(), actionListenerCaptor.capture()); + assertEquals(request.getName(), datasourceCaptor.getValue().getName()); + assertEquals(request.getEndpoint(), datasourceCaptor.getValue().getEndpoint()); + assertEquals(request.getUpdateInterval().days(), datasourceCaptor.getValue().getSchedule().getInterval()); + + // Run next listener.onResponse + actionListenerCaptor.getValue().onResponse(null); + // Verify verify(listener).onResponse(new AcknowledgedResponse(true)); } - public void testIndexResponseListenerFailure() { + public void testGetIndexResponseListener_whenVersionConflict_thenFailure() { Datasource datasource = new Datasource(); ActionListener listener = mock(ActionListener.class); action.getIndexResponseListener(datasource, listener) @@ -91,7 +82,8 @@ public void testIndexResponseListenerFailure() { verify(listener).onFailure(any(ResourceAlreadyExistsException.class)); } - public void testCreateDatasourceInvalidState() throws Exception { + @SneakyThrows + public void testCreateDatasource_whenInvalidState_thenUpdateStateAsFailed() { Datasource datasource = new Datasource(); datasource.setState(randomStateExcept(DatasourceState.CREATING)); datasource.getUpdateStats().setLastFailedAt(null); @@ -105,7 +97,8 @@ public void testCreateDatasourceInvalidState() throws Exception { verify(datasourceFacade).updateDatasource(datasource); } - public void testCreateDatasourceWithException() throws Exception { + @SneakyThrows + public void testCreateDatasource_whenExceptionHappens_thenUpdateStateAsFailed() { Datasource datasource = new Datasource(); doThrow(new RuntimeException()).when(datasourceUpdateService).updateOrCreateGeoIpData(datasource); @@ -118,7 +111,8 @@ public void testCreateDatasourceWithException() throws Exception { verify(datasourceFacade).updateDatasource(datasource); } - public void testCreateDatasource() throws Exception { + @SneakyThrows + public void testCreateDatasource_whenValidInput_thenUpdateStateAsCreating() { Datasource datasource = new Datasource(); // Run diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index bac7f7dd..25f37b74 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -19,6 +19,8 @@ import java.util.Arrays; import java.util.List; +import lombok.SneakyThrows; + import org.apache.lucene.search.TotalHits; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -144,6 +146,26 @@ public void testUpdateDatasource_whenValidInput_thenSucceed() throws Exception { assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); } + @SneakyThrows + public void testPutDatasource_whenValidInpu_thenSucceed() { + Datasource datasource = randomDatasource(); + Instant previousTime = Instant.now().minusMillis(1); + datasource.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest indexRequest = (IndexRequest) actionRequest; + assertEquals(DatasourceExtension.JOB_INDEX_NAME, indexRequest.index()); + assertEquals(datasource.getName(), indexRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); + assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); + return null; + }); + + datasourceFacade.putDatasource(datasource, mock(ActionListener.class)); + assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); + } + public void testGetDatasource_whenException_thenNull() throws Exception { Datasource datasource = setupClientForGetRequest(true, new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME)); assertNull(datasourceFacade.getDatasource(datasource.getName())); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 34e4ab3b..f32d0ab4 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -29,8 +29,7 @@ public class DatasourceRunnerTests extends Ip2GeoTestCase { @Before public void init() { - DatasourceRunner.getJobRunnerInstance() - .initialize(clusterService, client, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + DatasourceRunner.getJobRunnerInstance().initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); } public void testRunJobInvalidClass() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 8d78f44b..91dc6c09 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -35,7 +35,7 @@ public class DatasourceUpdateServiceTests extends Ip2GeoTestCase { @Before public void init() { - datasourceUpdateService = new DatasourceUpdateService(clusterService, client, datasourceFacade, geoIpDataFacade); + datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); } public void testUpdateDatasourceSkip() throws Exception { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 6028b029..0e1cf1c2 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -47,7 +47,7 @@ public class Ip2GeoProcessorTests extends Ip2GeoTestCase { @Before public void init() { - factory = new Ip2GeoProcessor.Factory(client, ingestService, datasourceFacade, geoIpDataFacade); + factory = new Ip2GeoProcessor.Factory(ingestService, datasourceFacade, geoIpDataFacade); } public void testCreateWithNoDatasource() { From 056159b7c18f91efb322ba3d4be87addbd13a9f5 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 8 May 2023 15:56:58 -0700 Subject: [PATCH 23/61] Update copyright header (#298) Signed-off-by: Heemin Kim --- .../geospatial/ip2geo/action/GetDatasourceAction.java | 5 +---- .../geospatial/ip2geo/action/GetDatasourceRequest.java | 5 +---- .../geospatial/ip2geo/action/GetDatasourceResponse.java | 5 +---- .../ip2geo/action/GetDatasourceTransportAction.java | 5 +---- .../geospatial/ip2geo/action/PutDatasourceAction.java | 5 +---- .../geospatial/ip2geo/action/PutDatasourceRequest.java | 5 +---- .../ip2geo/action/PutDatasourceTransportAction.java | 5 +---- .../geospatial/ip2geo/action/RestGetDatasourceHandler.java | 5 +---- .../geospatial/ip2geo/action/RestPutDatasourceHandler.java | 5 +---- .../geospatial/ip2geo/common/DatasourceFacade.java | 5 +---- .../geospatial/ip2geo/common/DatasourceManifest.java | 5 +---- .../opensearch/geospatial/ip2geo/common/DatasourceState.java | 5 +---- .../opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java | 5 +---- .../opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java | 5 +---- .../opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java | 5 +---- .../geospatial/ip2geo/jobscheduler/Datasource.java | 5 +---- .../geospatial/ip2geo/jobscheduler/DatasourceExtension.java | 5 +---- .../geospatial/ip2geo/jobscheduler/DatasourceRunner.java | 5 +---- .../ip2geo/jobscheduler/DatasourceUpdateService.java | 5 +---- .../geospatial/ip2geo/processor/Ip2GeoProcessor.java | 5 +---- .../org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java | 5 +---- .../geospatial/ip2geo/action/GetDatasourceRequestTests.java | 5 +---- .../geospatial/ip2geo/action/GetDatasourceResponseTests.java | 5 +---- .../ip2geo/action/GetDatasourceTransportActionTests.java | 5 +---- .../geospatial/ip2geo/action/PutDatasourceRequestTests.java | 5 +---- .../ip2geo/action/PutDatasourceTransportActionTests.java | 5 +---- .../ip2geo/action/RestGetDatasourceHandlerTests.java | 5 +---- .../ip2geo/action/RestPutDatasourceHandlerTests.java | 5 +---- .../geospatial/ip2geo/common/DatasourceFacadeTests.java | 5 +---- .../geospatial/ip2geo/common/GeoIpDataFacadeTests.java | 5 +---- .../geospatial/ip2geo/common/Ip2GeoSettingsTests.java | 5 +---- .../ip2geo/jobscheduler/DatasourceExtensionTests.java | 5 +---- .../ip2geo/jobscheduler/DatasourceRunnerTests.java | 5 +---- .../geospatial/ip2geo/jobscheduler/DatasourceTests.java | 5 +---- .../ip2geo/jobscheduler/DatasourceUpdateServiceTests.java | 5 +---- .../geospatial/ip2geo/processor/Ip2GeoProcessorTests.java | 5 +---- 36 files changed, 36 insertions(+), 144 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java index 09f90934..7771fd30 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java index a3de669d..aa32c8f7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java index e4915fa8..8e403774 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java index 4bc69949..905d8165 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java index 8db6278f..2554b4f5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceAction.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index ddb60ae4..3347a243 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index 40882492..7ae05cdd 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java index 0dc50530..2c9379b5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandler.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java index d35b7751..0fac4ec9 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 71d8122b..bbc5a787 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java index 9a43ca6f..fca40213 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java index 85b0aecf..3fbb064c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceState.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index dd498d1b..d06f2de5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java index c2127482..f6230e04 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoExecutor.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index f8348435..db565dd8 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index d56f5184..2604490f 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java index 6def5295..edea50bd 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index 3e902c61..0ad8438b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 98423a7a..2a6155d3 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index ab810fb1..99a29865 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.processor; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 7163c7cd..ee57b5ba 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java index 41dc2d97..09f1b67c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java index e07feaaa..cca230c7 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java index bf8de2eb..3dd4ddf5 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 2334cf0e..dc150b18 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 3d255cd2..0c7bdf6d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java index 5d4bef42..e1177da6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestGetDatasourceHandlerTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index 521f3420..97c3cb7d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.action; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 25f37b74..baacde79 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index dd54bf88..d86c2795 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java index 451d0f8e..ffd40f5f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.common; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java index 75dea954..0ea22117 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtensionTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index f32d0ab4..274689ac 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 1f547bf9..679a043d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 91dc6c09..f7ca7d2e 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.jobscheduler; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 0e1cf1c2..31451ea8 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.geospatial.ip2geo.processor; From 462249203a496ba34465dfcda90cff5c94495405 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 8 May 2023 20:34:56 -0700 Subject: [PATCH 24/61] Run system index handling code with stashed thread context (#297) Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceFacade.java | 75 ++++++++------ .../ip2geo/common/GeoIpDataFacade.java | 98 +++++++++++-------- .../shared/StashedThreadContext.java | 41 ++++++++ 3 files changed, 142 insertions(+), 72 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/shared/StashedThreadContext.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index bbc5a787..6e03a937 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -49,6 +49,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -89,7 +90,7 @@ public void createIndexIfNotExists(final StepListener stepListener) { indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(DatasourceExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) .settings(indexSettings); - client.admin().indices().create(createIndexRequest, new ActionListener<>() { + StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { @Override public void onResponse(final CreateIndexResponse createIndexResponse) { stepListener.onResponse(null); @@ -104,7 +105,7 @@ public void onFailure(final Exception e) { } stepListener.onFailure(e); } - }); + })); } private String getIndexMapping() { @@ -123,17 +124,22 @@ private String getIndexMapping() { * Update datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} * @param datasource the datasource * @return index response - * @throws IOException exception */ - public IndexResponse updateDatasource(final Datasource datasource) throws IOException { + public IndexResponse updateDatasource(final Datasource datasource) { datasource.setLastUpdateTime(Instant.now()); - return client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) - .setId(datasource.getName()) - .setOpType(DocWriteRequest.OpType.INDEX) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .execute() - .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + return StashedThreadContext.run(client, () -> { + try { + return client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) + .setId(datasource.getName()) + .setOpType(DocWriteRequest.OpType.INDEX) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } /** @@ -141,16 +147,21 @@ public IndexResponse updateDatasource(final Datasource datasource) throws IOExce * * @param datasource the datasource * @param listener the listener - * @throws IOException exception */ - public void putDatasource(final Datasource datasource, final ActionListener listener) throws IOException { + public void putDatasource(final Datasource datasource, final ActionListener listener) { datasource.setLastUpdateTime(Instant.now()); - client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) - .setId(datasource.getName()) - .setOpType(DocWriteRequest.OpType.CREATE) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .execute(listener); + StashedThreadContext.run(client, () -> { + try { + client.prepareIndex(DatasourceExtension.JOB_INDEX_NAME) + .setId(datasource.getName()) + .setOpType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setSource(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .execute(listener); + } catch (IOException e) { + new RuntimeException(e); + } + }); } /** @@ -163,7 +174,7 @@ public Datasource getDatasource(final String name) throws IOException { GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); GetResponse response; try { - response = client.get(request).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + response = StashedThreadContext.run(client, () -> client.get(request).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT))); if (response.isExists() == false) { log.error("Datasource[{}] does not exist in an index[{}]", name, DatasourceExtension.JOB_INDEX_NAME); return null; @@ -188,7 +199,7 @@ public Datasource getDatasource(final String name) throws IOException { */ public void getDatasource(final String name, final ActionListener actionListener) { GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); - client.get(request, new ActionListener() { + StashedThreadContext.run(client, () -> client.get(request, new ActionListener<>() { @Override public void onResponse(final GetResponse response) { if (response.isExists() == false) { @@ -212,7 +223,7 @@ public void onResponse(final GetResponse response) { public void onFailure(final Exception e) { actionListener.onFailure(e); } - }); + })); } /** @@ -221,9 +232,12 @@ public void onFailure(final Exception e) { * @param actionListener the action listener */ public void getDatasources(final String[] names, final ActionListener> actionListener) { - client.prepareMultiGet() - .add(DatasourceExtension.JOB_INDEX_NAME, names) - .execute(createGetDataSourceQueryActionLister(MultiGetResponse.class, actionListener)); + StashedThreadContext.run( + client, + () -> client.prepareMultiGet() + .add(DatasourceExtension.JOB_INDEX_NAME, names) + .execute(createGetDataSourceQueryActionLister(MultiGetResponse.class, actionListener)) + ); } /** @@ -231,10 +245,13 @@ public void getDatasources(final String[] names, final ActionListener> actionListener) { - client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(MAX_SIZE) - .execute(createGetDataSourceQueryActionLister(SearchResponse.class, actionListener)); + StashedThreadContext.run( + client, + () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(MAX_SIZE) + .execute(createGetDataSourceQueryActionLister(SearchResponse.class, actionListener)) + ); } private ActionListener createGetDataSourceQueryActionLister( diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index d06f2de5..bf436c33 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -52,6 +52,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.query.QueryBuilders; /** @@ -92,7 +93,10 @@ public void createIndexIfNotExists(final String indexName) { indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); - client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + StashedThreadContext.run( + client, + () -> client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) + ); } /** @@ -207,31 +211,34 @@ public String createDocument(final String[] fields, final String[] values) { * @param actionListener action listener */ public void getGeoIpData(final String indexName, final String ip, final ActionListener> actionListener) { - client.prepareSearch(indexName) - .setSize(1) - .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) - .setPreference("_local") - .setRequestCache(true) - .execute(new ActionListener<>() { - @Override - public void onResponse(final SearchResponse searchResponse) { - if (searchResponse.getHits().getHits().length == 0) { - actionListener.onResponse(Collections.emptyMap()); - } else { - Map geoIpData = (Map) XContentHelper.convertToMap( - searchResponse.getHits().getAt(0).getSourceRef(), - false, - XContentType.JSON - ).v2().get(DATA_FIELD_NAME); - actionListener.onResponse(geoIpData); + StashedThreadContext.run( + client, + () -> client.prepareSearch(indexName) + .setSize(1) + .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) + .setPreference("_local") + .setRequestCache(true) + .execute(new ActionListener<>() { + @Override + public void onResponse(final SearchResponse searchResponse) { + if (searchResponse.getHits().getHits().length == 0) { + actionListener.onResponse(Collections.emptyMap()); + } else { + Map geoIpData = (Map) XContentHelper.convertToMap( + searchResponse.getHits().getAt(0).getSourceRef(), + false, + XContentType.JSON + ).v2().get(DATA_FIELD_NAME); + actionListener.onResponse(geoIpData); + } } - } - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - }); + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); + } + }) + ); } /** @@ -281,7 +288,7 @@ public void getGeoIpData( return; } - mRequestBuilder.execute(new ActionListener<>() { + StashedThreadContext.run(client, () -> mRequestBuilder.execute(new ActionListener<>() { @Override public void onResponse(final MultiSearchResponse items) { for (int i = 0; i < ipsToSearch.size(); i++) { @@ -315,7 +322,7 @@ public void onResponse(final MultiSearchResponse items) { public void onFailure(final Exception e) { actionListener.onFailure(e); } - }); + })); } /** @@ -328,14 +335,14 @@ public void onFailure(final Exception e) { */ public void putGeoIpData(final String indexName, final String[] fields, final Iterator iterator, final int bulkSize) { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); - BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); while (iterator.hasNext()) { CSVRecord record = iterator.next(); String document = createDocument(fields, record.values()); IndexRequest request = Requests.indexRequest(indexName).source(document, XContentType.JSON); bulkRequest.add(request); if (iterator.hasNext() == false || bulkRequest.requests().size() == bulkSize) { - BulkResponse response = client.bulk(bulkRequest).actionGet(timeout); + BulkResponse response = StashedThreadContext.run(client, () -> client.bulk(bulkRequest).actionGet(timeout)); if (response.hasFailures()) { throw new OpenSearchException( "error occurred while ingesting GeoIP data in {} with an error {}", @@ -343,17 +350,19 @@ public void putGeoIpData(final String indexName, final String[] fields, final It response.buildFailureMessage() ); } - bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + bulkRequest.requests().clear(); } } - client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); - client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); - client.admin() - .indices() - .prepareUpdateSettings(indexName) - .setSettings(Map.of(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2())) - .execute() - .actionGet(timeout); + StashedThreadContext.run(client, () -> { + client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); + client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + client.admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Map.of(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2())) + .execute() + .actionGet(timeout); + }); } public AcknowledgedResponse deleteIp2GeoDataIndex(final String index) { @@ -364,11 +373,14 @@ public AcknowledgedResponse deleteIp2GeoDataIndex(final String index) { IP2GEO_DATA_INDEX_NAME_PREFIX ); } - return client.admin() - .indices() - .prepareDelete(index) - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) - .execute() - .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + return StashedThreadContext.run( + client, + () -> client.admin() + .indices() + .prepareDelete(index) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) + ); } } diff --git a/src/main/java/org/opensearch/geospatial/shared/StashedThreadContext.java b/src/main/java/org/opensearch/geospatial/shared/StashedThreadContext.java new file mode 100644 index 00000000..1ee59297 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/shared/StashedThreadContext.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.shared; + +import java.util.function.Supplier; + +import org.opensearch.client.Client; +import org.opensearch.common.util.concurrent.ThreadContext; + +/** + * Helper class to run code with stashed thread context + * + * Code need to be run with stashed thread context if it interacts with system index + * when security plugin is enabled. + */ +public class StashedThreadContext { + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing + */ + public static void run(final Client client, final Runnable function) { + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + function.run(); + } + } + + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function supplier function that needs to be executed after thread context has been stashed, return object + */ + public static T run(final Client client, final Supplier function) { + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + return function.get(); + } + } +} From 50a29e8583024ae226a196e71564c075ec17708d Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 9 May 2023 19:39:20 -0700 Subject: [PATCH 25/61] Reduce lock duration and renew the lock during update (#299) Signed-off-by: Heemin Kim --- .../action/PutDatasourceTransportAction.java | 61 ++++++++---- .../ip2geo/common/GeoIpDataFacade.java | 11 ++- .../ip2geo/common/Ip2GeoLockService.java | 38 ++++++-- .../ip2geo/jobscheduler/Datasource.java | 4 +- .../ip2geo/jobscheduler/DatasourceRunner.java | 25 ++--- .../jobscheduler/DatasourceUpdateService.java | 13 ++- .../geospatial/plugin/GeospatialPlugin.java | 14 ++- .../geospatial/ip2geo/Ip2GeoTestCase.java | 20 +++- .../action/PutDatasourceRequestTests.java | 4 + .../PutDatasourceTransportActionTests.java | 83 +++++++++++++++-- .../ip2geo/common/GeoIpDataFacadeTests.java | 9 +- .../ip2geo/common/Ip2GeoLockServiceTests.java | 74 +++++++++++++-- .../jobscheduler/DatasourceRunnerTests.java | 92 ++++++++++++++++--- .../DatasourceUpdateServiceTests.java | 37 ++++++-- .../plugin/GeospatialPluginTests.java | 4 +- 15 files changed, 401 insertions(+), 88 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index 7ae05cdd..fa3fcf48 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -5,11 +5,14 @@ package org.opensearch.geospatial.ip2geo.action; -import java.io.IOException; +import static org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService.LOCK_DURATION_IN_SECONDS; + import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; import lombok.extern.log4j.Log4j2; +import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; @@ -21,9 +24,11 @@ import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -36,6 +41,7 @@ public class PutDatasourceTransportAction extends HandledTransportAction listener) { - try { - StepListener createIndexStep = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(createIndexStep); - createIndexStep.whenComplete(v -> putDatasource(request, listener), exception -> listener.onFailure(exception)); - } catch (Exception e) { - listener.onFailure(e); - } + lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + return; + } + try { + internalDoExecute(request, lock, listener); + } catch (Exception e) { + listener.onFailure(e); + } finally { + lockService.releaseLock( + lock, + ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) + ); + } + }, exception -> { listener.onFailure(exception); })); } @VisibleForTesting - protected void putDatasource(final PutDatasourceRequest request, final ActionListener listener) - throws IOException { - Datasource datasource = Datasource.Builder.build(request); - datasourceFacade.putDatasource(datasource, getIndexResponseListener(datasource, listener)); + protected void internalDoExecute( + final PutDatasourceRequest request, + final LockModel lock, + final ActionListener listener + ) { + StepListener createIndexStep = new StepListener<>(); + datasourceFacade.createIndexIfNotExists(createIndexStep); + createIndexStep.whenComplete(v -> { + Datasource datasource = Datasource.Builder.build(request); + datasourceFacade.putDatasource( + datasource, + getIndexResponseListener(datasource, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), listener) + ); + }, exception -> listener.onFailure(exception)); } @VisibleForTesting protected ActionListener getIndexResponseListener( final Datasource datasource, + final Runnable renewLock, final ActionListener listener ) { return new ActionListener<>() { @@ -87,7 +116,7 @@ protected ActionListener getIndexResponseListener( public void onResponse(final IndexResponse indexResponse) { // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread // pool. - threadPool.generic().submit(() -> { createDatasource(datasource); }); + threadPool.generic().submit(() -> { createDatasource(datasource, renewLock); }); listener.onResponse(new AcknowledgedResponse(true)); } @@ -103,7 +132,7 @@ public void onFailure(final Exception e) { } @VisibleForTesting - protected void createDatasource(final Datasource datasource) { + protected void createDatasource(final Datasource datasource, final Runnable renewLock) { if (DatasourceState.CREATING.equals(datasource.getState()) == false) { log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.CREATING, datasource.getState()); markDatasourceAsCreateFailed(datasource); @@ -111,7 +140,7 @@ protected void createDatasource(final Datasource datasource) { } try { - datasourceUpdateService.updateOrCreateGeoIpData(datasource); + datasourceUpdateService.updateOrCreateGeoIpData(datasource, renewLock); } catch (Exception e) { log.error("Failed to create datasource for {}", datasource.getName(), e); markDatasourceAsCreateFailed(datasource); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index bf436c33..95ae8003 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -25,6 +25,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import lombok.NonNull; import lombok.extern.log4j.Log4j2; import org.apache.commons.csv.CSVFormat; @@ -332,8 +333,15 @@ public void onFailure(final Exception e) { * @param fields Field name matching with data in CSVRecord in order * @param iterator GeoIP data to insert * @param bulkSize Bulk size of data to process + * @param renewLock Runnable to renew lock */ - public void putGeoIpData(final String indexName, final String[] fields, final Iterator iterator, final int bulkSize) { + public void putGeoIpData( + @NonNull final String indexName, + @NonNull final String[] fields, + @NonNull final Iterator iterator, + final int bulkSize, + @NonNull final Runnable renewLock + ) { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); while (iterator.hasNext()) { @@ -352,6 +360,7 @@ public void putGeoIpData(final String indexName, final String[] fields, final It } bulkRequest.requests().clear(); } + renewLock.run(); } StashedThreadContext.run(client, () -> { client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java index cd41251a..ab646628 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java @@ -7,15 +7,15 @@ import static org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension.JOB_INDEX_NAME; +import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.inject.Inject; -import org.opensearch.common.unit.TimeValue; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.utils.LockService; @@ -23,8 +23,9 @@ * A wrapper of job scheduler's lock service for datasource */ public class Ip2GeoLockService { + public static final long LOCK_DURATION_IN_SECONDS = 300l; + public static final long RENEW_AFTER_IN_SECONDS = 120l; private final ClusterService clusterService; - private final Client client; private final LockService lockService; /** @@ -33,10 +34,8 @@ public class Ip2GeoLockService { * @param clusterService the cluster service * @param client the client */ - @Inject public Ip2GeoLockService(final ClusterService clusterService, final Client client) { this.clusterService = clusterService; - this.client = client; this.lockService = new LockService(client, clusterService); } @@ -68,10 +67,9 @@ public void releaseLock(final LockModel lockModel, final ActionListener * Synchronous method of LockService#renewLock * * @param lockModel lock to renew - * @param timeout timeout in milliseconds precise * @return renewed lock if renew succeed and null otherwise */ - public LockModel renewLock(final LockModel lockModel, final TimeValue timeout) { + public LockModel renewLock(final LockModel lockModel) { AtomicReference lockReference = new AtomicReference(); CountDownLatch countDownLatch = new CountDownLatch(1); lockService.renewLock(lockModel, new ActionListener<>() { @@ -89,10 +87,34 @@ public void onFailure(final Exception e) { }); try { - countDownLatch.await(timeout.getMillis(), TimeUnit.MILLISECONDS); + countDownLatch.await(clusterService.getClusterSettings().get(Ip2GeoSettings.TIMEOUT).getSeconds(), TimeUnit.SECONDS); return lockReference.get(); } catch (InterruptedException e) { return null; } } + + /** + * Return a runnable which can renew the given lock model + * + * The runnable renews the lock and store the renewed lock in the AtomicReference. + * It only renews the lock when it passed {@code RENEW_AFTER_IN_SECONDS} since + * the last time the lock was renewed to avoid resource abuse. + * + * @param lockModel lock model to renew + * @return runnable which can renew the given lock for every call + */ + public Runnable getRenewLockRunnable(final AtomicReference lockModel) { + return () -> { + LockModel preLock = lockModel.get(); + if (Instant.now().isBefore(preLock.getLockTime().plusSeconds(RENEW_AFTER_IN_SECONDS))) { + return; + } + + lockModel.set(renewLock(lockModel.get())); + if (lockModel.get() == null) { + new OpenSearchException("failed to renew a lock [{}]", preLock); + } + }; + } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 2604490f..f27ab286 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -33,6 +33,7 @@ import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; @@ -50,7 +51,6 @@ public class Datasource implements Writeable, ScheduledJobParameter { * Prefix of indices having Ip2Geo data */ public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".ip2geo-data"; - private static final long LOCK_DURATION_IN_SECONDS = 60 * 60; private static final long MAX_JITTER_IN_MINUTES = 5; private static final long ONE_DAY_IN_HOURS = 24; private static final long ONE_HOUR_IN_MINUTES = 60; @@ -282,7 +282,7 @@ public boolean isEnabled() { @Override public Long getLockDurationSeconds() { - return LOCK_DURATION_IN_SECONDS; + return Ip2GeoLockService.LOCK_DURATION_IN_SECONDS; } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index 0ad8438b..4233c8a8 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; import lombok.extern.log4j.Log4j2; @@ -16,10 +17,10 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.jobscheduler.spi.utils.LockService; /** * Datasource update task @@ -52,6 +53,7 @@ public static DatasourceRunner getJobRunnerInstance() { private DatasourceUpdateService datasourceUpdateService; private Ip2GeoExecutor ip2GeoExecutor; private DatasourceFacade datasourceFacade; + private Ip2GeoLockService ip2GeoLockService; private boolean initialized; private DatasourceRunner() { @@ -65,12 +67,14 @@ public void initialize( final ClusterService clusterService, final DatasourceUpdateService datasourceUpdateService, final Ip2GeoExecutor ip2GeoExecutor, - final DatasourceFacade datasourceFacade + final DatasourceFacade datasourceFacade, + final Ip2GeoLockService ip2GeoLockService ) { this.clusterService = clusterService; this.datasourceUpdateService = datasourceUpdateService; this.ip2GeoExecutor = ip2GeoExecutor; this.datasourceFacade = datasourceFacade; + this.ip2GeoLockService = ip2GeoLockService; this.initialized = true; } @@ -87,30 +91,27 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC ); } - ip2GeoExecutor.forDatasourceUpdate().submit(updateDatasourceRunner(jobParameter, context)); + ip2GeoExecutor.forDatasourceUpdate().submit(updateDatasourceRunner(jobParameter)); } /** * Update GeoIP data * * Lock is used so that only one of nodes run this task. - * Lock duration is 1 hour to avoid refreshing. This is okay because update interval is 1 day minimum. * * @param jobParameter job parameter - * @param context context */ @VisibleForTesting - protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter, final JobExecutionContext context) { - final LockService lockService = context.getLockService(); + protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter) { return () -> { - lockService.acquireLock(jobParameter, context, ActionListener.wrap(lock -> { + ip2GeoLockService.acquireLock(jobParameter.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { return; } try { - updateDatasource(jobParameter); + updateDatasource(jobParameter, ip2GeoLockService.getRenewLockRunnable(new AtomicReference<>(lock))); } finally { - lockService.release( + ip2GeoLockService.releaseLock( lock, ActionListener.wrap(released -> {}, exception -> { log.error("Failed to release lock [{}]", lock, exception); }) ); @@ -120,7 +121,7 @@ protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParamet } @VisibleForTesting - protected void updateDatasource(final ScheduledJobParameter jobParameter) throws IOException { + protected void updateDatasource(final ScheduledJobParameter jobParameter, final Runnable renewLock) throws IOException { Datasource datasource = datasourceFacade.getDatasource(jobParameter.getName()); /** * If delete request comes while update task is waiting on a queue for other update tasks to complete, @@ -143,7 +144,7 @@ protected void updateDatasource(final ScheduledJobParameter jobParameter) throws try { datasourceUpdateService.deleteUnusedIndices(datasource); - datasourceUpdateService.updateOrCreateGeoIpData(datasource); + datasourceUpdateService.updateOrCreateGeoIpData(datasource, renewLock); datasourceUpdateService.deleteUnusedIndices(datasource); } catch (Exception e) { log.error("Failed to update datasource for {}", datasource.getName(), e); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 2a6155d3..57be4d4a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -48,10 +48,11 @@ public DatasourceUpdateService( /** * Update GeoIp data * - * @param datasource + * @param datasource the datasource + * @param renewLock runnable to renew lock * @throws Exception */ - public void updateOrCreateGeoIpData(final Datasource datasource) throws Exception { + public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws Exception { URL url = new URL(datasource.getEndpoint()); DatasourceManifest manifest = DatasourceManifest.Builder.build(url); @@ -77,7 +78,13 @@ public void updateOrCreateGeoIpData(final Datasource datasource) throws Exceptio datasource.getDatabase().getFields().toString() ); } - geoIpDataFacade.putGeoIpData(indexName, header, reader.iterator(), clusterSettings.get(Ip2GeoSettings.INDEXING_BULK_SIZE)); + geoIpDataFacade.putGeoIpData( + indexName, + header, + reader.iterator(), + clusterSettings.get(Ip2GeoSettings.INDEXING_BULK_SIZE), + renewLock + ); } Instant endTime = Instant.now(); diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index e52939b0..6febdf3f 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -45,6 +45,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; @@ -130,13 +131,22 @@ public Collection createComponents( DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService); DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); + Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client); /** * We don't need to return datasource runner because it is used only by job scheduler and job scheduler * does not use DI but it calls DatasourceExtension#getJobRunner to get DatasourceRunner instance. */ - DatasourceRunner.getJobRunnerInstance().initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + DatasourceRunner.getJobRunnerInstance() + .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade, ip2GeoLockService); - return List.of(UploadStats.getInstance(), datasourceUpdateService, datasourceFacade, ip2GeoExecutor, geoIpDataFacade); + return List.of( + UploadStats.getInstance(), + datasourceUpdateService, + datasourceFacade, + ip2GeoExecutor, + geoIpDataFacade, + ip2GeoLockService + ); } @Override diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index ee57b5ba..a28ba089 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -15,7 +15,6 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Locale; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -43,10 +42,12 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.ingest.IngestService; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.utils.LockService; import org.opensearch.tasks.Task; @@ -66,8 +67,6 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { @Mock protected Ip2GeoExecutor ip2GeoExecutor; @Mock - protected ExecutorService executorService; - @Mock protected GeoIpDataFacade geoIpDataFacade; @Mock protected ClusterState clusterState; @@ -81,6 +80,8 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected ThreadPool threadPool; @Mock protected TransportService transportService; + @Mock + protected Ip2GeoLockService ip2GeoLockService; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; protected LockService lockService; @@ -101,7 +102,7 @@ public void prepareIp2GeoTestCase() { when(clusterService.state()).thenReturn(clusterState); when(clusterState.metadata()).thenReturn(metadata); when(clusterState.routingTable()).thenReturn(RoutingTable.EMPTY_ROUTING_TABLE); - when(ip2GeoExecutor.forDatasourceUpdate()).thenReturn(executorService); + when(ip2GeoExecutor.forDatasourceUpdate()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); when(ingestService.getClusterService()).thenReturn(clusterService); when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); } @@ -190,6 +191,17 @@ protected Datasource randomDatasource() { return datasource; } + protected LockModel randomLockModel() { + LockModel lockModel = new LockModel( + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString(), + Instant.now(), + randomPositiveLong(), + false + ); + return lockModel; + } + /** * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) * is merged in OpenSearch core diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index dc150b18..2c8bf9f6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -36,7 +36,11 @@ public void testValidate_whenInvalidManifestFile_thenFails() { PutDatasourceRequest request = new PutDatasourceRequest(datasourceName); request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); request.setUpdateInterval(TimeValue.timeValueDays(1)); + + // Run ActionRequestValidationException exception = request.validate(); + + // Verify assertEquals(1, exception.validationErrors().size()); assertTrue(exception.validationErrors().get(0).contains("Error occurred while reading a file")); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 0c7bdf6d..339c65cb 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -6,14 +6,22 @@ package org.opensearch.geospatial.ip2geo.action; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.time.Instant; import lombok.SneakyThrows; import org.junit.Before; import org.mockito.ArgumentCaptor; +import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; @@ -24,6 +32,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.tasks.Task; public class PutDatasourceTransportActionTests extends Ip2GeoTestCase { @@ -31,19 +40,75 @@ public class PutDatasourceTransportActionTests extends Ip2GeoTestCase { @Before public void init() { - action = new PutDatasourceTransportAction(transportService, actionFilters, threadPool, datasourceFacade, datasourceUpdateService); + action = new PutDatasourceTransportAction( + transportService, + actionFilters, + threadPool, + datasourceFacade, + datasourceUpdateService, + ip2GeoLockService + ); + } + + @SneakyThrows + public void testDoExecute_whenFailedToAcquireLock_thenError() { + validateDoExecute(null, null); } @SneakyThrows public void testDoExecute_whenValidInput_thenSucceed() { + String jobIndexName = GeospatialTestHelper.randomLowerCaseString(); + String jobId = GeospatialTestHelper.randomLowerCaseString(); + LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); + validateDoExecute(lockModel, null); + } + + @SneakyThrows + public void testDoExecute_whenException_thenError() { + validateDoExecute(null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { Task task = mock(Task.class); + Datasource datasource = randomDatasource(); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + PutDatasourceRequest request = new PutDatasourceRequest(datasource.getName()); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + if (exception == null) { + // Run + captor.getValue().onResponse(lockModel); + + // Verify + if (lockModel == null) { + verify(listener).onFailure(any(OpenSearchException.class)); + } else { + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + } else { + // Run + captor.getValue().onFailure(exception); + // Verify + verify(listener).onFailure(exception); + } + } + + @SneakyThrows + public void testInternalDoExecute_whenValidInput_thenSucceed() { PutDatasourceRequest request = new PutDatasourceRequest(GeospatialTestHelper.randomLowerCaseString()); request.setEndpoint(sampleManifestUrl()); request.setUpdateInterval(TimeValue.timeValueDays(1)); ActionListener listener = mock(ActionListener.class); // Run - action.doExecute(task, request, listener); + action.internalDoExecute(request, randomLockModel(), listener); // Verify ArgumentCaptor captor = ArgumentCaptor.forClass(StepListener.class); @@ -68,7 +133,7 @@ public void testDoExecute_whenValidInput_thenSucceed() { public void testGetIndexResponseListener_whenVersionConflict_thenFailure() { Datasource datasource = new Datasource(); ActionListener listener = mock(ActionListener.class); - action.getIndexResponseListener(datasource, listener) + action.getIndexResponseListener(datasource, mock(Runnable.class), listener) .onFailure( new VersionConflictEngineException( null, @@ -86,21 +151,22 @@ public void testCreateDatasource_whenInvalidState_thenUpdateStateAsFailed() { datasource.getUpdateStats().setLastFailedAt(null); // Run - action.createDatasource(datasource); + action.createDatasource(datasource, mock(Runnable.class)); // Verify assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); assertNotNull(datasource.getUpdateStats().getLastFailedAt()); verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceUpdateService, never()).updateOrCreateGeoIpData(any(Datasource.class), any(Runnable.class)); } @SneakyThrows public void testCreateDatasource_whenExceptionHappens_thenUpdateStateAsFailed() { Datasource datasource = new Datasource(); - doThrow(new RuntimeException()).when(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + doThrow(new RuntimeException()).when(datasourceUpdateService).updateOrCreateGeoIpData(any(Datasource.class), any(Runnable.class)); // Run - action.createDatasource(datasource); + action.createDatasource(datasource, mock(Runnable.class)); // Verify assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); @@ -112,11 +178,12 @@ public void testCreateDatasource_whenExceptionHappens_thenUpdateStateAsFailed() public void testCreateDatasource_whenValidInput_thenUpdateStateAsCreating() { Datasource datasource = new Datasource(); + Runnable renewLock = mock(Runnable.class); // Run - action.createDatasource(datasource); + action.createDatasource(datasource, renewLock); // Verify - verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource, renewLock); assertEquals(DatasourceState.CREATING, datasource.getState()); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index d86c2795..54c98a41 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -25,6 +25,8 @@ import java.util.Locale; import java.util.Map; +import lombok.SneakyThrows; + import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -157,7 +159,8 @@ public void testDeleteIp2GeoDataIndexWithNonIp2GeoDataIndex() { verify(verifyingClient, never()).index(any()); } - public void testPutGeoIpData() throws Exception { + @SneakyThrows + public void testPutGeoIpData_whenValidInput_thenSucceed() { String index = GeospatialTestHelper.randomLowerCaseString(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { if (actionRequest instanceof BulkRequest) { @@ -188,10 +191,12 @@ public void testPutGeoIpData() throws Exception { throw new RuntimeException("invalid request is called"); } }); + Runnable renewLock = mock(Runnable.class); try (CSVParser csvParser = CSVParser.parse(sampleIp2GeoFile(), StandardCharsets.UTF_8, CSVFormat.RFC4180)) { Iterator iterator = csvParser.iterator(); String[] fields = iterator.next().values(); - verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, 1); + verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, 1, renewLock); + verify(renewLock, times(2)).run(); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java index a746f314..f0bc8ce7 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java @@ -6,41 +6,103 @@ package org.opensearch.geospatial.ip2geo.common; import static org.mockito.Mockito.mock; +import static org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService.RENEW_AFTER_IN_SECONDS; import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.opensearch.action.ActionListener; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.index.shard.ShardId; +import org.opensearch.jobscheduler.spi.LockModel; public class Ip2GeoLockServiceTests extends Ip2GeoTestCase { private Ip2GeoLockService ip2GeoLockService; + private Ip2GeoLockService noOpsLockService; @Before public void init() { - ip2GeoLockService = new Ip2GeoLockService(clusterService, client); + ip2GeoLockService = new Ip2GeoLockService(clusterService, verifyingClient); + noOpsLockService = new Ip2GeoLockService(clusterService, client); } public void testAcquireLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - ip2GeoLockService.acquireLock(GeospatialTestHelper.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); + noOpsLockService.acquireLock(GeospatialTestHelper.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); } public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - ip2GeoLockService.releaseLock(null, mock(ActionListener.class)); + noOpsLockService.releaseLock(null, mock(ActionListener.class)); } public void testRenewLock_whenCalled_thenNotBlocked() { - long timeoutInMillis = 10000; long expectedDurationInMillis = 1000; Instant before = Instant.now(); - assertNull(ip2GeoLockService.renewLock(null, TimeValue.timeValueMillis(timeoutInMillis))); + assertNull(ip2GeoLockService.renewLock(null)); Instant after = Instant.now(); assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); } + + public void testGetRenewLockRunnable_whenLockIsFresh_thenDoNotRenew() { + LockModel lockModel = new LockModel( + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + GeospatialTestHelper.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + ip2GeoLockService.getRenewLockRunnable(reference).run(); + assertEquals(lockModel, reference.get()); + } + + public void testGetRenewLockRunnable_whenLockIsStale_thenRenew() { + LockModel lockModel = new LockModel( + GeospatialTestHelper.randomLowerCaseString(), + GeospatialTestHelper.randomLowerCaseString(), + Instant.now().minusSeconds(RENEW_AFTER_IN_SECONDS), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + GeospatialTestHelper.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + ip2GeoLockService.getRenewLockRunnable(reference).run(); + assertNotEquals(lockModel, reference.get()); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 274689ac..1ccd88a5 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -6,6 +6,8 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -14,52 +16,111 @@ import static org.mockito.internal.verification.VerificationModeFactory.times; import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; +import java.io.IOException; import java.time.Instant; +import lombok.SneakyThrows; + import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.action.ActionListener; +import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.jobscheduler.spi.JobDocVersion; import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; public class DatasourceRunnerTests extends Ip2GeoTestCase { @Before public void init() { - DatasourceRunner.getJobRunnerInstance().initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade); + DatasourceRunner.getJobRunnerInstance() + .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade, ip2GeoLockService); } - public void testRunJobInvalidClass() { + public void testRunJob_whenInvalidClass_thenThrowException() { JobExecutionContext jobExecutionContext = mock(JobExecutionContext.class); ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); expectThrows(IllegalStateException.class, () -> DatasourceRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); } - public void testRunJob() { + public void testRunJob_whenValidInput_thenSucceed() { JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); String jobIndexName = randomLowerCaseString(); String jobId = randomLowerCaseString(); JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); - Datasource datasource = new Datasource(); + Datasource datasource = randomDatasource(); // Run DatasourceRunner.getJobRunnerInstance().runJob(datasource, jobExecutionContext); // Verify - verify(executorService).submit(any(Runnable.class)); + verify(ip2GeoLockService).acquireLock( + eq(datasource.getName()), + eq(Ip2GeoLockService.LOCK_DURATION_IN_SECONDS), + any(ActionListener.class) + ); + } + + @SneakyThrows + public void testUpdateDatasourceRunner_whenFailedToAcquireLock_thenError() { + validateDoExecute(null, null); + } + + @SneakyThrows + public void testUpdateDatasourceRunner_whenValidInput_thenSucceed() { + String jobIndexName = GeospatialTestHelper.randomLowerCaseString(); + String jobId = GeospatialTestHelper.randomLowerCaseString(); + LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); + validateDoExecute(lockModel, null); + } + + @SneakyThrows + public void testUpdateDatasourceRunner_whenException_thenError() { + validateDoExecute(null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasourceRunner(jobParameter).run(); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(jobParameter.getName()), anyLong(), captor.capture()); + + if (exception == null) { + // Run + captor.getValue().onResponse(lockModel); + + // Verify + verify(ip2GeoLockService, lockModel == null ? never() : times(1)).releaseLock(eq(lockModel), any(ActionListener.class)); + } else { + // Run + captor.getValue().onFailure(exception); + + // Verify + verify(ip2GeoLockService, never()).releaseLock(eq(lockModel), any(ActionListener.class)); + } } - public void testUpdateDatasourceNull() throws Exception { + @SneakyThrows + public void testUpdateDatasource_whenDatasourceDoesNotExist_thenDoNothing() { Datasource datasource = new Datasource(); // Run - DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, mock(Runnable.class)); // Verify verify(datasourceUpdateService, never()).deleteUnusedIndices(any()); } - public void testUpdateDatasourceInvalidState() throws Exception { + @SneakyThrows + public void testUpdateDatasource_whenInvalidState_thenUpdateLastFailedAt() { Datasource datasource = new Datasource(); datasource.enable(); datasource.getUpdateStats().setLastFailedAt(null); @@ -67,7 +128,7 @@ public void testUpdateDatasourceInvalidState() throws Exception { when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); // Run - DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, mock(Runnable.class)); // Verify assertFalse(datasource.isEnabled()); @@ -75,21 +136,24 @@ public void testUpdateDatasourceInvalidState() throws Exception { verify(datasourceFacade).updateDatasource(datasource); } - public void testUpdateDatasource() throws Exception { + @SneakyThrows + public void testUpdateDatasource_whenValidInput_thenSucceed() { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.setName(randomLowerCaseString()); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); // Run - DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, renewLock); // Verify verify(datasourceUpdateService, times(2)).deleteUnusedIndices(datasource); - verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource); + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource, renewLock); } - public void testUpdateDatasourceExceptionHandling() throws Exception { + @SneakyThrows + public void testUpdateDatasourceExceptionHandling() { Datasource datasource = new Datasource(); datasource.setName(randomLowerCaseString()); datasource.getUpdateStats().setLastFailedAt(null); @@ -97,7 +161,7 @@ public void testUpdateDatasourceExceptionHandling() throws Exception { doThrow(new RuntimeException("test failure")).when(datasourceUpdateService).deleteUnusedIndices(any()); // Run - DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource); + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, mock(Runnable.class)); // Verify assertNotNull(datasource.getUpdateStats().getLastFailedAt()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index f7ca7d2e..6e51c926 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -6,6 +6,10 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -14,6 +18,9 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; +import java.util.Iterator; + +import lombok.SneakyThrows; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -35,7 +42,8 @@ public void init() { datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); } - public void testUpdateDatasourceSkip() throws Exception { + @SneakyThrows + public void testUpdateOrCreateGeoIpData_whenHashValueIsSame_thenSkipUpdate() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); @@ -47,14 +55,15 @@ public void testUpdateDatasourceSkip() throws Exception { datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); // Run - datasourceUpdateService.updateOrCreateGeoIpData(datasource); + datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class)); // Verify assertNotNull(datasource.getUpdateStats().getLastSkippedAt()); verify(datasourceFacade).updateDatasource(datasource); } - public void testUpdateDatasourceInvalidFile() throws Exception { + @SneakyThrows + public void testUpdateOrCreateGeoIpData_whenInvalidData_thenThrowException() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); @@ -71,10 +80,11 @@ public void testUpdateDatasourceInvalidFile() throws Exception { datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); // Run - expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource)); + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class))); } - public void testUpdateDatasourceIncompatibleFields() throws Exception { + @SneakyThrows + public void testUpdateOrCreateGeoIpData_whenIncompatibleFields_thenThrowException() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); @@ -89,10 +99,11 @@ public void testUpdateDatasourceIncompatibleFields() throws Exception { datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); // Run - expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource)); + expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class))); } - public void testUpdateDatasource() throws Exception { + @SneakyThrows + public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); @@ -109,7 +120,7 @@ public void testUpdateDatasource() throws Exception { datasource.getUpdateStats().setLastProcessingTimeInMillis(null); // Run - datasourceUpdateService.updateOrCreateGeoIpData(datasource); + datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class)); // Verify assertEquals(manifest.getProvider(), datasource.getDatabase().getProvider()); @@ -119,9 +130,17 @@ public void testUpdateDatasource() throws Exception { assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); assertNotNull(datasource.getUpdateStats().getLastProcessingTimeInMillis()); verify(datasourceFacade, times(2)).updateDatasource(datasource); + verify(geoIpDataFacade).putGeoIpData( + eq(datasource.currentIndexName()), + isA(String[].class), + any(Iterator.class), + anyInt(), + any(Runnable.class) + ); } - public void testDeleteUnusedIndices() throws Exception { + @SneakyThrows + public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String indexPrefix = String.format(".ip2geo-data.%s.", datasourceName); Instant now = Instant.now(); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 3917e7c7..cc31f5a7 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -37,6 +37,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.processor.FeatureProcessor; @@ -71,7 +72,8 @@ public class GeospatialPluginTests extends OpenSearchTestCase { DatasourceUpdateService.class, DatasourceFacade.class, Ip2GeoExecutor.class, - GeoIpDataFacade.class + GeoIpDataFacade.class, + Ip2GeoLockService.class ); @Mock From 235431eb1125e388e8fd3c41644ac6ee3d1680d9 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 9 May 2023 20:47:52 -0700 Subject: [PATCH 26/61] Implements delete datasource API (#291) Signed-off-by: Heemin Kim --- .../ip2geo/action/DeleteDatasourceAction.java | 27 +++ .../action/DeleteDatasourceRequest.java | 58 +++++ .../DeleteDatasourceTransportAction.java | 134 ++++++++++++ .../ip2geo/action/GetDatasourceAction.java | 2 +- .../action/RestDeleteDatasourceHandler.java | 49 +++++ .../ip2geo/common/DatasourceFacade.java | 38 +++- .../ip2geo/processor/Ip2GeoProcessor.java | 5 + .../geospatial/plugin/GeospatialPlugin.java | 9 +- .../geospatial/ip2geo/Ip2GeoTestCase.java | 6 + .../action/DeleteDatasourceRequestTests.java | 30 +++ .../DeleteDatasourceTransportActionTests.java | 202 ++++++++++++++++++ .../RestDeleteDatasourceHandlerTests.java | 49 +++++ .../ip2geo/common/DatasourceFacadeTests.java | 38 ++++ .../plugin/GeospatialPluginTests.java | 4 +- 14 files changed, 645 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandler.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandlerTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceAction.java new file mode 100644 index 00000000..b08e0861 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceAction.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.AcknowledgedResponse; + +/** + * Ip2Geo datasource delete action + */ +public class DeleteDatasourceAction extends ActionType { + /** + * Delete datasource action instance + */ + public static final DeleteDatasourceAction INSTANCE = new DeleteDatasourceAction(); + /** + * Delete datasource action name + */ + public static final String NAME = "cluster:admin/geospatial/datasource/delete"; + + private DeleteDatasourceAction() { + super(NAME, AcknowledgedResponse::new); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java new file mode 100644 index 00000000..f8acbbfb --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +/** + * GeoIP datasource delete request + */ +@Getter +@Setter +@AllArgsConstructor +public class DeleteDatasourceRequest extends ActionRequest { + /** + * @param name the datasource name + * @return the datasource name + */ + private String name; + + /** + * Constructor + * + * @param in the stream input + * @throws IOException IOException + */ + public DeleteDatasourceRequest(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = null; + if (name == null || name.isBlank()) { + errors = new ActionRequestValidationException(); + errors.addValidationError("Datasource name should not be empty"); + } + return errors; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java new file mode 100644 index 00000000..2acdfd52 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; + +import lombok.extern.log4j.Log4j2; + +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.IngestService; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +/** + * Transport action to delete datasource + */ +@Log4j2 +public class DeleteDatasourceTransportAction extends HandledTransportAction { + private static final long LOCK_DURATION_IN_SECONDS = 300l; + private final Ip2GeoLockService lockService; + private final IngestService ingestService; + private final DatasourceFacade datasourceFacade; + + /** + * Constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param lockService the lock service + * @param ingestService the ingest service + * @param datasourceFacade the datasource facade + */ + @Inject + public DeleteDatasourceTransportAction( + final TransportService transportService, + final ActionFilters actionFilters, + final Ip2GeoLockService lockService, + final IngestService ingestService, + final DatasourceFacade datasourceFacade + ) { + super(DeleteDatasourceAction.NAME, transportService, actionFilters, DeleteDatasourceRequest::new); + this.lockService = lockService; + this.ingestService = ingestService; + this.datasourceFacade = datasourceFacade; + } + + /** + * We delete datasource regardless of its state as long as we can acquire a lock + * + * @param task the task + * @param request the request + * @param listener the listener + */ + @Override + protected void doExecute(final Task task, final DeleteDatasourceRequest request, final ActionListener listener) { + lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + return; + } + try { + deleteDatasource(request.getName()); + listener.onResponse(new AcknowledgedResponse(true)); + } catch (Exception e) { + listener.onFailure(e); + } finally { + lockService.releaseLock( + lock, + ActionListener.wrap( + released -> { log.info("Released lock for datasource[{}]", request.getName()); }, + exception -> { log.error("Failed to release the lock", exception); } + ) + ); + } + }, exception -> { listener.onFailure(exception); })); + } + + @VisibleForTesting + protected void deleteDatasource(final String datasourceName) throws IOException { + Datasource datasource = datasourceFacade.getDatasource(datasourceName); + if (datasource == null) { + throw new ResourceNotFoundException("no such datasource exist"); + } + + setDatasourceStateAsDeleting(datasource); + datasourceFacade.deleteDatasource(datasource); + } + + private void setDatasourceStateAsDeleting(final Datasource datasource) throws IOException { + if (isSafeToDelete(datasource) == false) { + throw new OpenSearchException("datasource is being used by one of processors"); + } + + DatasourceState previousState = datasource.getState(); + datasource.setState(DatasourceState.DELETING); + datasourceFacade.updateDatasource(datasource); + + // Check again as processor might just have been created. + // If it fails to update the state back to the previous state, the new processor + // will fail to convert an ip to a geo data. + // In such case, user have to delete the processor and delete this datasource again. + if (isSafeToDelete(datasource) == false) { + datasource.setState(previousState); + datasourceFacade.updateDatasource(datasource); + throw new OpenSearchException("datasource is being used by one of processors"); + } + } + + private boolean isSafeToDelete(Datasource datasource) { + IngestMetadata ingestMetadata = ingestService.getClusterService().state().getMetadata().custom(IngestMetadata.TYPE); + return ingestMetadata.getPipelines() + .keySet() + .stream() + .flatMap(pipelineId -> ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class).stream()) + .filter(ip2GeoProcessor -> ip2GeoProcessor.getDatasourceName().equals(datasource.getName())) + .findAny() + .isEmpty(); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java index 7771fd30..039ab35b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceAction.java @@ -16,7 +16,7 @@ public class GetDatasourceAction extends ActionType { */ public static final GetDatasourceAction INSTANCE = new GetDatasourceAction(); /** - * Name of a get datasource action + * Get datasource action name */ public static final String NAME = "cluster:admin/geospatial/datasource/get"; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandler.java new file mode 100644 index 00000000..dc2dd117 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandler.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.rest.RestRequest.Method.DELETE; + +import java.util.List; +import java.util.Locale; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +/** + * Rest handler for Ip2Geo datasource delete request + */ +public class RestDeleteDatasourceHandler extends BaseRestHandler { + private static final String ACTION_NAME = "ip2geo_datasource_delete"; + private static final String PARAMS_NAME = "name"; + + @Override + public String getName() { + return ACTION_NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + final String name = request.param(PARAMS_NAME); + final DeleteDatasourceRequest deleteDatasourceRequest = new DeleteDatasourceRequest(name); + + return channel -> client.executeLocally( + DeleteDatasourceAction.INSTANCE, + deleteDatasourceRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + String path = String.join(URL_DELIMITER, getPluginURLPrefix(), String.format(Locale.ROOT, "ip2geo/datasource/{%s}", PARAMS_NAME)); + return List.of(new Route(DELETE, path)); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 6e03a937..a14a9c72 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -18,23 +18,24 @@ import java.util.Objects; import java.util.stream.Collectors; -import javax.swing.*; - import lombok.extern.log4j.Log4j2; import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; import org.opensearch.action.get.MultiGetResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; @@ -52,6 +53,7 @@ import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; /** @@ -164,6 +166,38 @@ public void putDatasource(final Datasource datasource, final ActionListener list }); } + /** + * Delete datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * + * @param datasource the datasource + * + */ + public void deleteDatasource(final Datasource datasource) { + if (client.admin() + .indices() + .prepareDelete(datasource.getIndices().toArray(new String[0])) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) + .isAcknowledged() == false) { + throw new OpenSearchException("failed to delete data[{}] in datasource", String.join(",", datasource.getIndices())); + } + DeleteResponse response = client.prepareDelete() + .setIndex(DatasourceExtension.JOB_INDEX_NAME) + .setId(datasource.getName()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + + if (response.status().equals(RestStatus.OK)) { + log.info("deleted datasource[{}] successfully", datasource.getName()); + } else if (response.status().equals(RestStatus.NOT_FOUND)) { + throw new ResourceNotFoundException("datasource[{}] does not exist", datasource.getName()); + } else { + throw new OpenSearchException("failed to delete datasource[{}] with status[{}]", datasource.getName(), response.status()); + } + } + /** * Get datasource from an index {@code DatasourceExtension.JOB_INDEX_NAME} * @param name the name of a datasource diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 99a29865..acd56150 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -20,6 +20,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; +import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; @@ -44,6 +45,10 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private static final String PROPERTY_IP = "ip"; private final String field; private final String targetField; + /** + * @return The datasource name + */ + @Getter private final String datasourceName; private final Set properties; private final boolean ignoreMissing; diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 6febdf3f..4547e7f3 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -36,10 +36,13 @@ import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapper; import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldTypeParser; import org.opensearch.geospatial.index.query.xyshape.XYShapeQueryBuilder; +import org.opensearch.geospatial.ip2geo.action.DeleteDatasourceAction; +import org.opensearch.geospatial.ip2geo.action.DeleteDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.action.GetDatasourceAction; import org.opensearch.geospatial.ip2geo.action.GetDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.action.PutDatasourceAction; import org.opensearch.geospatial.ip2geo.action.PutDatasourceTransportAction; +import org.opensearch.geospatial.ip2geo.action.RestDeleteDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -163,7 +166,8 @@ public List getRestHandlers( new RestUploadStatsAction(), new RestUploadGeoJSONAction(), new RestPutDatasourceHandler(clusterSettings), - new RestGetDatasourceHandler() + new RestGetDatasourceHandler(), + new RestDeleteDatasourceHandler() ); } @@ -173,7 +177,8 @@ public List getRestHandlers( new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class), new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class), new ActionHandler<>(PutDatasourceAction.INSTANCE, PutDatasourceTransportAction.class), - new ActionHandler<>(GetDatasourceAction.INSTANCE, GetDatasourceTransportAction.class) + new ActionHandler<>(GetDatasourceAction.INSTANCE, GetDatasourceTransportAction.class), + new ActionHandler<>(DeleteDatasourceAction.INSTANCE, DeleteDatasourceTransportAction.class) ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index a28ba089..429bd031 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -13,6 +13,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Locale; import java.util.concurrent.atomic.AtomicReference; @@ -46,6 +47,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @@ -82,6 +84,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected TransportService transportService; @Mock protected Ip2GeoLockService ip2GeoLockService; + protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; protected LockService lockService; @@ -97,10 +100,13 @@ public void prepareIp2GeoTestCase() { verifyingClient = spy(new VerifyingClient(this.getTestName())); clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings())); lockService = new LockService(client, clusterService); + ingestMetadata = new IngestMetadata(Collections.emptyMap()); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); when(clusterService.state()).thenReturn(clusterState); when(clusterState.metadata()).thenReturn(metadata); + when(clusterState.getMetadata()).thenReturn(metadata); when(clusterState.routingTable()).thenReturn(RoutingTable.EMPTY_ROUTING_TABLE); when(ip2GeoExecutor.forDatasourceUpdate()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); when(ingestService.getClusterService()).thenReturn(clusterService); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java new file mode 100644 index 00000000..8bd84924 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import lombok.SneakyThrows; + +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; + +public class DeleteDatasourceRequestTests extends Ip2GeoTestCase { + @SneakyThrows + public void testStreamInOut_whenValidInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + DeleteDatasourceRequest request = new DeleteDatasourceRequest(datasourceName); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + DeleteDatasourceRequest copiedRequest = new DeleteDatasourceRequest(input); + + // Verify + assertEquals(request.getName(), copiedRequest.getName()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java new file mode 100644 index 00000000..535e5422 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -0,0 +1,202 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import lombok.SneakyThrows; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.PipelineConfiguration; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.tasks.Task; + +public class DeleteDatasourceTransportActionTests extends Ip2GeoTestCase { + private DeleteDatasourceTransportAction action; + + @Before + public void init() { + action = new DeleteDatasourceTransportAction(transportService, actionFilters, ip2GeoLockService, ingestService, datasourceFacade); + } + + @SneakyThrows + public void testDoExecute_whenFailedToAcquireLock_thenError() { + validateDoExecute(null, null); + } + + @SneakyThrows + public void testDoExecute_whenValidInput_thenSucceed() { + String jobIndexName = GeospatialTestHelper.randomLowerCaseString(); + String jobId = GeospatialTestHelper.randomLowerCaseString(); + LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); + validateDoExecute(lockModel, null); + } + + @SneakyThrows + public void testDoExecute_whenException_thenError() { + validateDoExecute(null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { + Task task = mock(Task.class); + Datasource datasource = randomDatasource(); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + DeleteDatasourceRequest request = new DeleteDatasourceRequest(datasource.getName()); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + if (exception == null) { + // Run + captor.getValue().onResponse(lockModel); + + // Verify + if (lockModel == null) { + verify(listener).onFailure(any(OpenSearchException.class)); + } else { + verify(listener).onResponse(new AcknowledgedResponse(true)); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + } else { + // Run + captor.getValue().onFailure(exception); + // Verify + verify(listener).onFailure(exception); + } + } + + @SneakyThrows + public void testDeleteDatasource_whenNull_thenThrowException() { + Datasource datasource = randomDatasource(); + expectThrows(ResourceNotFoundException.class, () -> action.deleteDatasource(datasource.getName())); + } + + @SneakyThrows + public void testDeleteDatasource_whenSafeToDelete_thenDelete() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + + // Run + action.deleteDatasource(datasource.getName()); + + // Verify + assertEquals(DatasourceState.DELETING, datasource.getState()); + verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceFacade).deleteDatasource(datasource); + } + + @SneakyThrows + public void testDeleteDatasource_whenProcessorIsUsingDatasource_thenThrowException() { + Datasource datasource = randomDatasource(); + datasource.setState(DatasourceState.AVAILABLE); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + + String pipelineId = GeospatialTestHelper.randomLowerCaseString(); + Map pipelines = new HashMap<>(); + pipelines.put(pipelineId, createPipelineConfiguration()); + IngestMetadata ingestMetadata = new IngestMetadata(pipelines); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn( + Arrays.asList(createIp2GeoProcessor(datasource.getName())) + ); + + // Run + expectThrows(OpenSearchException.class, () -> action.deleteDatasource(datasource.getName())); + + // Verify + assertEquals(DatasourceState.AVAILABLE, datasource.getState()); + verify(datasourceFacade, never()).updateDatasource(datasource); + verify(datasourceFacade, never()).deleteDatasource(datasource); + } + + @SneakyThrows + public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowException() { + Datasource datasource = randomDatasource(); + datasource.setState(DatasourceState.AVAILABLE); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + + String pipelineId = GeospatialTestHelper.randomLowerCaseString(); + Map pipelines = new HashMap<>(); + pipelines.put(pipelineId, createPipelineConfiguration()); + IngestMetadata ingestMetadata = new IngestMetadata(pipelines); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn( + Collections.emptyList(), + Arrays.asList(createIp2GeoProcessor(datasource.getName())) + ); + + // Run + expectThrows(OpenSearchException.class, () -> action.deleteDatasource(datasource.getName())); + + // Verify + verify(datasourceFacade, times(2)).updateDatasource(datasource); + verify(datasourceFacade, never()).deleteDatasource(datasource); + } + + private PipelineConfiguration createPipelineConfiguration() { + String id = GeospatialTestHelper.randomLowerCaseString(); + ByteBuffer byteBuffer = ByteBuffer.wrap(GeospatialTestHelper.randomLowerCaseString().getBytes(StandardCharsets.US_ASCII)); + BytesReference config = BytesReference.fromByteBuffer(byteBuffer); + return new PipelineConfiguration(id, config, XContentType.JSON); + } + + private Ip2GeoProcessor createIp2GeoProcessor(String datasourceName) { + String tag = GeospatialTestHelper.randomLowerCaseString(); + String description = GeospatialTestHelper.randomLowerCaseString(); + String field = GeospatialTestHelper.randomLowerCaseString(); + String targetField = GeospatialTestHelper.randomLowerCaseString(); + Set properties = Set.of(GeospatialTestHelper.randomLowerCaseString()); + Ip2GeoProcessor ip2GeoProcessor = new Ip2GeoProcessor( + tag, + description, + field, + targetField, + datasourceName, + properties, + true, + true, + clusterSettings, + datasourceFacade, + geoIpDataFacade + ); + return ip2GeoProcessor; + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandlerTests.java new file mode 100644 index 00000000..937c5532 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestDeleteDatasourceHandlerTests.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; + +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Before; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; + +public class RestDeleteDatasourceHandlerTests extends RestActionTestCase { + private String path; + private RestDeleteDatasourceHandler action; + + @Before + public void setupAction() { + action = new RestDeleteDatasourceHandler(); + controller().registerHandler(action); + path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s"); + } + + public void testPrepareRequest_whenValidInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.DELETE) + .withPath(String.format(Locale.ROOT, path, datasourceName)) + .build(); + AtomicBoolean isExecuted = new AtomicBoolean(false); + + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof DeleteDatasourceRequest); + DeleteDatasourceRequest deleteDatasourceRequest = (DeleteDatasourceRequest) actionRequest; + assertEquals(datasourceName, deleteDatasourceRequest.getName()); + isExecuted.set(true); + return null; + }); + + dispatchRequest(request); + assertTrue(isExecuted.get()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index baacde79..d2ca28bb 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -26,6 +26,9 @@ import org.opensearch.action.DocWriteRequest; import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; @@ -34,7 +37,9 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; @@ -45,6 +50,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -209,6 +215,38 @@ private Datasource setupClientForGetRequest(final boolean isExist, final Runtime return datasource; } + public void testDeleteDatasource_whenValidInput_thenSucceed() { + Datasource datasource = randomDatasource(); + verifyingClient.setExecuteVerifier( + (actionResponse, actionRequest) -> { + // Verify + if (actionRequest instanceof DeleteIndexRequest) { + DeleteIndexRequest request = (DeleteIndexRequest) actionRequest; + assertEquals(datasource.getIndices().size(), request.indices().length); + assertEquals(IndicesOptions.LENIENT_EXPAND_OPEN, request.indicesOptions()); + + AcknowledgedResponse response = new AcknowledgedResponse(true); + return response; + } else if (actionRequest instanceof DeleteRequest) { + DeleteRequest request = (DeleteRequest) actionRequest; + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); + assertEquals(datasource.getName(), request.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.OK); + return response; + } else { + throw new RuntimeException("Not expected request type is passed" + actionRequest.getClass()); + } + } + ); + + // Run + datasourceFacade.deleteDatasource(datasource); + } + public void testGetDatasources_whenValidInput_thenSucceed() { List datasources = Arrays.asList(randomDatasource(), randomDatasource()); String[] names = datasources.stream().map(Datasource::getName).toArray(String[]::new); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index cc31f5a7..1913d830 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -32,6 +32,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; +import org.opensearch.geospatial.ip2geo.action.RestDeleteDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -62,7 +63,8 @@ public class GeospatialPluginTests extends OpenSearchTestCase { new RestUploadGeoJSONAction(), new RestUploadStatsAction(), new RestPutDatasourceHandler(clusterSettings), - new RestGetDatasourceHandler() + new RestGetDatasourceHandler(), + new RestDeleteDatasourceHandler() ); private final Set SUPPORTED_SYSTEM_INDEX_PATTERN = Set.of(IP2GEO_DATA_INDEX_NAME_PREFIX); From 062d73bbb8e69af4af900ca4345b4ef69831fc48 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 10 May 2023 09:36:22 -0700 Subject: [PATCH 27/61] Set User-Agent in http request (#300) Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceManifest.java | 36 ++++++++++++------ .../ip2geo/common/GeoIpDataFacade.java | 34 ++++++++++------- .../ip2geo/common/Ip2GeoSettings.java | 3 +- .../geospatial/shared/Constants.java | 15 ++++++++ .../action/RestPutDatasourceHandlerTests.java | 2 +- .../common/DatasourceManifestTests.java | 38 +++++++++++++++++++ .../ip2geo/common/GeoIpDataFacadeTests.java | 25 ++++++++++++ src/test/resources/ip2geo/manifest.json | 2 +- .../ip2geo/manifest_invalid_url.json | 2 +- .../resources/ip2geo/manifest_template.json | 2 +- 10 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/shared/Constants.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifestTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java index fca40213..5382aa56 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; +import java.net.URLConnection; import java.nio.CharBuffer; import java.security.AccessController; import java.security.PrivilegedAction; @@ -25,6 +26,8 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.shared.Constants; /** * Ip2Geo datasource manifest file object @@ -39,7 +42,7 @@ public class DatasourceManifest { private static final ParseField DB_NAME_FIELD = new ParseField("db_name"); private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); - private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); + private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at_in_epoch_milli"); private static final ParseField PROVIDER_FIELD = new ParseField("provider"); /** @@ -114,20 +117,31 @@ public static class Builder { public static DatasourceManifest build(final URL url) { SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedAction) () -> { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) { - CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES); - reader.read(charBuffer); - charBuffer.flip(); - XContentParser parser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.IGNORE_DEPRECATIONS, - charBuffer.toString() - ); - return PARSER.parse(parser, null); + try { + URLConnection connection = url.openConnection(); + return internalBuild(connection); } catch (IOException e) { throw new RuntimeException(e); } }); } + + @VisibleForTesting + @SuppressForbidden(reason = "Need to connect to http endpoint to read manifest file") + protected static DatasourceManifest internalBuild(final URLConnection connection) throws IOException { + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + InputStreamReader inputStreamReader = new InputStreamReader(connection.getInputStream()); + try (BufferedReader reader = new BufferedReader(inputStreamReader)) { + CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES); + reader.read(charBuffer); + charBuffer.flip(); + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + charBuffer.toString() + ); + return PARSER.parse(parser, null); + } + } } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 95ae8003..dfd2f099 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -12,6 +12,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; @@ -53,6 +54,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.query.QueryBuilders; @@ -139,26 +142,29 @@ public CSVParser getDatabaseReader(final DatasourceManifest manifest) { return AccessController.doPrivileged((PrivilegedAction) () -> { try { URL zipUrl = new URL(manifest.getUrl()); - ZipInputStream zipIn = new ZipInputStream(zipUrl.openStream()); - ZipEntry zipEntry = zipIn.getNextEntry(); - while (zipEntry != null) { - if (zipEntry.getName().equalsIgnoreCase(manifest.getDbName()) == false) { - zipEntry = zipIn.getNextEntry(); - continue; - } - return new CSVParser(new BufferedReader(new InputStreamReader(zipIn)), CSVFormat.RFC4180); - } + return internalGetDatabaseReader(manifest, zipUrl.openConnection()); } catch (IOException e) { throw new OpenSearchException("failed to read geoip data from {}", manifest.getUrl(), e); } - throw new OpenSearchException( - "database file [{}] does not exist in the zip file [{}]", - manifest.getDbName(), - manifest.getUrl() - ); }); } + @VisibleForTesting + @SuppressForbidden(reason = "Need to connect to http endpoint to read GeoIP database file") + protected CSVParser internalGetDatabaseReader(final DatasourceManifest manifest, final URLConnection connection) throws IOException { + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + ZipInputStream zipIn = new ZipInputStream(connection.getInputStream()); + ZipEntry zipEntry = zipIn.getNextEntry(); + while (zipEntry != null) { + if (zipEntry.getName().equalsIgnoreCase(manifest.getDbName()) == false) { + zipEntry = zipIn.getNextEntry(); + continue; + } + return new CSVParser(new BufferedReader(new InputStreamReader(zipIn)), CSVFormat.RFC4180); + } + throw new OpenSearchException("database file [{}] does not exist in the zip file [{}]", manifest.getDbName(), manifest.getUrl()); + } + /** * Create a document in json string format to ingest in datasource database index * diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index db565dd8..b236e95c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -23,8 +23,7 @@ public class Ip2GeoSettings { */ public static final Setting DATASOURCE_ENDPOINT = Setting.simpleString( "plugins.geospatial.ip2geo.datasource.endpoint", - // TODO: This value is not correct. Update it later once CDN server is ready. - "https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", + "https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", new DatasourceEndpointValidator(), Setting.Property.NodeScope, Setting.Property.Dynamic diff --git a/src/main/java/org/opensearch/geospatial/shared/Constants.java b/src/main/java/org/opensearch/geospatial/shared/Constants.java new file mode 100644 index 00000000..7b6488a4 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/shared/Constants.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.shared; + +import java.util.Locale; + +import org.opensearch.Version; + +public class Constants { + public static final String USER_AGENT_KEY = "User-Agent"; + public static final String USER_AGENT_VALUE = String.format(Locale.ROOT, "OpenSearch/%s vanilla", Version.CURRENT.toString()); +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index 97c3cb7d..3ec81de5 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -69,7 +69,7 @@ public void testPrepareRequestDefaultValue() { verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof PutDatasourceRequest); PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; - assertEquals("https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", putDatasourceRequest.getEndpoint()); + assertEquals("https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", putDatasourceRequest.getEndpoint()); assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval()); assertEquals(datasourceName, putDatasourceRequest.getName()); isExecuted.set(true); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifestTests.java new file mode 100644 index 00000000..f7b689e1 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifestTests.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileInputStream; +import java.net.URLConnection; + +import lombok.SneakyThrows; + +import org.opensearch.common.SuppressForbidden; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.shared.Constants; + +@SuppressForbidden(reason = "unit test") +public class DatasourceManifestTests extends Ip2GeoTestCase { + + @SneakyThrows + public void testInternalBuild_whenCalled_thenCorrectUserAgentValueIsSet() { + URLConnection connection = mock(URLConnection.class); + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + when(connection.getInputStream()).thenReturn(new FileInputStream(manifestFile)); + + // Run + DatasourceManifest manifest = DatasourceManifest.Builder.internalBuild(connection); + + // Verify + verify(connection).addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + assertEquals("https://test.com/db.zip", manifest.getUrl()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 54c98a41..5a269539 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -14,6 +14,8 @@ import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; import java.io.File; +import java.io.FileInputStream; +import java.net.URLConnection; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -55,6 +57,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.shared.Constants; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -139,6 +142,28 @@ public void testGetDatabaseReaderNoFile() throws Exception { assertTrue(exception.getMessage().contains("does not exist")); } + @SneakyThrows + public void testInternalGetDatabaseReader_whenCalled_thenSetUserAgent() { + File zipFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.zip").getFile()); + DatasourceManifest manifest = new DatasourceManifest( + zipFile.toURI().toURL().toExternalForm(), + "sample_valid.csv", + "fake_sha256", + 1l, + Instant.now().toEpochMilli(), + "tester" + ); + + URLConnection connection = mock(URLConnection.class); + when(connection.getInputStream()).thenReturn(new FileInputStream(zipFile)); + + // Run + noOpsGeoIpDataFacade.internalGetDatabaseReader(manifest, connection); + + // Verify + verify(connection).addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + } + public void testDeleteIp2GeoDataIndex() { String index = String.format(Locale.ROOT, "%s.%s", IP2GEO_DATA_INDEX_NAME_PREFIX, GeospatialTestHelper.randomLowerCaseString()); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { diff --git a/src/test/resources/ip2geo/manifest.json b/src/test/resources/ip2geo/manifest.json index 4986fbd8..86a76e47 100644 --- a/src/test/resources/ip2geo/manifest.json +++ b/src/test/resources/ip2geo/manifest.json @@ -3,6 +3,6 @@ "db_name": "sample_valid.csv", "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, - "updated_at": 3134012341236, + "updated_at_in_epoch_milli": 3134012341236, "provider": "sample_provider" } \ No newline at end of file diff --git a/src/test/resources/ip2geo/manifest_invalid_url.json b/src/test/resources/ip2geo/manifest_invalid_url.json index 4e806f49..c9f1723e 100644 --- a/src/test/resources/ip2geo/manifest_invalid_url.json +++ b/src/test/resources/ip2geo/manifest_invalid_url.json @@ -3,6 +3,6 @@ "db_name": "sample_valid.csv", "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, - "updated_at": 3134012341236, + "updated_at_in_epoch_milli": 3134012341236, "provider": "sample_provider" } \ No newline at end of file diff --git a/src/test/resources/ip2geo/manifest_template.json b/src/test/resources/ip2geo/manifest_template.json index 92ceb590..39665b74 100644 --- a/src/test/resources/ip2geo/manifest_template.json +++ b/src/test/resources/ip2geo/manifest_template.json @@ -3,6 +3,6 @@ "db_name": "sample_valid.csv", "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, - "updated_at": 3134012341236, + "updated_at_in_epoch_milli": 3134012341236, "provider": "maxmind" } \ No newline at end of file From e42d65a37aafd0c691665156d2147be5478dc408 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 10 May 2023 10:01:39 -0700 Subject: [PATCH 28/61] Implement datasource update API (#292) Signed-off-by: Heemin Kim --- .../action/RestUpdateDatasourceHandler.java | 52 ++++ .../ip2geo/action/UpdateDatasourceAction.java | 27 ++ .../action/UpdateDatasourceRequest.java | 168 +++++++++++++ .../UpdateDatasourceTransportAction.java | 185 ++++++++++++++ .../jobscheduler/DatasourceUpdateService.java | 27 +- .../geospatial/plugin/GeospatialPlugin.java | 28 ++- .../geospatial/ip2geo/Ip2GeoTestCase.java | 9 +- .../action/RestPutDatasourceHandlerTests.java | 5 +- .../RestUpdateDatasourceHandlerTests.java | 56 +++++ .../action/UpdateDatasourceRequestTests.java | 121 +++++++++ .../UpdateDatasourceTransportActionTests.java | 231 ++++++++++++++++++ .../DatasourceUpdateServiceTests.java | 26 ++ .../plugin/GeospatialPluginTests.java | 2 + 13 files changed, 925 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceAction.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java new file mode 100644 index 00000000..77abae84 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.rest.RestRequest.Method.PUT; + +import java.io.IOException; +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +/** + * Rest handler for Ip2Geo datasource update request + */ +public class RestUpdateDatasourceHandler extends BaseRestHandler { + private static final String ACTION_NAME = "ip2geo_datasource_update"; + + @Override + public String getName() { + return ACTION_NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("name")); + if (request.hasContentOrSourceParam()) { + try (XContentParser parser = request.contentOrSourceParamParser()) { + PutDatasourceRequest.PARSER.parse(parser, putDatasourceRequest, null); + } + } + return channel -> client.executeLocally( + UpdateDatasourceAction.INSTANCE, + putDatasourceRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + String path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/{name}/_settings"); + return List.of(new Route(PUT, path)); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceAction.java new file mode 100644 index 00000000..96cd00df --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceAction.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.AcknowledgedResponse; + +/** + * Ip2Geo datasource update action + */ +public class UpdateDatasourceAction extends ActionType { + /** + * Update datasource action instance + */ + public static final UpdateDatasourceAction INSTANCE = new UpdateDatasourceAction(); + /** + * Update datasource action name + */ + public static final String NAME = "cluster:admin/geospatial/datasource/update"; + + private UpdateDatasourceAction() { + super(NAME, AcknowledgedResponse::new); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java new file mode 100644 index 00000000..fdc0b357 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java @@ -0,0 +1,168 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Locale; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.log4j.Log4j2; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; + +/** + * Ip2Geo datasource update request + */ +@Getter +@Setter +@Log4j2 +@EqualsAndHashCode(callSuper = false) +public class UpdateDatasourceRequest extends AcknowledgedRequest { + private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + private static final int MAX_DATASOURCE_NAME_BYTES = 255; + /** + * @param name the datasource name + * @return the datasource name + */ + private String name; + /** + * @param endpoint url to a manifest file for a datasource + * @return url to a manifest file for a datasource + */ + private String endpoint; + /** + * @param updateInterval update interval of a datasource + * @return update interval of a datasource + */ + private TimeValue updateInterval; + + /** + * Parser of a datasource + */ + public static final ObjectParser PARSER; + static { + PARSER = new ObjectParser<>("update_datasource"); + PARSER.declareString((request, val) -> request.setEndpoint(val), ENDPOINT_FIELD); + PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); + } + + /** + * Constructor + * @param name name of a datasource + */ + public UpdateDatasourceRequest(final String name) { + this.name = name; + } + + /** + * Constructor + * @param in the stream input + * @throws IOException IOException + */ + public UpdateDatasourceRequest(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.endpoint = in.readOptionalString(); + this.updateInterval = in.readOptionalTimeValue(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeOptionalString(endpoint); + out.writeOptionalTimeValue(updateInterval); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + if (endpoint == null && updateInterval == null) { + errors.addValidationError("no values to update"); + } + + validateEndpoint(errors); + validateUpdateInterval(errors); + + return errors.validationErrors().isEmpty() ? null : errors; + } + + /** + * Conduct following validation on endpoint + * 1. endpoint format complies with RFC-2396 + * 2. validate manifest file from the endpoint + * + * @param errors the errors to add error messages + */ + private void validateEndpoint(final ActionRequestValidationException errors) { + if (endpoint == null) { + return; + } + + try { + URL url = new URL(endpoint); + url.toURI(); // Validate URL complies with RFC-2396 + validateManifestFile(url, errors); + } catch (MalformedURLException | URISyntaxException e) { + log.info("Invalid URL[{}] is provided", endpoint, e); + errors.addValidationError("Invalid URL format is provided"); + } + } + + /** + * Conduct following validation on url + * 1. can read manifest file from the endpoint + * 2. the url in the manifest file complies with RFC-2396 + * + * @param url the url to validate + * @param errors the errors to add error messages + */ + private void validateManifestFile(final URL url, final ActionRequestValidationException errors) { + DatasourceManifest manifest; + try { + manifest = DatasourceManifest.Builder.build(url); + } catch (Exception e) { + log.info("Error occurred while reading a file from {}", url, e); + errors.addValidationError(String.format(Locale.ROOT, "Error occurred while reading a file from %s: %s", url, e.getMessage())); + return; + } + + try { + new URL(manifest.getUrl()).toURI(); // Validate URL complies with RFC-2396 + } catch (MalformedURLException | URISyntaxException e) { + log.info("Invalid URL[{}] is provided for url field in the manifest file", manifest.getUrl(), e); + errors.addValidationError("Invalid URL format is provided for url field in the manifest file"); + } + } + + /** + * Validate updateInterval is equal or larger than 1 + * + * @param errors the errors to add error messages + */ + private void validateUpdateInterval(final ActionRequestValidationException errors) { + if (updateInterval == null) { + return; + } + + if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { + errors.addValidationError("Update interval should be equal to or larger than 1 day"); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java new file mode 100644 index 00000000..20634552 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -0,0 +1,185 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.net.URL; +import java.security.InvalidParameterException; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Locale; + +import lombok.extern.log4j.Log4j2; + +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +/** + * Transport action to update datasource + */ +@Log4j2 +public class UpdateDatasourceTransportAction extends HandledTransportAction { + private static final long LOCK_DURATION_IN_SECONDS = 300l; + private final Ip2GeoLockService lockService; + private final DatasourceFacade datasourceFacade; + private final DatasourceUpdateService datasourceUpdateService; + + /** + * Constructor + * + * @param transportService the transport service + * @param actionFilters the action filters + * @param lockService the lock service + * @param datasourceFacade the datasource facade + * @param datasourceUpdateService the datasource update service + */ + @Inject + public UpdateDatasourceTransportAction( + final TransportService transportService, + final ActionFilters actionFilters, + final Ip2GeoLockService lockService, + final DatasourceFacade datasourceFacade, + final DatasourceUpdateService datasourceUpdateService + ) { + super(UpdateDatasourceAction.NAME, transportService, actionFilters, UpdateDatasourceRequest::new); + this.lockService = lockService; + this.datasourceUpdateService = datasourceUpdateService; + this.datasourceFacade = datasourceFacade; + } + + /** + * Get a lock and update datasource + * + * @param task the task + * @param request the request + * @param listener the listener + */ + @Override + protected void doExecute(final Task task, final UpdateDatasourceRequest request, final ActionListener listener) { + lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + return; + } + try { + Datasource datasource = datasourceFacade.getDatasource(request.getName()); + if (datasource == null) { + listener.onFailure(new ResourceNotFoundException("no such datasource exist")); + return; + } + validate(request, datasource); + updateIfChanged(request, datasource); + listener.onResponse(new AcknowledgedResponse(true)); + } catch (Exception e) { + listener.onFailure(e); + } finally { + lockService.releaseLock( + lock, + ActionListener.wrap( + released -> { log.info("Released lock for datasource[{}]", request.getName()); }, + exception -> { log.error("Failed to release the lock", exception); } + ) + ); + } + }, exception -> { listener.onFailure(exception); })); + } + + private void updateIfChanged(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { + boolean isChanged = false; + if (isEndpointChanged(request, datasource)) { + datasource.setEndpoint(request.getEndpoint()); + isChanged = true; + } + + if (isUpdateIntervalChanged(request, datasource)) { + datasource.setSchedule( + new IntervalSchedule(datasource.getSchedule().getStartTime(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS) + ); + isChanged = true; + } + + if (isChanged) { + datasourceFacade.updateDatasource(datasource); + } + } + + /** + * Additional validation based on an existing datasource + * + * Basic validation is done in UpdateDatasourceRequest#validate + * In this method we do additional validation based on an existing datasource + * + * 1. Check the compatibility of new fields and old fields + * 2. Check the updateInterval is less than validForInDays in datasource + * + * This method throws exception if one of validation fails. + * + * @param request the update request + * @param datasource the existing datasource + * @throws IOException the exception + */ + private void validate(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { + validateFieldsCompatibility(request, datasource); + validateUpdateIntervalIsLessThanValidForInDays(request, datasource); + } + + private void validateFieldsCompatibility(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { + if (isEndpointChanged(request, datasource) == false) { + return; + } + + List fields = datasourceUpdateService.getHeaderFields(request.getEndpoint()); + if (datasource.isCompatible(fields) == false) { + throw new OpenSearchException( + "new fields [{}] does not contain all old fields [{}]", + fields.toString(), + datasource.getDatabase().getFields().toString() + ); + } + } + + private void validateUpdateIntervalIsLessThanValidForInDays(final UpdateDatasourceRequest request, final Datasource datasource) + throws IOException { + if (isEndpointChanged(request, datasource) == false && isUpdateIntervalChanged(request, datasource) == false) { + return; + } + + long validForInDays = isEndpointChanged(request, datasource) + ? DatasourceManifest.Builder.build(new URL(request.getEndpoint())).getValidForInDays() + : datasource.getDatabase().getValidForInDays(); + + long updateInterval = isUpdateIntervalChanged(request, datasource) + ? request.getUpdateInterval().days() + : datasource.getSchedule().getInterval(); + + if (updateInterval >= validForInDays) { + throw new InvalidParameterException( + String.format(Locale.ROOT, "updateInterval %d should be smaller than %d", updateInterval, validForInDays) + ); + } + } + + private boolean isEndpointChanged(final UpdateDatasourceRequest request, final Datasource datasource) { + return request.getEndpoint() != null && request.getEndpoint().equals(datasource.getEndpoint()) == false; + } + + private boolean isUpdateIntervalChanged(final UpdateDatasourceRequest request, final Datasource datasource) { + return request.getUpdateInterval() != null && (int) request.getUpdateInterval().days() != datasource.getSchedule().getInterval(); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 57be4d4a..4d9530db 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -48,11 +48,15 @@ public DatasourceUpdateService( /** * Update GeoIp data * + * The first column is ip range field regardless its header name. + * Therefore, we don't store the first column's header name. + * * @param datasource the datasource * @param renewLock runnable to renew lock - * @throws Exception + * + * @throws IOException */ - public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws Exception { + public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws IOException { URL url = new URL(datasource.getEndpoint()); DatasourceManifest manifest = DatasourceManifest.Builder.build(url); @@ -91,6 +95,25 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable updateDatasourceAsSucceeded(datasource, manifest, fieldsToStore, startTime, endTime); } + /** + * Return header fields of geo data with given url of a manifest file + * + * The first column is ip range field regardless its header name. + * Therefore, we don't store the first column's header name. + * + * @param manifestUrl the url of a manifest file + * @return header fields of geo data + */ + public List getHeaderFields(String manifestUrl) throws IOException { + URL url = new URL(manifestUrl); + DatasourceManifest manifest = DatasourceManifest.Builder.build(url); + + try (CSVParser reader = geoIpDataFacade.getDatabaseReader(manifest)) { + String[] fields = reader.iterator().next().values(); + return Arrays.asList(fields).subList(1, fields.length); + } + } + /** * Delete all indices except the one which are being used * diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 4547e7f3..f014256d 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -45,6 +45,9 @@ import org.opensearch.geospatial.ip2geo.action.RestDeleteDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; +import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; +import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceAction; +import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; @@ -162,24 +165,39 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of( - new RestUploadStatsAction(), - new RestUploadGeoJSONAction(), + List geoJsonHandlers = List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction()); + + List ip2geoHandlers = List.of( new RestPutDatasourceHandler(clusterSettings), new RestGetDatasourceHandler(), + new RestUpdateDatasourceHandler(), new RestDeleteDatasourceHandler() ); + + List allHandlers = new ArrayList<>(); + allHandlers.addAll(geoJsonHandlers); + allHandlers.addAll(ip2geoHandlers); + return allHandlers; } @Override public List> getActions() { - return List.of( + List> geoJsonHandlers = List.of( new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class), - new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class), + new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class) + ); + + List> ip2geoHandlers = List.of( new ActionHandler<>(PutDatasourceAction.INSTANCE, PutDatasourceTransportAction.class), new ActionHandler<>(GetDatasourceAction.INSTANCE, GetDatasourceTransportAction.class), + new ActionHandler<>(UpdateDatasourceAction.INSTANCE, UpdateDatasourceTransportAction.class), new ActionHandler<>(DeleteDatasourceAction.INSTANCE, DeleteDatasourceTransportAction.class) ); + + List> allHandlers = new ArrayList<>(); + allHandlers.addAll(geoJsonHandlers); + allHandlers.addAll(ip2geoHandlers); + return allHandlers; } @Override diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 429bd031..25775a39 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -20,6 +20,8 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import lombok.SneakyThrows; + import org.junit.After; import org.junit.Before; import org.mockito.Mock; @@ -147,8 +149,9 @@ protected String randomIpAddress() { ); } + @SneakyThrows @SuppressForbidden(reason = "unit test") - protected String sampleManifestUrl() throws Exception { + protected String sampleManifestUrl() { return Paths.get(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").toURI()).toUri().toURL().toExternalForm(); } @@ -174,10 +177,10 @@ protected Datasource randomDatasource() { Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(30) + 1, ChronoUnit.DAYS)); + datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(10) + 1, ChronoUnit.DAYS)); datasource.setState(randomState()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); - datasource.setEndpoint(GeospatialTestHelper.randomLowerCaseString()); + datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase() .setFields(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase().setProvider(GeospatialTestHelper.randomLowerCaseString()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index 3ec81de5..f3689a69 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -9,6 +9,7 @@ import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; import java.util.HashSet; +import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; @@ -40,7 +41,7 @@ public void testPrepareRequest() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}"; RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath(String.format(path, datasourceName)) + .withPath(String.format(Locale.ROOT, path, datasourceName)) .withContent(new BytesArray(content), XContentType.JSON) .build(); AtomicBoolean isExecuted = new AtomicBoolean(false); @@ -62,7 +63,7 @@ public void testPrepareRequest() { public void testPrepareRequestDefaultValue() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath(String.format(path, datasourceName)) + .withPath(String.format(Locale.ROOT, path, datasourceName)) .withContent(new BytesArray("{}"), XContentType.JSON) .build(); AtomicBoolean isExecuted = new AtomicBoolean(false); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java new file mode 100644 index 00000000..6af74414 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; + +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Before; +import org.opensearch.common.bytes.BytesArray; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; + +public class RestUpdateDatasourceHandlerTests extends RestActionTestCase { + private String path; + private RestUpdateDatasourceHandler handler; + + @Before + public void setupAction() { + handler = new RestUpdateDatasourceHandler(); + controller().registerHandler(handler); + path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s/_settings"); + } + + public void testPrepareRequest_whenValidInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}"; + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath(String.format(Locale.ROOT, path, datasourceName)) + .withContent(new BytesArray(content), XContentType.JSON) + .build(); + AtomicBoolean isExecuted = new AtomicBoolean(false); + + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof PutDatasourceRequest); + PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; + assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); + assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval()); + assertEquals(datasourceName, putDatasourceRequest.getName()); + isExecuted.set(true); + return null; + }); + + dispatchRequest(request); + assertTrue(isExecuted.get()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java new file mode 100644 index 00000000..36ff271f --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.util.Locale; + +import lombok.SneakyThrows; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.Randomness; +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; + +public class UpdateDatasourceRequestTests extends Ip2GeoTestCase { + + public void testValidate_whenNullValues_thenFails() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertEquals("no values to update", exception.validationErrors().get(0)); + } + + public void testValidate_whenInvalidUrl_thenFails() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setEndpoint("invalidUrl"); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertEquals("Invalid URL format is provided", exception.validationErrors().get(0)); + } + + public void testValidate_whenInvalidManifestFile_thenFails() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String domain = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("Error occurred while reading a file")); + } + + @SneakyThrows + public void testValidate_whenValidInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + + // Run and verify + assertNull(request.validate()); + } + + @SneakyThrows + public void testValidate_whenZeroUpdateInterval_thenFails() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setUpdateInterval(TimeValue.timeValueDays(0)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertEquals( + String.format(Locale.ROOT, "Update interval should be equal to or larger than 1 day"), + exception.validationErrors().get(0) + ); + } + + @SneakyThrows + public void testValidate_whenInvalidUrlInsideManifest_thenFail() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setEndpoint(sampleManifestUrlWithInvalidUrl()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("Invalid URL format")); + } + + @SneakyThrows + public void testStreamInOut_whenValidInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String domain = GeospatialTestHelper.randomLowerCaseString(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasourceName); + request.setEndpoint(String.format(Locale.ROOT, "https://%s.com", domain)); + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(29) + 1)); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + UpdateDatasourceRequest copiedRequest = new UpdateDatasourceRequest(input); + + // Verify + assertEquals(request, copiedRequest); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java new file mode 100644 index 00000000..5364a147 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -0,0 +1,231 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.security.InvalidParameterException; +import java.util.List; + +import lombok.SneakyThrows; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.tasks.Task; + +public class UpdateDatasourceTransportActionTests extends Ip2GeoTestCase { + private UpdateDatasourceTransportAction action; + + @Before + public void init() { + action = new UpdateDatasourceTransportAction( + transportService, + actionFilters, + ip2GeoLockService, + datasourceFacade, + datasourceUpdateService + ); + } + + public void testDoExecute_whenFailedToAcquireLock_thenError() { + validateDoExecuteWithLockError(null); + } + + public void testDoExecute_whenExceptionToAcquireLock_thenError() { + validateDoExecuteWithLockError(new RuntimeException()); + } + + private void validateDoExecuteWithLockError(final Exception exception) { + Task task = mock(Task.class); + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + if (exception == null) { + // Run + captor.getValue().onResponse(null); + // Verify + verify(listener).onFailure(any(OpenSearchException.class)); + } else { + // Run + captor.getValue().onFailure(exception); + // Verify + verify(listener).onFailure(exception); + } + } + + @SneakyThrows + public void testDoExecute_whenValidInput_thenUpdate() { + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval() + 1)); + request.setEndpoint(sampleManifestUrl()); + + Task task = mock(Task.class); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceUpdateService.getHeaderFields(request.getEndpoint())).thenReturn(datasource.getDatabase().getFields()); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + verify(datasourceFacade).getDatasource(datasource.getName()); + verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceUpdateService).getHeaderFields(request.getEndpoint()); + assertEquals(request.getEndpoint(), datasource.getEndpoint()); + assertEquals(request.getUpdateInterval().days(), datasource.getSchedule().getInterval()); + verify(listener).onResponse(new AcknowledgedResponse(true)); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + + @SneakyThrows + public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval())); + request.setEndpoint(datasource.getEndpoint()); + + Task task = mock(Task.class); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + verify(datasourceFacade).getDatasource(datasource.getName()); + verify(datasourceUpdateService, never()).getHeaderFields(anyString()); + verify(datasourceFacade, never()).updateDatasource(datasource); + verify(listener).onResponse(new AcknowledgedResponse(true)); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + + @SneakyThrows + public void testDoExecute_whenNoDatasource_thenError() { + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + + Task task = mock(Task.class); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener).onFailure(exceptionCaptor.capture()); + assertEquals(ResourceNotFoundException.class, exceptionCaptor.getValue().getClass()); + exceptionCaptor.getValue().getMessage().contains("no such datasource exist"); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + + @SneakyThrows + public void testDoExecute_whenIncompatibleFields_thenError() { + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + request.setEndpoint(sampleManifestUrl()); + + Task task = mock(Task.class); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + List newFields = datasource.getDatabase().getFields().subList(0, 0); + when(datasourceUpdateService.getHeaderFields(request.getEndpoint())).thenReturn(newFields); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener).onFailure(exceptionCaptor.capture()); + assertEquals(OpenSearchException.class, exceptionCaptor.getValue().getClass()); + exceptionCaptor.getValue().getMessage().contains("does not contain"); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } + + @SneakyThrows + public void testDoExecute_whenInvalidUpdateInterval_thenError() { + Datasource datasource = randomDatasource(); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + request.setUpdateInterval(TimeValue.timeValueDays(datasource.getDatabase().getValidForInDays())); + + Task task = mock(Task.class); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener).onFailure(exceptionCaptor.capture()); + assertEquals(InvalidParameterException.class, exceptionCaptor.getValue().getClass()); + exceptionCaptor.getValue().getMessage().contains("should be smaller"); + verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 6e51c926..108029fe 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -19,6 +19,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import lombok.SneakyThrows; @@ -139,6 +140,17 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { ); } + @SneakyThrows + public void testGetHeaderFields_whenValidInput_thenReturnCorrectValue() { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + // Run + assertEquals(Arrays.asList("country_name"), datasourceUpdateService.getHeaderFields(manifestFile.toURI().toURL().toExternalForm())); + } + @SneakyThrows public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); @@ -165,4 +177,18 @@ public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { assertEquals(currentIndex, datasource.getIndices().get(0)); verify(datasourceFacade).updateDatasource(datasource); } + + @SneakyThrows + public void testGetHeaderFields_whenValidInput_thenSucceed() { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + // Run + List fields = datasourceUpdateService.getHeaderFields(manifestFile.toURI().toURL().toExternalForm()); + + // Verify + List expectedFields = Arrays.asList("country_name"); + assertEquals(expectedFields, fields); + } } diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 1913d830..596bd3f7 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -35,6 +35,7 @@ import org.opensearch.geospatial.ip2geo.action.RestDeleteDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; +import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; @@ -64,6 +65,7 @@ public class GeospatialPluginTests extends OpenSearchTestCase { new RestUploadStatsAction(), new RestPutDatasourceHandler(clusterSettings), new RestGetDatasourceHandler(), + new RestUpdateDatasourceHandler(), new RestDeleteDatasourceHandler() ); From befcf47727fa56463d83879d5f30c93751917b09 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 11 May 2023 14:13:22 -0700 Subject: [PATCH 29/61] Refactoring test code (#302) Make buildGeoJSONFeatureProcessorConfig method to be more general Signed-off-by: Heemin Kim --- .../opensearch/geospatial/GeospatialRestTestCase.java | 9 ++++----- .../geospatial/processor/FeatureProcessorIT.java | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 1e5c1439..af6f33c5 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -47,7 +47,6 @@ import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapper; import org.opensearch.geospatial.index.query.xyshape.XYShapeQueryBuilder; -import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.ingest.Pipeline; @@ -75,7 +74,7 @@ private static String buildPipelinePath(String name) { return String.join(URL_DELIMITER, "_ingest", "pipeline", name); } - protected static void createPipeline(String name, Optional description, List> processorConfigs) + protected static Response createPipeline(String name, Optional description, List> processorConfigs) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); if (description.isPresent()) { @@ -88,7 +87,7 @@ protected static void createPipeline(String name, Optional description, Request request = new Request("PUT", buildPipelinePath(name)); request.setJsonEntity(org.opensearch.common.Strings.toString(builder)); - client().performRequest(request); + return client().performRequest(request); } protected static void deletePipeline(String name) throws IOException { @@ -137,9 +136,9 @@ public static String indexDocument(String indexName, String docID, String body, return docID; } - protected Map buildGeoJSONFeatureProcessorConfig(Map properties) { + protected Map buildProcessorConfig(final String processorType, final Map properties) { Map featureProcessor = new HashMap<>(); - featureProcessor.put(FeatureProcessor.TYPE, properties); + featureProcessor.put(processorType, properties); return featureProcessor; } diff --git a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java index 585f8628..7034a633 100644 --- a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java @@ -49,7 +49,7 @@ public void testIndexGeoJSONSuccess() throws Exception { Map processorProperties = new HashMap<>(); processorProperties.put(FeatureProcessor.FIELD_KEY, geoShapeField); - Map geoJSONProcessorConfig = buildGeoJSONFeatureProcessorConfig(processorProperties); + Map geoJSONProcessorConfig = buildProcessorConfig(FeatureProcessor.TYPE, processorProperties); List> configs = new ArrayList<>(); configs.add(geoJSONProcessorConfig); From 0da53eb9b9aceb64f90641437d08c490c30f6848 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 11 May 2023 15:18:05 -0700 Subject: [PATCH 30/61] Add ip2geo processor integ test for failure case (#303) Signed-off-by: Heemin Kim --- .../ip2geo/processor/Ip2GeoProcessor.java | 20 ++++--- .../ip2geo/processor/Ip2GeoProcessorIT.java | 54 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index acd56150..83383158 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -43,6 +43,14 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private static final Map DATA_EXPIRED = Map.of("error", "ip2geo_data_expired"); private static final String PROPERTY_IP = "ip"; + + public static final String CONFIG_FIELD = "field"; + public static final String CONFIG_TARGET_FIELD = "target_field"; + public static final String CONFIG_DATASOURCE = "datasource"; + public static final String CONFIG_PROPERTIES = "target_field"; + public static final String CONFIG_IGNORE_MISSING = "ignore_missing"; + public static final String CONFIG_FIRST_ONLY = "first_only"; + private final String field; private final String targetField; /** @@ -352,12 +360,12 @@ public Ip2GeoProcessor create( final String description, final Map config ) throws IOException { - String ipField = readStringProperty(TYPE, processorTag, config, "field"); - String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "ip2geo"); - String datasourceName = readStringProperty(TYPE, processorTag, config, "datasource"); - List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); - boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); + String ipField = readStringProperty(TYPE, processorTag, config, CONFIG_FIELD); + String targetField = readStringProperty(TYPE, processorTag, config, CONFIG_TARGET_FIELD, "ip2geo"); + String datasourceName = readStringProperty(TYPE, processorTag, config, CONFIG_DATASOURCE); + List propertyNames = readOptionalList(TYPE, processorTag, config, CONFIG_PROPERTIES); + boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, CONFIG_IGNORE_MISSING, false); + boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, CONFIG_FIRST_ONLY, true); // Skip validation for the call by cluster applier service if (Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME) == false) { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java new file mode 100644 index 00000000..34430d06 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.processor; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import lombok.SneakyThrows; + +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.geospatial.GeospatialRestTestCase; +import org.opensearch.geospatial.GeospatialTestHelper; + +public class Ip2GeoProcessorIT extends GeospatialRestTestCase { + + @SneakyThrows + public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { + String pipelineName = GeospatialTestHelper.randomLowerCaseString(); + + // Run + ResponseException exception = expectThrows( + ResponseException.class, + () -> createIp2GeoProcessorPipeline(pipelineName, Collections.emptyMap()) + ); + + // Verify + assertTrue(exception.getMessage().contains("doesn't exist")); + } + + private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map properties) throws IOException { + String field = GeospatialTestHelper.randomLowerCaseString(); + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Map defaultProperties = Map.of( + Ip2GeoProcessor.CONFIG_FIELD, + field, + Ip2GeoProcessor.CONFIG_DATASOURCE, + datasourceName + ); + Map baseProperties = new HashMap<>(); + baseProperties.putAll(defaultProperties); + baseProperties.putAll(properties); + Map processorConfig = buildProcessorConfig(Ip2GeoProcessor.TYPE, baseProperties); + + return createPipeline(pipelineName, Optional.empty(), Arrays.asList(processorConfig)); + } +} From 8d5c6bf33054a4a4f184a818a356338c02ceb81e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 12 May 2023 14:50:15 -0700 Subject: [PATCH 31/61] Bug fix and refactoring of code (#305) 1. Bugfix: Ingest metadata can be null if there is no processor created 2. Refactoring: Moved private method to another class for better testing support 3. Refactoring: Set some private static final variable as public so that unit test can use it 4. Refactoring: Changed string value to static variable Signed-off-by: Heemin Kim --- .../DeleteDatasourceTransportAction.java | 25 ++---- .../ip2geo/common/Ip2GeoProcessorFacade.java | 37 +++++++++ .../ip2geo/processor/Ip2GeoProcessor.java | 2 +- .../geospatial/ip2geo/Ip2GeoTestCase.java | 32 +++++++- .../DeleteDatasourceTransportActionTests.java | 69 ++++------------ .../common/Ip2GeoProcessorFacadeTests.java | 78 +++++++++++++++++++ 6 files changed, 167 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index 2acdfd52..27055d41 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -20,9 +20,8 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; -import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -36,6 +35,7 @@ public class DeleteDatasourceTransportAction extends HandledTransportAction ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class).stream()) - .filter(ip2GeoProcessor -> ip2GeoProcessor.getDatasourceName().equals(datasource.getName())) - .findAny() - .isEmpty(); - } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java new file mode 100644 index 00000000..0f1b764f --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.IngestService; + +public class Ip2GeoProcessorFacade { + private final IngestService ingestService; + + @Inject + public Ip2GeoProcessorFacade(final IngestService ingestService) { + this.ingestService = ingestService; + } + + public List getProcessors(final String datasourceName) { + IngestMetadata ingestMetadata = ingestService.getClusterService().state().getMetadata().custom(IngestMetadata.TYPE); + if (ingestMetadata == null) { + return Collections.emptyList(); + } + return ingestMetadata.getPipelines() + .keySet() + .stream() + .flatMap(pipelineId -> ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class).stream()) + .filter(ip2GeoProcessor -> ip2GeoProcessor.getDatasourceName().equals(datasourceName)) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 83383158..b2c7e678 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -47,7 +47,7 @@ public final class Ip2GeoProcessor extends AbstractProcessor { public static final String CONFIG_FIELD = "field"; public static final String CONFIG_TARGET_FIELD = "target_field"; public static final String CONFIG_DATASOURCE = "datasource"; - public static final String CONFIG_PROPERTIES = "target_field"; + public static final String CONFIG_PROPERTIES = "properties"; public static final String CONFIG_IGNORE_MISSING = "ignore_missing"; public static final String CONFIG_FIRST_ONLY = "first_only"; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 25775a39..707c43a9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Locale; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -46,9 +47,11 @@ import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; import org.opensearch.jobscheduler.spi.LockModel; @@ -86,6 +89,8 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected TransportService transportService; @Mock protected Ip2GeoLockService ip2GeoLockService; + @Mock + protected Ip2GeoProcessorFacade ip2GeoProcessorFacade; protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; @@ -174,10 +179,11 @@ protected long randomPositiveLong() { } protected Datasource randomDatasource() { + int validForInDays = Randomness.get().nextInt(30) + 2; Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(10) + 1, ChronoUnit.DAYS)); + datasource.setSchedule(new IntervalSchedule(now, validForInDays - 1, ChronoUnit.DAYS)); datasource.setState(randomState()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); @@ -186,7 +192,7 @@ protected Datasource randomDatasource() { datasource.getDatabase().setProvider(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setUpdatedAt(now); datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); - datasource.getDatabase().setValidForInDays(Randomness.get().nextInt(30) + 1l); + datasource.getDatabase().setValidForInDays((long) validForInDays); datasource.getUpdateStats().setLastSkippedAt(now); datasource.getUpdateStats().setLastSucceededAt(now); datasource.getUpdateStats().setLastFailedAt(now); @@ -211,6 +217,28 @@ protected LockModel randomLockModel() { return lockModel; } + protected Ip2GeoProcessor randomIp2GeoProcessor(String datasourceName) { + String tag = GeospatialTestHelper.randomLowerCaseString(); + String description = GeospatialTestHelper.randomLowerCaseString(); + String field = GeospatialTestHelper.randomLowerCaseString(); + String targetField = GeospatialTestHelper.randomLowerCaseString(); + Set properties = Set.of(GeospatialTestHelper.randomLowerCaseString()); + Ip2GeoProcessor ip2GeoProcessor = new Ip2GeoProcessor( + tag, + description, + field, + targetField, + datasourceName, + properties, + true, + true, + clusterSettings, + datasourceFacade, + geoIpDataFacade + ); + return ip2GeoProcessor; + } + /** * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) * is merged in OpenSearch core diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java index 535e5422..54f9ed61 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -15,14 +15,9 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; import lombok.SneakyThrows; @@ -32,15 +27,10 @@ import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; -import org.opensearch.ingest.IngestMetadata; -import org.opensearch.ingest.PipelineConfiguration; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.tasks.Task; @@ -49,7 +39,14 @@ public class DeleteDatasourceTransportActionTests extends Ip2GeoTestCase { @Before public void init() { - action = new DeleteDatasourceTransportAction(transportService, actionFilters, ip2GeoLockService, ingestService, datasourceFacade); + action = new DeleteDatasourceTransportAction( + transportService, + actionFilters, + ip2GeoLockService, + ingestService, + datasourceFacade, + ip2GeoProcessorFacade + ); } @SneakyThrows @@ -113,6 +110,7 @@ public void testDeleteDatasource_whenNull_thenThrowException() { public void testDeleteDatasource_whenSafeToDelete_thenDelete() { Datasource datasource = randomDatasource(); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn(Collections.emptyList()); // Run action.deleteDatasource(datasource.getName()); @@ -128,14 +126,8 @@ public void testDeleteDatasource_whenProcessorIsUsingDatasource_thenThrowExcepti Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); - - String pipelineId = GeospatialTestHelper.randomLowerCaseString(); - Map pipelines = new HashMap<>(); - pipelines.put(pipelineId, createPipelineConfiguration()); - IngestMetadata ingestMetadata = new IngestMetadata(pipelines); - when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); - when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn( - Arrays.asList(createIp2GeoProcessor(datasource.getName())) + when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn( + Arrays.asList(randomIp2GeoProcessor(datasource.getName())) ); // Run @@ -152,15 +144,9 @@ public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowE Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); - - String pipelineId = GeospatialTestHelper.randomLowerCaseString(); - Map pipelines = new HashMap<>(); - pipelines.put(pipelineId, createPipelineConfiguration()); - IngestMetadata ingestMetadata = new IngestMetadata(pipelines); - when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); - when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn( + when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn( Collections.emptyList(), - Arrays.asList(createIp2GeoProcessor(datasource.getName())) + Arrays.asList(randomIp2GeoProcessor(datasource.getName())) ); // Run @@ -170,33 +156,4 @@ public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowE verify(datasourceFacade, times(2)).updateDatasource(datasource); verify(datasourceFacade, never()).deleteDatasource(datasource); } - - private PipelineConfiguration createPipelineConfiguration() { - String id = GeospatialTestHelper.randomLowerCaseString(); - ByteBuffer byteBuffer = ByteBuffer.wrap(GeospatialTestHelper.randomLowerCaseString().getBytes(StandardCharsets.US_ASCII)); - BytesReference config = BytesReference.fromByteBuffer(byteBuffer); - return new PipelineConfiguration(id, config, XContentType.JSON); - } - - private Ip2GeoProcessor createIp2GeoProcessor(String datasourceName) { - String tag = GeospatialTestHelper.randomLowerCaseString(); - String description = GeospatialTestHelper.randomLowerCaseString(); - String field = GeospatialTestHelper.randomLowerCaseString(); - String targetField = GeospatialTestHelper.randomLowerCaseString(); - Set properties = Set.of(GeospatialTestHelper.randomLowerCaseString()); - Ip2GeoProcessor ip2GeoProcessor = new Ip2GeoProcessor( - tag, - description, - field, - targetField, - datasourceName, - properties, - true, - true, - clusterSettings, - datasourceFacade, - geoIpDataFacade - ); - return ip2GeoProcessor; - } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java new file mode 100644 index 00000000..05fed2ca --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import static org.mockito.Mockito.when; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.PipelineConfiguration; + +public class Ip2GeoProcessorFacadeTests extends Ip2GeoTestCase { + private Ip2GeoProcessorFacade ip2GeoProcessorFacade; + + @Before + public void init() { + ip2GeoProcessorFacade = new Ip2GeoProcessorFacade(ingestService); + } + + public void testGetProcessors_whenNullMetadata_thenReturnEmpty() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(null); + + List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceName); + assertTrue(ip2GeoProcessorList.isEmpty()); + } + + public void testGetProcessors_whenNoProcessorForGivenDatasource_thenReturnEmpty() { + String datasourceBeingUsed = GeospatialTestHelper.randomLowerCaseString(); + String datasourceNotBeingUsed = GeospatialTestHelper.randomLowerCaseString(); + String pipelineId = GeospatialTestHelper.randomLowerCaseString(); + Map pipelines = new HashMap<>(); + pipelines.put(pipelineId, createPipelineConfiguration()); + IngestMetadata ingestMetadata = new IngestMetadata(pipelines); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + Ip2GeoProcessor ip2GeoProcessor = randomIp2GeoProcessor(datasourceBeingUsed); + when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn(Arrays.asList(ip2GeoProcessor)); + + List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceNotBeingUsed); + assertTrue(ip2GeoProcessorList.isEmpty()); + } + + public void testGetProcessors_whenProcessorsForGivenDatasource_thenReturnProcessors() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String pipelineId = GeospatialTestHelper.randomLowerCaseString(); + Map pipelines = new HashMap<>(); + pipelines.put(pipelineId, createPipelineConfiguration()); + IngestMetadata ingestMetadata = new IngestMetadata(pipelines); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + Ip2GeoProcessor ip2GeoProcessor = randomIp2GeoProcessor(datasourceName); + when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn(Arrays.asList(ip2GeoProcessor)); + + List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceName); + assertEquals(1, ip2GeoProcessorList.size()); + assertEquals(ip2GeoProcessor.getDatasourceName(), ip2GeoProcessorList.get(0).getDatasourceName()); + } + + private PipelineConfiguration createPipelineConfiguration() { + String id = GeospatialTestHelper.randomLowerCaseString(); + ByteBuffer byteBuffer = ByteBuffer.wrap(GeospatialTestHelper.randomLowerCaseString().getBytes(StandardCharsets.US_ASCII)); + BytesReference config = BytesReference.fromByteBuffer(byteBuffer); + return new PipelineConfiguration(id, config, XContentType.JSON); + } +} From 3adb06ab2b40b77b3550a071872a027ef7471213 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 15 May 2023 13:06:54 -0700 Subject: [PATCH 32/61] Add integration test for Ip2GeoProcessor (#306) Signed-off-by: Heemin Kim --- .../exceptions/ResourceInUseException.java | 35 ++++ .../DeleteDatasourceTransportAction.java | 5 +- .../ip2geo/action/PutDatasourceRequest.java | 4 +- .../geospatial/GeospatialRestTestCase.java | 87 ++++++++- .../geospatial/ip2geo/Ip2GeoDataServer.java | 127 +++++++++++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 4 +- .../UpdateDatasourceTransportActionTests.java | 4 +- .../ip2geo/processor/Ip2GeoProcessorIT.java | 169 +++++++++++++++++- .../processor/FeatureProcessorIT.java | 2 +- .../resources/ip2geo/server/city/city.zip | Bin 0 -> 294 bytes .../ip2geo/server/city/manifest.json | 8 + .../ip2geo/server/city/manifest_local.json | 8 + .../ip2geo/server/country/country.zip | Bin 0 -> 263 bytes .../ip2geo/server/country/manifest.json | 8 + .../ip2geo/server/country/manifest_local.json | 8 + 15 files changed, 456 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoDataServer.java create mode 100644 src/test/resources/ip2geo/server/city/city.zip create mode 100644 src/test/resources/ip2geo/server/city/manifest.json create mode 100644 src/test/resources/ip2geo/server/city/manifest_local.json create mode 100644 src/test/resources/ip2geo/server/country/country.zip create mode 100644 src/test/resources/ip2geo/server/country/manifest.json create mode 100644 src/test/resources/ip2geo/server/country/manifest_local.json diff --git a/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java b/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java new file mode 100644 index 00000000..c606abea --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import java.io.IOException; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.rest.RestStatus; + +/** + * Generic ResourceInUseException corresponding to the {@link RestStatus#BAD_REQUEST} status code + */ +public class ResourceInUseException extends OpenSearchException { + + public ResourceInUseException(String msg, Object... args) { + super(msg, args); + } + + public ResourceInUseException(String msg, Throwable cause, Object... args) { + super(msg, cause, args); + } + + public ResourceInUseException(StreamInput in) throws IOException { + super(in); + } + + @Override + public final RestStatus status() { + return RestStatus.BAD_REQUEST; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index 27055d41..753c107d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -17,6 +17,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.exceptions.ResourceInUseException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; @@ -105,7 +106,7 @@ protected void deleteDatasource(final String datasourceName) throws IOException private void setDatasourceStateAsDeleting(final Datasource datasource) { if (ip2GeoProcessorFacade.getProcessors(datasource.getName()).isEmpty() == false) { - throw new OpenSearchException("datasource is being used by one of processors"); + throw new ResourceInUseException("datasource is being used by one of processors"); } DatasourceState previousState = datasource.getState(); @@ -119,7 +120,7 @@ private void setDatasourceStateAsDeleting(final Datasource datasource) { if (ip2GeoProcessorFacade.getProcessors(datasource.getName()).isEmpty() == false) { datasource.setState(previousState); datasourceFacade.updateDatasource(datasource); - throw new OpenSearchException("datasource is being used by one of processors"); + throw new ResourceInUseException("datasource is being used by one of processors"); } } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 3347a243..3426008b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -36,9 +36,9 @@ @Log4j2 @EqualsAndHashCode public class PutDatasourceRequest extends AcknowledgedRequest { - private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); - private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); private static final int MAX_DATASOURCE_NAME_BYTES = 255; + public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); /** * @param name the datasource name * @return the datasource name diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index af6f33c5..5c5aecc4 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -17,9 +17,12 @@ import static org.opensearch.search.aggregations.Aggregations.AGGREGATIONS_FIELD; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; @@ -47,11 +50,11 @@ import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; import org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapper; import org.opensearch.geospatial.index.query.xyshape.XYShapeQueryBuilder; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.ingest.Pipeline; public abstract class GeospatialRestTestCase extends OpenSearchSecureRestTestCase { - public static final String SOURCE = "_source"; public static final String DOC = "_doc"; public static final String URL_DELIMITER = "/"; @@ -69,11 +72,23 @@ public abstract class GeospatialRestTestCase extends OpenSearchSecureRestTestCas public static final String SHAPE_ID_FIELD = "id"; public static final String SHAPE_INDEX_PATH_FIELD = "path"; public static final String QUERY_PARAM_TOKEN = "?"; + private static final String SETTINGS = "_settings"; + private static final String SIMULATE = "_simulate"; + private static final String DOCS = "docs"; + private static final String DATASOURCES = "datasources"; + private static final String STATE = "state"; + private static final String PUT = "PUT"; + private static final String GET = "GET"; + private static final String DELETE = "DELETE"; private static String buildPipelinePath(String name) { return String.join(URL_DELIMITER, "_ingest", "pipeline", name); } + private static String buildDatasourcePath(String name) { + return String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource", name); + } + protected static Response createPipeline(String name, Optional description, List> processorConfigs) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -95,6 +110,74 @@ protected static void deletePipeline(String name) throws IOException { client().performRequest(request); } + protected Response createDatasource(final String name, Map properties) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + for (Map.Entry config : properties.entrySet()) { + builder.field(config.getKey(), config.getValue()); + } + builder.endObject(); + + Request request = new Request(PUT, buildDatasourcePath(name)); + request.setJsonEntity(Strings.toString(builder)); + return client().performRequest(request); + } + + protected void waitForDatasourceToBeAvailable(final String name, final Duration timeout) throws Exception { + Instant start = Instant.now(); + while (DatasourceState.AVAILABLE.equals(getDatasourceState(name)) == false) { + if (Duration.between(start, Instant.now()).compareTo(timeout) > 0) { + throw new RuntimeException( + String.format( + Locale.ROOT, + "Datasource state didn't change to %s after %d seconds", + DatasourceState.AVAILABLE.name(), + timeout.toSeconds() + ) + ); + } + Thread.sleep(1000); + } + } + + private DatasourceState getDatasourceState(final String name) throws Exception { + List> datasources = (List>) getDatasource(name).get(DATASOURCES); + return DatasourceState.valueOf((String) datasources.get(0).get(STATE)); + } + + protected Response deleteDatasource(final String name) throws IOException { + Request request = new Request(DELETE, buildDatasourcePath(name)); + return client().performRequest(request); + } + + protected Map getDatasource(final String name) throws Exception { + Request request = new Request(GET, buildDatasourcePath(name)); + Response response = client().performRequest(request); + return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); + } + + protected Response updateDatasource(final String name, Map config) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + if (config != null && !config.isEmpty()) { + builder.value(config); + } + builder.endObject(); + + Request request = new Request(PUT, String.join(URL_DELIMITER, buildDatasourcePath(name), SETTINGS)); + request.setJsonEntity(Strings.toString(builder)); + return client().performRequest(request); + } + + protected Map simulatePipeline(final String name, List docs) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.field(DOCS, docs); + builder.endObject(); + + Request request = new Request(GET, String.join(URL_DELIMITER, buildPipelinePath(name), SIMULATE)); + request.setJsonEntity(Strings.toString(builder)); + Response response = client().performRequest(request); + return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); + } + protected static void createIndex(String name, Settings settings, Map fieldMap) throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject(MAPPING_PROPERTIES_KEY); for (Map.Entry entry : fieldMap.entrySet()) { @@ -136,7 +219,7 @@ public static String indexDocument(String indexName, String docID, String body, return docID; } - protected Map buildProcessorConfig(final String processorType, final Map properties) { + protected Map buildProcessorConfig(final String processorType, final Map properties) { Map featureProcessor = new HashMap<>(); featureProcessor.put(processorType, properties); return featureProcessor; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoDataServer.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoDataServer.java new file mode 100644 index 00000000..ba1e4909 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoDataServer.java @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Paths; + +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; + +import org.opensearch.common.SuppressForbidden; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +/** + * Simple http server to serve static files under test/java/resources/ip2geo/server for integration testing + */ +@Log4j2 +@SuppressForbidden(reason = "used only for testing") +public class Ip2GeoDataServer { + private static final String SYS_PROPERTY_KEY_CLUSTER_ENDPOINT = "tests.rest.cluster"; + private static final String LOCAL_CLUSTER_ENDPOINT = "127.0.0.1"; + private static final String ROOT = "ip2geo/server"; + private static final int PORT = 8001; + private static final String EXTERNAL_ENDPOINT_PREFIX = + "https://github.com/opensearch-project/geospatial/blob/main/src/test/resources/ip2geo/server"; + + private static HttpServer server; + private static volatile int counter = 0; + private static String endpointPrefix = "http://localhost:" + PORT; + private static String cityFilePath = endpointPrefix + "/city/manifest_local.json"; + private static String countryFilePath = endpointPrefix + "/country/manifest_local.json"; + + /** + * Return an endpoint to a manifest file for a sample city data + * The sample data should contain three lines as follows + * + * cidr,city,country + * 10.0.0.0/8,Seattle,USA + * 127.0.0.0/12,Vancouver,Canada + * fd12:2345:6789:1::/64,Bengaluru,India + * + */ + public static String getEndpointCity() { + return cityFilePath; + } + + /** + * Return an endpoint to a manifest file for a sample country data + * The sample data should contain three lines as follows + * + * cidr,country + * 10.0.0.0/8,USA + * 127.0.0.0/12,Canada + * fd12:2345:6789:1::/64,India + * + */ + public static String getEndpointCountry() { + return countryFilePath; + } + + @SneakyThrows + synchronized public static void start() { + log.info("Start server is called"); + // If it is remote cluster test, use external endpoint and do not launch local server + if (System.getProperty(SYS_PROPERTY_KEY_CLUSTER_ENDPOINT).contains(LOCAL_CLUSTER_ENDPOINT) == false) { + log.info("Remote cluster[{}] testing. Skip launching local server", System.getProperty(SYS_PROPERTY_KEY_CLUSTER_ENDPOINT)); + cityFilePath = EXTERNAL_ENDPOINT_PREFIX + "/city/manifest.json"; + countryFilePath = EXTERNAL_ENDPOINT_PREFIX + "/country/manifest.json"; + return; + } + + counter++; + if (server != null) { + log.info("Server has started already"); + return; + } + server = HttpServer.create(new InetSocketAddress("localhost", PORT), 0); + server.createContext("/", new Ip2GeoHttpHandler()); + server.start(); + log.info("Local file server started on port {}", PORT); + } + + synchronized public static void stop() { + log.info("Stop server is called"); + if (server == null) { + log.info("Server has stopped already"); + return; + } + counter--; + if (counter > 0) { + log.info("[{}] processors are still using the server", counter); + return; + } + + server.stop(0); + server = null; + log.info("Server stopped"); + } + + @SuppressForbidden(reason = "used only for testing") + private static class Ip2GeoHttpHandler implements HttpHandler { + @Override + public void handle(final HttpExchange exchange) throws IOException { + try { + byte[] data = Files.readAllBytes( + Paths.get(this.getClass().getClassLoader().getResource(ROOT + exchange.getRequestURI().getPath()).toURI()) + ); + exchange.sendResponseHeaders(200, data.length); + OutputStream outputStream = exchange.getResponseBody(); + outputStream.write(data); + outputStream.flush(); + outputStream.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 707c43a9..4e76d6d4 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -179,11 +179,11 @@ protected long randomPositiveLong() { } protected Datasource randomDatasource() { - int validForInDays = Randomness.get().nextInt(30) + 2; + int validForInDays = Randomness.get().nextInt(30); Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setSchedule(new IntervalSchedule(now, validForInDays - 1, ChronoUnit.DAYS)); + datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(29), ChronoUnit.DAYS)); datasource.setState(randomState()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 5364a147..2d3a5602 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -25,6 +25,7 @@ import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.Randomness; import org.opensearch.common.unit.TimeValue; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; @@ -83,8 +84,9 @@ private void validateDoExecuteWithLockError(final Exception exception) { public void testDoExecute_whenValidInput_thenUpdate() { Datasource datasource = randomDatasource(); UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); - request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval() + 1)); request.setEndpoint(sampleManifestUrl()); + // Sample manifest has validForDays of 30. Update interval should be less than that. + request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(29))); Task task = mock(Task.class); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index 34430d06..9c59039c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -6,20 +6,31 @@ package org.opensearch.geospatial.ip2geo.processor; import java.io.IOException; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import lombok.SneakyThrows; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.common.Randomness; import org.opensearch.geospatial.GeospatialRestTestCase; import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; +import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; public class Ip2GeoProcessorIT extends GeospatialRestTestCase { + private static final String CITY = "city"; + private static final String COUNTRY = "country"; + private static final String IP = "ip"; + private static final String SOURCE = "_source"; @SneakyThrows public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { @@ -33,22 +44,174 @@ public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { // Verify assertTrue(exception.getMessage().contains("doesn't exist")); + assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); } - private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map properties) throws IOException { + @SneakyThrows + public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { + Ip2GeoDataServer.start(); + try { + String pipelineName = GeospatialTestHelper.randomLowerCaseString(); + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String targetField = GeospatialTestHelper.randomLowerCaseString(); + String field = GeospatialTestHelper.randomLowerCaseString(); + + Map datasourceProperties = Map.of( + PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + Ip2GeoDataServer.getEndpointCity() + ); + + // Create datasource and wait for it to be available + createDatasource(datasourceName, datasourceProperties); + waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10)); + + Map processorProperties = Map.of( + Ip2GeoProcessor.CONFIG_FIELD, + field, + Ip2GeoProcessor.CONFIG_DATASOURCE, + datasourceName, + Ip2GeoProcessor.CONFIG_TARGET_FIELD, + targetField, + Ip2GeoProcessor.CONFIG_PROPERTIES, + Arrays.asList(IP, CITY) + ); + + // Create ip2geo processor + createIp2GeoProcessorPipeline(pipelineName, processorProperties); + + Map> sampleData = getSampleData(); + List docs = sampleData.entrySet() + .stream() + .map(entry -> createDocument(field, entry.getKey())) + .collect(Collectors.toList()); + + // Simulate processor + Map response = simulatePipeline(pipelineName, docs); + + // Verify data added to document + List> sources = convertToListOfSources(response, targetField); + sources.stream().forEach(source -> { + assertFalse(source.containsKey(COUNTRY)); + assertEquals(sampleData.get(source.get(IP)).get(CITY), source.get(CITY)); + }); + + // Delete datasource fails when there is a process using it + ResponseException exception = expectThrows(ResponseException.class, () -> deleteDatasource(datasourceName)); + // Verify + assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); + + // Delete resources + deletePipeline(pipelineName); + deleteDatasource(datasourceName); + } finally { + Ip2GeoDataServer.stop(); + } + } + + private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map properties) throws IOException { String field = GeospatialTestHelper.randomLowerCaseString(); String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Map defaultProperties = Map.of( + Map defaultProperties = Map.of( Ip2GeoProcessor.CONFIG_FIELD, field, Ip2GeoProcessor.CONFIG_DATASOURCE, datasourceName ); - Map baseProperties = new HashMap<>(); + Map baseProperties = new HashMap<>(); baseProperties.putAll(defaultProperties); baseProperties.putAll(properties); Map processorConfig = buildProcessorConfig(Ip2GeoProcessor.TYPE, baseProperties); return createPipeline(pipelineName, Optional.empty(), Arrays.asList(processorConfig)); } + + private Map> getSampleData() { + Map> sampleData = new HashMap<>(); + sampleData.put( + String.format( + Locale.ROOT, + "10.%d.%d.%d", + Randomness.get().nextInt(255), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255) + ), + Map.of(CITY, "Seattle", COUNTRY, "USA") + ); + sampleData.put( + String.format( + Locale.ROOT, + "127.%d.%d.%d", + Randomness.get().nextInt(15), + Randomness.get().nextInt(255), + Randomness.get().nextInt(255) + ), + Map.of(CITY, "Vancouver", COUNTRY, "Canada") + ); + sampleData.put( + String.format( + Locale.ROOT, + "fd12:2345:6789:1:%x:%x:%x:%x", + Randomness.get().nextInt(65535), + Randomness.get().nextInt(65535), + Randomness.get().nextInt(65535), + Randomness.get().nextInt(65535) + ), + Map.of(CITY, "Bengaluru", COUNTRY, "India") + ); + return sampleData; + } + + private Map> createDocument(String... args) { + if (args.length % 2 == 1) { + throw new RuntimeException("Number of arguments should be even"); + } + Map source = new HashMap<>(); + for (int i = 0; i < args.length; i += 2) { + source.put(args[0], args[1]); + } + return Map.of(SOURCE, source); + } + + /** + * This method convert returned value of simulatePipeline method to a list of sources + * + * For example, + * Input: + * { + * "docs" : [ + * { + * "doc" : { + * "_index" : "_index", + * "_id" : "_id", + * "_source" : { + * "ip2geo" : { + * "ip" : "127.0.0.1", + * "city" : "Seattle" + * }, + * "_ip" : "127.0.0.1" + * }, + * "_ingest" : { + * "timestamp" : "2023-05-12T17:41:42.939703Z" + * } + * } + * } + * ] + * } + * + * Output: + * [ + * { + * "ip" : "127.0.0.1", + * "city" : "Seattle" + * } + * ] + * + */ + private List> convertToListOfSources(final Map data, final String targetField) { + List>> docs = (List>>) data.get("docs"); + return docs.stream() + .map(doc -> (Map>) doc.get("doc").get(SOURCE)) + .map(source -> source.get(targetField)) + .collect(Collectors.toList()); + } } diff --git a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java index 7034a633..5b75292c 100644 --- a/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/processor/FeatureProcessorIT.java @@ -47,7 +47,7 @@ public void testIndexGeoJSONSuccess() throws Exception { Map geoFields = new HashMap<>(); geoFields.put(geoShapeField, "geo_shape"); - Map processorProperties = new HashMap<>(); + Map processorProperties = new HashMap<>(); processorProperties.put(FeatureProcessor.FIELD_KEY, geoShapeField); Map geoJSONProcessorConfig = buildProcessorConfig(FeatureProcessor.TYPE, processorProperties); List> configs = new ArrayList<>(); diff --git a/src/test/resources/ip2geo/server/city/city.zip b/src/test/resources/ip2geo/server/city/city.zip new file mode 100644 index 0000000000000000000000000000000000000000..12fbd7198d101bd245d5e531ccb8da49a0d423bf GIT binary patch literal 294 zcmWIWW@Zs#-~htL%r#*QP>>6xIT#ceQW8rN^^%LrLPK~N*eB18OPK=1AiA`In}Lz# z2O|RmP&WgE?x}-ZOo{>x4|kt8N=aU3w*A$W%9XztY~wKEUeuT$uk-P~-M@&S=7?V{ zJ9uB_YT3wrD``+Ies6lXc&C=hzxNprj()mrzw_YSR^ip$AI?Mn{5;tud;=MasNeUt*U59Gc8Z$>5&W`x_3U2 KP;l%5aTov(y;=YO literal 0 HcmV?d00001 diff --git a/src/test/resources/ip2geo/server/city/manifest.json b/src/test/resources/ip2geo/server/city/manifest.json new file mode 100644 index 00000000..de1e3f3b --- /dev/null +++ b/src/test/resources/ip2geo/server/city/manifest.json @@ -0,0 +1,8 @@ +{ + "url": "https://github.com/opensearch-project/geospatial/blob/main/src/test/resources/ip2geo/server/city/city.zip", + "db_name": "data.csv", + "sha256_hash": "oDPgEv+9+kNov7bdQQiLrhr8jQeEPdLnuJ22Hz5npvk=", + "valid_for_in_days": 30, + "updated_at_in_epoch_milli": 1683590400000, + "provider": "opensearch" +} diff --git a/src/test/resources/ip2geo/server/city/manifest_local.json b/src/test/resources/ip2geo/server/city/manifest_local.json new file mode 100644 index 00000000..a69ccbef --- /dev/null +++ b/src/test/resources/ip2geo/server/city/manifest_local.json @@ -0,0 +1,8 @@ +{ + "url": "http://localhost:8001/city/city.zip", + "db_name": "data.csv", + "sha256_hash": "oDPgEv+9+kNov7bdQQiLrhr8jQeEPdLnuJ22Hz5npvk=", + "valid_for_in_days": 30, + "updated_at_in_epoch_milli": 1683590400000, + "provider": "opensearch" +} diff --git a/src/test/resources/ip2geo/server/country/country.zip b/src/test/resources/ip2geo/server/country/country.zip new file mode 100644 index 0000000000000000000000000000000000000000..1c930b1a7441a3f6abed9d9ee089600522a93459 GIT binary patch literal 263 zcmWIWW@Zs#-~d8~sx@H@P~ZckIT#ceQW8rN^^%LrLPK~N*ca@IOHte%mja?oE4UdL zS$;4wFaUKkFnFKy(Y)k&PXDa7=Bg(~CKs5UYq{rMNO0G4RxZ}Kf5GFprJ=r|>O^hb zqZdw`I&tI3l~ZT#9J+Ms!YQx&XIzdO>#G`1^{lHNcyd4WynC2todx4dO5W0Fi4)?EnA( literal 0 HcmV?d00001 diff --git a/src/test/resources/ip2geo/server/country/manifest.json b/src/test/resources/ip2geo/server/country/manifest.json new file mode 100644 index 00000000..25460e5b --- /dev/null +++ b/src/test/resources/ip2geo/server/country/manifest.json @@ -0,0 +1,8 @@ +{ + "url": "https://github.com/opensearch-project/geospatial/blob/main/src/test/resources/ip2geo/server/country/country.zip", + "db_name": "data.csv", + "sha256_hash": "oDPgEv+4+kNov7bdQQiLrhr8jQeEPdLnuJ11Hz5npvk=", + "valid_for_in_days": 30, + "updated_at_in_epoch_milli": 1683590400000, + "provider": "opensearch" +} diff --git a/src/test/resources/ip2geo/server/country/manifest_local.json b/src/test/resources/ip2geo/server/country/manifest_local.json new file mode 100644 index 00000000..4c63840b --- /dev/null +++ b/src/test/resources/ip2geo/server/country/manifest_local.json @@ -0,0 +1,8 @@ +{ + "url": "http://localhost:8001/country/country.zip", + "db_name": "data.csv", + "sha256_hash": "oDPgEv+4+kNov7bdQQiLrhr8jQeEPdLnuJ11Hz5npvk=", + "valid_for_in_days": 30, + "updated_at_in_epoch_milli": 1683590400000, + "provider": "opensearch" +} From 9da3d85e62653f93dc1bdfc4c5ced290ed3cd2b3 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 15 May 2023 17:11:52 -0700 Subject: [PATCH 33/61] Add ConcurrentModificationException (#308) Signed-off-by: Heemin Kim --- .../ConcurrentModificationException.java | 37 ++++++++++++++++ .../DeleteDatasourceTransportAction.java | 6 ++- .../action/PutDatasourceTransportAction.java | 6 ++- .../UpdateDatasourceTransportAction.java | 5 ++- .../ConcurrentModificationExceptionTests.java | 42 +++++++++++++++++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 2 +- 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java create mode 100644 src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java diff --git a/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java b/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java new file mode 100644 index 00000000..579f22f4 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import java.io.IOException; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.rest.RestStatus; + +/** + * General ConcurrentModificationException corresponding to the {@link RestStatus#BAD_REQUEST} status code + * + * The exception is thrown when multiple mutation API is called for a same resource at the same time + */ +public class ConcurrentModificationException extends OpenSearchException { + + public ConcurrentModificationException(String msg, Object... args) { + super(msg, args); + } + + public ConcurrentModificationException(String msg, Throwable cause, Object... args) { + super(msg, cause, args); + } + + public ConcurrentModificationException(StreamInput in) throws IOException { + super(in); + } + + @Override + public final RestStatus status() { + return RestStatus.BAD_REQUEST; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index 753c107d..c287e0f1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -9,7 +9,6 @@ import lombok.extern.log4j.Log4j2; -import org.opensearch.OpenSearchException; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; @@ -17,6 +16,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.exceptions.ResourceInUseException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; @@ -73,7 +73,9 @@ public DeleteDatasourceTransportAction( protected void doExecute(final Task task, final DeleteDatasourceRequest request, final ActionListener listener) { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { - listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); return; } try { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index fa3fcf48..e6cf7f24 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -12,7 +12,6 @@ import lombok.extern.log4j.Log4j2; -import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; @@ -22,6 +21,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; @@ -72,7 +72,9 @@ public PutDatasourceTransportAction( protected void doExecute(final Task task, final PutDatasourceRequest request, final ActionListener listener) { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { - listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); return; } try { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index 20634552..a9b933ca 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -21,6 +21,7 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; @@ -74,7 +75,9 @@ public UpdateDatasourceTransportAction( protected void doExecute(final Task task, final UpdateDatasourceRequest request, final ActionListener listener) { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { - listener.onFailure(new OpenSearchException("another processor is holding a lock on the resource. Try again later")); + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); return; } try { diff --git a/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java new file mode 100644 index 00000000..98ddb3d7 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import lombok.SneakyThrows; + +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; + +public class ConcurrentModificationExceptionTests extends OpenSearchTestCase { + public void testConstructor_whenCreated_thenSucceed() { + ConcurrentModificationException exception = new ConcurrentModificationException("Resource is being modified by another processor"); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + public void testConstructor_whenCreatedWithRootCause_thenSucceed() { + ConcurrentModificationException exception = new ConcurrentModificationException( + "Resource is being modified by another processor", + new RuntimeException() + ); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + @SneakyThrows + public void testConstructor_whenCreatedWithStream_thenSucceed() { + ConcurrentModificationException exception = new ConcurrentModificationException( + "New datasource is not compatible with existing datasource" + ); + + BytesStreamOutput output = new BytesStreamOutput(); + exception.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + ConcurrentModificationException copiedException = new ConcurrentModificationException(input); + assertEquals(exception.getMessage(), copiedException.getMessage()); + assertEquals(exception.status(), copiedException.status()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 4e76d6d4..931dfdc1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -183,7 +183,7 @@ protected Datasource randomDatasource() { Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(29), ChronoUnit.DAYS)); + datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(28) + 1, ChronoUnit.DAYS)); datasource.setState(randomState()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); From f5d5bee898a8e6776612a74ee6cfbefe6242dbd2 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 15 May 2023 17:12:27 -0700 Subject: [PATCH 34/61] Add integration test for UpdateDatasource API (#307) Signed-off-by: Heemin Kim --- .../IncompatibleDatasourceException.java | 38 +++++++ .../action/RestUpdateDatasourceHandler.java | 6 +- .../action/UpdateDatasourceRequest.java | 4 +- .../UpdateDatasourceTransportAction.java | 3 +- .../geospatial/GeospatialRestTestCase.java | 25 ++++- .../IncompatibleDatasourceExceptionTests.java | 44 ++++++++ .../ResourceInUseExceptionTests.java | 37 +++++++ .../RestUpdateDatasourceHandlerTests.java | 33 +++++- .../ip2geo/action/UpdateDatasourceIT.java | 104 ++++++++++++++++++ .../UpdateDatasourceTransportActionTests.java | 3 +- .../ip2geo/processor/Ip2GeoProcessorIT.java | 48 ++++++-- 11 files changed, 316 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java create mode 100644 src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java diff --git a/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java b/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java new file mode 100644 index 00000000..d4f00c02 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import java.io.IOException; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.rest.RestStatus; + +/** + * IncompatibleDatasourceException corresponding to the {@link RestStatus#BAD_REQUEST} status code + * + * The exception is thrown when a user tries to update datasource with new endpoint which is not compatible + * with current datasource + */ +public class IncompatibleDatasourceException extends OpenSearchException { + + public IncompatibleDatasourceException(String msg, Object... args) { + super(msg, args); + } + + public IncompatibleDatasourceException(String msg, Throwable cause, Object... args) { + super(msg, cause, args); + } + + public IncompatibleDatasourceException(StreamInput in) throws IOException { + super(in); + } + + @Override + public final RestStatus status() { + return RestStatus.BAD_REQUEST; + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java index 77abae84..f9ba73ec 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java @@ -31,15 +31,15 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final PutDatasourceRequest putDatasourceRequest = new PutDatasourceRequest(request.param("name")); + final UpdateDatasourceRequest updateDatasourceRequest = new UpdateDatasourceRequest(request.param("name")); if (request.hasContentOrSourceParam()) { try (XContentParser parser = request.contentOrSourceParamParser()) { - PutDatasourceRequest.PARSER.parse(parser, putDatasourceRequest, null); + UpdateDatasourceRequest.PARSER.parse(parser, updateDatasourceRequest, null); } } return channel -> client.executeLocally( UpdateDatasourceAction.INSTANCE, - putDatasourceRequest, + updateDatasourceRequest, new RestToXContentListener<>(channel) ); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java index fdc0b357..41a7fd90 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java @@ -33,8 +33,8 @@ @Log4j2 @EqualsAndHashCode(callSuper = false) public class UpdateDatasourceRequest extends AcknowledgedRequest { - private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); - private static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); + public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); + public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); private static final int MAX_DATASOURCE_NAME_BYTES = 255; /** * @param name the datasource name diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index a9b933ca..31c50007 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -21,6 +21,7 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; @@ -149,7 +150,7 @@ private void validateFieldsCompatibility(final UpdateDatasourceRequest request, List fields = datasourceUpdateService.getHeaderFields(request.getEndpoint()); if (datasource.isCompatible(fields) == false) { - throw new OpenSearchException( + throw new IncompatibleDatasourceException( "new fields [{}] does not contain all old fields [{}]", fields.toString(), datasource.getDatabase().getFields().toString() diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 5c5aecc4..1fc8f3e0 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -110,9 +110,9 @@ protected static void deletePipeline(String name) throws IOException { client().performRequest(request); } - protected Response createDatasource(final String name, Map properties) throws IOException { + protected Response createDatasource(final String name, Map properties) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - for (Map.Entry config : properties.entrySet()) { + for (Map.Entry config : properties.entrySet()) { builder.field(config.getKey(), config.getValue()); } builder.endObject(); @@ -149,16 +149,31 @@ protected Response deleteDatasource(final String name) throws IOException { return client().performRequest(request); } + protected Response deleteDatasource(final String name, final int retry) throws Exception { + for (int i = 0; i < retry; i++) { + try { + Request request = new Request(DELETE, buildDatasourcePath(name)); + return client().performRequest(request); + } catch (Exception e) { + if (i + 1 == retry) { + throw e; + } + Thread.sleep(1000); + } + } + throw new RuntimeException("should not reach here"); + } + protected Map getDatasource(final String name) throws Exception { Request request = new Request(GET, buildDatasourcePath(name)); Response response = client().performRequest(request); return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); } - protected Response updateDatasource(final String name, Map config) throws IOException { + protected Response updateDatasource(final String name, Map properties) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - if (config != null && !config.isEmpty()) { - builder.value(config); + for (Map.Entry config : properties.entrySet()) { + builder.field(config.getKey(), config.getValue()); } builder.endObject(); diff --git a/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java new file mode 100644 index 00000000..9d03b10c --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import lombok.SneakyThrows; + +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; + +public class IncompatibleDatasourceExceptionTests extends OpenSearchTestCase { + public void testConstructor_whenCreated_thenSucceed() { + IncompatibleDatasourceException exception = new IncompatibleDatasourceException( + "New datasource is not compatible with existing datasource" + ); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + public void testConstructor_whenCreatedWithRootCause_thenSucceed() { + IncompatibleDatasourceException exception = new IncompatibleDatasourceException( + "New datasource is not compatible with existing datasource", + new RuntimeException() + ); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + @SneakyThrows + public void testConstructor_whenCreatedWithStream_thenSucceed() { + IncompatibleDatasourceException exception = new IncompatibleDatasourceException( + "New datasource is not compatible with existing datasource" + ); + + BytesStreamOutput output = new BytesStreamOutput(); + exception.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + IncompatibleDatasourceException copiedException = new IncompatibleDatasourceException(input); + assertEquals(exception.getMessage(), copiedException.getMessage()); + assertEquals(exception.status(), copiedException.status()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java new file mode 100644 index 00000000..00465840 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.exceptions; + +import lombok.SneakyThrows; + +import org.opensearch.common.io.stream.BytesStreamInput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.rest.RestStatus; +import org.opensearch.test.OpenSearchTestCase; + +public class ResourceInUseExceptionTests extends OpenSearchTestCase { + public void testConstructor_whenCreated_thenSucceed() { + ResourceInUseException exception = new ResourceInUseException("Resource is in use"); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + public void testConstructor_whenCreatedWithRootCause_thenSucceed() { + ResourceInUseException exception = new ResourceInUseException("Resource is in use", new RuntimeException()); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + } + + @SneakyThrows + public void testConstructor_whenCreatedWithStream_thenSucceed() { + ResourceInUseException exception = new ResourceInUseException("New datasource is not compatible with existing datasource"); + + BytesStreamOutput output = new BytesStreamOutput(); + exception.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + ResourceInUseException copiedException = new ResourceInUseException(input); + assertEquals(exception.getMessage(), copiedException.getMessage()); + assertEquals(exception.status(), copiedException.status()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java index 6af74414..4227e7c6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java @@ -41,11 +41,34 @@ public void testPrepareRequest_whenValidInput_thenSucceed() { AtomicBoolean isExecuted = new AtomicBoolean(false); verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { - assertTrue(actionRequest instanceof PutDatasourceRequest); - PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; - assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); - assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval()); - assertEquals(datasourceName, putDatasourceRequest.getName()); + assertTrue(actionRequest instanceof UpdateDatasourceRequest); + UpdateDatasourceRequest updateDatasourceRequest = (UpdateDatasourceRequest) actionRequest; + assertEquals("https://test.com", updateDatasourceRequest.getEndpoint()); + assertEquals(TimeValue.timeValueDays(1), updateDatasourceRequest.getUpdateInterval()); + assertEquals(datasourceName, updateDatasourceRequest.getName()); + isExecuted.set(true); + return null; + }); + + dispatchRequest(request); + assertTrue(isExecuted.get()); + } + + public void testPrepareRequest_whenNullInput_thenSucceed() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String content = "{}"; + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withPath(String.format(Locale.ROOT, path, datasourceName)) + .withContent(new BytesArray(content), XContentType.JSON) + .build(); + AtomicBoolean isExecuted = new AtomicBoolean(false); + + verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof UpdateDatasourceRequest); + UpdateDatasourceRequest updateDatasourceRequest = (UpdateDatasourceRequest) actionRequest; + assertNull(updateDatasourceRequest.getEndpoint()); + assertNull(updateDatasourceRequest.getUpdateInterval()); + assertEquals(datasourceName, updateDatasourceRequest.getName()); isExecuted.set(true); return null; }); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java new file mode 100644 index 00000000..706ea6c5 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.action; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import lombok.SneakyThrows; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.opensearch.client.ResponseException; +import org.opensearch.geospatial.GeospatialRestTestCase; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; +import org.opensearch.rest.RestStatus; + +public class UpdateDatasourceIT extends GeospatialRestTestCase { + // Use this value in resource name to avoid name conflict among tests + private static final String PREFIX = UpdateDatasourceIT.class.getSimpleName().toLowerCase(Locale.ROOT); + + @BeforeClass + public static void start() { + Ip2GeoDataServer.start(); + } + + @AfterClass + public static void stop() { + Ip2GeoDataServer.stop(); + } + + @SneakyThrows + public void testUpdateDatasource_whenValidInput_thenUpdated() { + boolean isDatasourceCreated = false; + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); + try { + Map datasourceProperties = Map.of( + PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + Ip2GeoDataServer.getEndpointCountry() + ); + + // Create datasource and wait for it to be available + createDatasource(datasourceName, datasourceProperties); + isDatasourceCreated = true; + waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10)); + + int updateIntervalInDays = 1; + updateDatasourceEndpoint(datasourceName, Ip2GeoDataServer.getEndpointCity(), updateIntervalInDays); + List> datasources = (List>) getDatasource(datasourceName).get("datasources"); + + assertEquals(Ip2GeoDataServer.getEndpointCity(), datasources.get(0).get("endpoint")); + assertEquals(updateIntervalInDays, datasources.get(0).get("update_interval_in_days")); + } finally { + if (isDatasourceCreated) { + deleteDatasource(datasourceName, 3); + } + } + } + + @SneakyThrows + public void testUpdateDatasource_whenIncompatibleFields_thenFails() { + boolean isDatasourceCreated = false; + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); + try { + Map datasourceProperties = Map.of( + PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + Ip2GeoDataServer.getEndpointCity() + ); + + // Create datasource and wait for it to be available + createDatasource(datasourceName, datasourceProperties); + isDatasourceCreated = true; + waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10)); + + // Update should fail as country data does not have every fields that city data has + int updateIntervalInDays = 1; + ResponseException exception = expectThrows( + ResponseException.class, + () -> updateDatasourceEndpoint(datasourceName, Ip2GeoDataServer.getEndpointCountry(), updateIntervalInDays) + ); + assertEquals(RestStatus.BAD_REQUEST.getStatus(), exception.getResponse().getStatusLine().getStatusCode()); + } finally { + if (isDatasourceCreated) { + deleteDatasource(datasourceName, 3); + } + } + } + + private void updateDatasourceEndpoint(final String datasourceName, final String endpoint, final int updateInterval) throws IOException { + Map properties = Map.of( + UpdateDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + endpoint, + UpdateDatasourceRequest.UPDATE_INTERVAL_IN_DAYS_FIELD.getPreferredName(), + updateInterval + ); + updateDatasource(datasourceName, properties); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 2d3a5602..76763fb1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -27,6 +27,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.Randomness; import org.opensearch.common.unit.TimeValue; +import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.jobscheduler.spi.LockModel; @@ -197,7 +198,7 @@ public void testDoExecute_whenIncompatibleFields_thenError() { // Verify ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); verify(listener).onFailure(exceptionCaptor.capture()); - assertEquals(OpenSearchException.class, exceptionCaptor.getValue().getClass()); + assertEquals(IncompatibleDatasourceException.class, exceptionCaptor.getValue().getClass()); exceptionCaptor.getValue().getMessage().contains("does not contain"); verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index 9c59039c..bf447a5a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -25,8 +25,11 @@ import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; +import org.opensearch.rest.RestStatus; public class Ip2GeoProcessorIT extends GeospatialRestTestCase { + // Use this value in resource name to avoid name conflict among tests + private static final String PREFIX = Ip2GeoProcessorIT.class.getSimpleName().toLowerCase(Locale.ROOT); private static final String CITY = "city"; private static final String COUNTRY = "country"; private static final String IP = "ip"; @@ -34,7 +37,7 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase { @SneakyThrows public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { - String pipelineName = GeospatialTestHelper.randomLowerCaseString(); + String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); // Run ResponseException exception = expectThrows( @@ -44,25 +47,35 @@ public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { // Verify assertTrue(exception.getMessage().contains("doesn't exist")); - assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); + assertEquals(RestStatus.BAD_REQUEST.getStatus(), exception.getResponse().getStatusLine().getStatusCode()); } @SneakyThrows public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { Ip2GeoDataServer.start(); + boolean isDatasourceCreated = false; + boolean isProcessorCreated = false; + String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); try { - String pipelineName = GeospatialTestHelper.randomLowerCaseString(); - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String targetField = GeospatialTestHelper.randomLowerCaseString(); String field = GeospatialTestHelper.randomLowerCaseString(); - Map datasourceProperties = Map.of( + Map datasourceProperties = Map.of( PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), Ip2GeoDataServer.getEndpointCity() ); // Create datasource and wait for it to be available createDatasource(datasourceName, datasourceProperties); + isDatasourceCreated = true; + // Creation of datasource with same name should fail + ResponseException createException = expectThrows( + ResponseException.class, + () -> createDatasource(datasourceName, datasourceProperties) + ); + // Verify + assertEquals(RestStatus.BAD_REQUEST.getStatus(), createException.getResponse().getStatusLine().getStatusCode()); waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10)); Map processorProperties = Map.of( @@ -78,6 +91,7 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { // Create ip2geo processor createIp2GeoProcessorPipeline(pipelineName, processorProperties); + isProcessorCreated = true; Map> sampleData = getSampleData(); List docs = sampleData.entrySet() @@ -96,21 +110,31 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { }); // Delete datasource fails when there is a process using it - ResponseException exception = expectThrows(ResponseException.class, () -> deleteDatasource(datasourceName)); + ResponseException deleteException = expectThrows(ResponseException.class, () -> deleteDatasource(datasourceName)); // Verify - assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); - - // Delete resources - deletePipeline(pipelineName); - deleteDatasource(datasourceName); + assertEquals(RestStatus.BAD_REQUEST.getStatus(), deleteException.getResponse().getStatusLine().getStatusCode()); } finally { + Exception exception = null; + try { + if (isProcessorCreated) { + deletePipeline(pipelineName); + } + if (isDatasourceCreated) { + deleteDatasource(datasourceName, 3); + } + } catch (Exception e) { + exception = e; + } Ip2GeoDataServer.stop(); + if (exception != null) { + throw exception; + } } } private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map properties) throws IOException { String field = GeospatialTestHelper.randomLowerCaseString(); - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); Map defaultProperties = Map.of( Ip2GeoProcessor.CONFIG_FIELD, field, From 3992850f527c931dd26ac27fac6f7e708570f5c1 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 16 May 2023 16:40:45 -0700 Subject: [PATCH 35/61] Bug fix on lock management and few performance improvements (#310) * Release lock before response back to caller for update/delete API * Release lock in background task for creation API * Change index settings to improve indexing performance Signed-off-by: Heemin Kim --- .../DeleteDatasourceTransportAction.java | 10 +---- .../action/PutDatasourceTransportAction.java | 36 ++++++++++----- .../UpdateDatasourceTransportAction.java | 19 +++----- .../ip2geo/common/GeoIpDataFacade.java | 44 +++++++++++++------ .../ip2geo/common/Ip2GeoLockService.java | 12 +++-- .../ip2geo/jobscheduler/DatasourceRunner.java | 5 +-- .../DeleteDatasourceTransportActionTests.java | 2 +- .../PutDatasourceTransportActionTests.java | 44 +++++++++++-------- .../UpdateDatasourceTransportActionTests.java | 10 ++--- .../ip2geo/common/GeoIpDataFacadeTests.java | 10 +++-- .../ip2geo/common/Ip2GeoLockServiceTests.java | 2 +- .../jobscheduler/DatasourceRunnerTests.java | 4 +- 12 files changed, 111 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index c287e0f1..a4c6e5c4 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -80,17 +80,11 @@ protected void doExecute(final Task task, final DeleteDatasourceRequest request, } try { deleteDatasource(request.getName()); + lockService.releaseLock(lock); listener.onResponse(new AcknowledgedResponse(true)); } catch (Exception e) { + lockService.releaseLock(lock); listener.onFailure(e); - } finally { - lockService.releaseLock( - lock, - ActionListener.wrap( - released -> { log.info("Released lock for datasource[{}]", request.getName()); }, - exception -> { log.error("Failed to release the lock", exception); } - ) - ); } }, exception -> { listener.onFailure(exception); })); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index e6cf7f24..75b30f29 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -80,16 +80,16 @@ protected void doExecute(final Task task, final PutDatasourceRequest request, fi try { internalDoExecute(request, lock, listener); } catch (Exception e) { + lockService.releaseLock(lock); listener.onFailure(e); - } finally { - lockService.releaseLock( - lock, - ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) - ); } }, exception -> { listener.onFailure(exception); })); } + /** + * This method takes lock as a parameter and is responsible for releasing lock + * unless exception is thrown + */ @VisibleForTesting protected void internalDoExecute( final PutDatasourceRequest request, @@ -100,17 +100,21 @@ protected void internalDoExecute( datasourceFacade.createIndexIfNotExists(createIndexStep); createIndexStep.whenComplete(v -> { Datasource datasource = Datasource.Builder.build(request); - datasourceFacade.putDatasource( - datasource, - getIndexResponseListener(datasource, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), listener) - ); - }, exception -> listener.onFailure(exception)); + datasourceFacade.putDatasource(datasource, getIndexResponseListener(datasource, lock, listener)); + }, exception -> { + lockService.releaseLock(lock); + listener.onFailure(exception); + }); } + /** + * This method takes lock as a parameter and is responsible for releasing lock + * unless exception is thrown + */ @VisibleForTesting protected ActionListener getIndexResponseListener( final Datasource datasource, - final Runnable renewLock, + final LockModel lock, final ActionListener listener ) { return new ActionListener<>() { @@ -118,12 +122,20 @@ protected ActionListener getIndexResponseListener( public void onResponse(final IndexResponse indexResponse) { // This is user initiated request. Therefore, we want to handle the first datasource update task in a generic thread // pool. - threadPool.generic().submit(() -> { createDatasource(datasource, renewLock); }); + threadPool.generic().submit(() -> { + AtomicReference lockReference = new AtomicReference<>(lock); + try { + createDatasource(datasource, lockService.getRenewLockRunnable(lockReference)); + } finally { + lockService.releaseLock(lockReference.get()); + } + }); listener.onResponse(new AcknowledgedResponse(true)); } @Override public void onFailure(final Exception e) { + lockService.releaseLock(lock); if (e instanceof VersionConflictEngineException) { listener.onFailure(new ResourceAlreadyExistsException("datasource [{}] already exists", datasource.getName())); } else { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index 31c50007..f082c2e4 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -14,15 +14,14 @@ import lombok.extern.log4j.Log4j2; -import org.opensearch.OpenSearchException; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; -import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; import org.opensearch.geospatial.exceptions.ConcurrentModificationException; +import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; @@ -81,27 +80,21 @@ protected void doExecute(final Task task, final UpdateDatasourceRequest request, ); return; } + try { Datasource datasource = datasourceFacade.getDatasource(request.getName()); if (datasource == null) { - listener.onFailure(new ResourceNotFoundException("no such datasource exist")); - return; + throw new ResourceNotFoundException("no such datasource exist"); } validate(request, datasource); updateIfChanged(request, datasource); + lockService.releaseLock(lock); listener.onResponse(new AcknowledgedResponse(true)); } catch (Exception e) { + lockService.releaseLock(lock); listener.onFailure(e); - } finally { - lockService.releaseLock( - lock, - ActionListener.wrap( - released -> { log.info("Released lock for datasource[{}]", request.getName()); }, - exception -> { log.error("Failed to release the lock", exception); } - ) - ); } - }, exception -> { listener.onFailure(exception); })); + }, exception -> listener.onFailure(exception))); } private void updateIfChanged(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index dfd2f099..5584f5eb 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -43,7 +43,6 @@ import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.Requests; @@ -67,6 +66,8 @@ public class GeoIpDataFacade { private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); + private static final Tuple INDEX_SETTING_NUM_OF_REPLICAS = new Tuple<>("index.number_of_replicas", 0); + private static final Tuple INDEX_SETTING_REFRESH_INTERVAL = new Tuple<>("index.refresh_interval", -1); private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); private static final Tuple INDEX_SETTING_READ_ONLY_ALLOW_DELETE = new Tuple<>( @@ -84,7 +85,12 @@ public GeoIpDataFacade(final ClusterService clusterService, final Client client) } /** - * Create an index of single shard with auto expand replicas to all nodes + * Create an index for GeoIP data + * + * Index setting start with single shard, zero replica, no refresh interval, and hidden. + * Once the GeoIP data is indexed, do refresh and force merge. + * Then, change the index setting to expand replica to all nodes, and read only allow delete. + * See {@link #freezeIndex} * * @param indexName index name */ @@ -94,7 +100,8 @@ public void createIndexIfNotExists(final String indexName) { } final Map indexSettings = new HashMap<>(); indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); - indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); + indexSettings.put(INDEX_SETTING_REFRESH_INTERVAL.v1(), INDEX_SETTING_REFRESH_INTERVAL.v2()); + indexSettings.put(INDEX_SETTING_NUM_OF_REPLICAS.v1(), INDEX_SETTING_NUM_OF_REPLICAS.v2()); indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); StashedThreadContext.run( @@ -103,6 +110,24 @@ public void createIndexIfNotExists(final String indexName) { ); } + private void freezeIndex(final String indexName) { + TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); + StashedThreadContext.run(client, () -> { + client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); + client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + Map settings = new HashMap<>(); + settings.put(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2()); + settings.put(INDEX_SETTING_NUM_OF_REPLICAS.v1(), null); + settings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); + client.admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(settings) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); + }); + } + /** * Generate XContentBuilder representing datasource database index mapping * @@ -349,7 +374,7 @@ public void putGeoIpData( @NonNull final Runnable renewLock ) { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); - final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + final BulkRequest bulkRequest = new BulkRequest(); while (iterator.hasNext()) { CSVRecord record = iterator.next(); String document = createDocument(fields, record.values()); @@ -368,16 +393,7 @@ public void putGeoIpData( } renewLock.run(); } - StashedThreadContext.run(client, () -> { - client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); - client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); - client.admin() - .indices() - .prepareUpdateSettings(indexName) - .setSettings(Map.of(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2())) - .execute() - .actionGet(timeout); - }); + freezeIndex(indexName); } public AcknowledgedResponse deleteIp2GeoDataIndex(final String index) { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java index ab646628..6b3c83b7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java @@ -12,6 +12,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import lombok.extern.log4j.Log4j2; + import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.client.Client; @@ -22,6 +24,7 @@ /** * A wrapper of job scheduler's lock service for datasource */ +@Log4j2 public class Ip2GeoLockService { public static final long LOCK_DURATION_IN_SECONDS = 300l; public static final long RENEW_AFTER_IN_SECONDS = 120l; @@ -57,10 +60,12 @@ public void acquireLock(final String datasourceName, final Long lockDurationSeco * Wrapper method of LockService#release * * @param lockModel the lock model - * @param listener the listener */ - public void releaseLock(final LockModel lockModel, final ActionListener listener) { - lockService.release(lockModel, listener); + public void releaseLock(final LockModel lockModel) { + lockService.release( + lockModel, + ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) + ); } /** @@ -110,7 +115,6 @@ public Runnable getRenewLockRunnable(final AtomicReference lockModel) if (Instant.now().isBefore(preLock.getLockTime().plusSeconds(RENEW_AFTER_IN_SECONDS))) { return; } - lockModel.set(renewLock(lockModel.get())); if (lockModel.get() == null) { new OpenSearchException("failed to renew a lock [{}]", preLock); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index 4233c8a8..0ae2953a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -111,10 +111,7 @@ protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParamet try { updateDatasource(jobParameter, ip2GeoLockService.getRenewLockRunnable(new AtomicReference<>(lock))); } finally { - ip2GeoLockService.releaseLock( - lock, - ActionListener.wrap(released -> {}, exception -> { log.error("Failed to release lock [{}]", lock, exception); }) - ); + ip2GeoLockService.releaseLock(lock); } }, exception -> { log.error("Failed to acquire lock for job [{}]", jobParameter.getName(), exception); })); }; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java index 54f9ed61..71447454 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -90,7 +90,7 @@ private void validateDoExecute(final LockModel lockModel, final Exception except verify(listener).onFailure(any(OpenSearchException.class)); } else { verify(listener).onResponse(new AcknowledgedResponse(true)); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } } else { // Run diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 339c65cb..156c9d0d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -12,22 +12,20 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import java.io.IOException; -import java.time.Instant; import lombok.SneakyThrows; import org.junit.Before; import org.mockito.ArgumentCaptor; -import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.StepListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.unit.TimeValue; import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; @@ -52,28 +50,32 @@ public void init() { @SneakyThrows public void testDoExecute_whenFailedToAcquireLock_thenError() { - validateDoExecute(null, null); + validateDoExecute(null, null, null); } @SneakyThrows - public void testDoExecute_whenValidInput_thenSucceed() { - String jobIndexName = GeospatialTestHelper.randomLowerCaseString(); - String jobId = GeospatialTestHelper.randomLowerCaseString(); - LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); - validateDoExecute(lockModel, null); + public void testDoExecute_whenAcquiredLock_thenSucceed() { + validateDoExecute(randomLockModel(), null, null); } @SneakyThrows - public void testDoExecute_whenException_thenError() { - validateDoExecute(null, new RuntimeException()); + public void testDoExecute_whenExceptionBeforeAcquiringLock_thenError() { + validateDoExecute(randomLockModel(), new RuntimeException(), null); } - private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { + @SneakyThrows + public void testDoExecute_whenExceptionAfterAcquiringLock_thenError() { + validateDoExecute(randomLockModel(), null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception before, final Exception after) throws IOException { Task task = mock(Task.class); Datasource datasource = randomDatasource(); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); PutDatasourceRequest request = new PutDatasourceRequest(datasource.getName()); ActionListener listener = mock(ActionListener.class); + if (after != null) { + doThrow(after).when(datasourceFacade).createIndexIfNotExists(any(StepListener.class)); + } // Run action.doExecute(task, request, listener); @@ -82,21 +84,25 @@ private void validateDoExecute(final LockModel lockModel, final Exception except ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); - if (exception == null) { + if (before == null) { // Run captor.getValue().onResponse(lockModel); // Verify if (lockModel == null) { - verify(listener).onFailure(any(OpenSearchException.class)); + verify(listener).onFailure(any(ConcurrentModificationException.class)); + } + if (after != null) { + verify(ip2GeoLockService).releaseLock(eq(lockModel)); + verify(listener).onFailure(after); } else { - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService, never()).releaseLock(eq(lockModel)); } } else { // Run - captor.getValue().onFailure(exception); + captor.getValue().onFailure(before); // Verify - verify(listener).onFailure(exception); + verify(listener).onFailure(before); } } @@ -133,7 +139,7 @@ public void testInternalDoExecute_whenValidInput_thenSucceed() { public void testGetIndexResponseListener_whenVersionConflict_thenFailure() { Datasource datasource = new Datasource(); ActionListener listener = mock(ActionListener.class); - action.getIndexResponseListener(datasource, mock(Runnable.class), listener) + action.getIndexResponseListener(datasource, randomLockModel(), listener) .onFailure( new VersionConflictEngineException( null, diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 76763fb1..9364115f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -112,7 +112,7 @@ public void testDoExecute_whenValidInput_thenUpdate() { assertEquals(request.getEndpoint(), datasource.getEndpoint()); assertEquals(request.getUpdateInterval().days(), datasource.getSchedule().getInterval()); verify(listener).onResponse(new AcknowledgedResponse(true)); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @SneakyThrows @@ -142,7 +142,7 @@ public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { verify(datasourceUpdateService, never()).getHeaderFields(anyString()); verify(datasourceFacade, never()).updateDatasource(datasource); verify(listener).onResponse(new AcknowledgedResponse(true)); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @SneakyThrows @@ -169,7 +169,7 @@ public void testDoExecute_whenNoDatasource_thenError() { verify(listener).onFailure(exceptionCaptor.capture()); assertEquals(ResourceNotFoundException.class, exceptionCaptor.getValue().getClass()); exceptionCaptor.getValue().getMessage().contains("no such datasource exist"); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @SneakyThrows @@ -200,7 +200,7 @@ public void testDoExecute_whenIncompatibleFields_thenError() { verify(listener).onFailure(exceptionCaptor.capture()); assertEquals(IncompatibleDatasourceException.class, exceptionCaptor.getValue().getClass()); exceptionCaptor.getValue().getMessage().contains("does not contain"); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @SneakyThrows @@ -229,6 +229,6 @@ public void testDoExecute_whenInvalidUpdateInterval_thenError() { verify(listener).onFailure(exceptionCaptor.capture()); assertEquals(InvalidParameterException.class, exceptionCaptor.getValue().getClass()); exceptionCaptor.getValue().getMessage().contains("should be smaller"); - verify(ip2GeoLockService).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 5a269539..19ee2bc7 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -51,7 +51,6 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.WriteRequest; import org.opensearch.common.Randomness; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesReference; @@ -89,8 +88,10 @@ public void testCreateIndexIfNotExistsWithoutExistingIndex() { assertTrue(actionRequest instanceof CreateIndexRequest); CreateIndexRequest request = (CreateIndexRequest) actionRequest; assertEquals(index, request.index()); - assertEquals(1, (int) request.settings().getAsInt("index.number_of_shards", 2)); - assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals(1, (int) request.settings().getAsInt("index.number_of_shards", 0)); + assertNull(request.settings().get("index.auto_expand_replicas")); + assertEquals(0, (int) request.settings().getAsInt("index.number_of_replicas", 1)); + assertEquals(-1, (int) request.settings().getAsInt("index.refresh_interval", 0)); assertEquals(true, request.settings().getAsBoolean("index.hidden", false)); assertEquals( @@ -191,7 +192,6 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { if (actionRequest instanceof BulkRequest) { BulkRequest request = (BulkRequest) actionRequest; assertEquals(1, request.numberOfActions()); - assertEquals(WriteRequest.RefreshPolicy.WAIT_UNTIL, request.getRefreshPolicy()); BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(false); return response; @@ -211,6 +211,8 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { assertEquals(1, request.indices().length); assertEquals(index, request.indices()[0]); assertEquals(true, request.settings().getAsBoolean("index.blocks.read_only_allow_delete", false)); + assertNull(request.settings().get("index.num_of_replica")); + assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); return null; } else { throw new RuntimeException("invalid request is called"); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java index f0bc8ce7..d6bfcd13 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java @@ -41,7 +41,7 @@ public void testAcquireLock_whenValidInput_thenSucceed() { public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - noOpsLockService.releaseLock(null, mock(ActionListener.class)); + noOpsLockService.releaseLock(null); } public void testRenewLock_whenCalled_thenNotBlocked() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 1ccd88a5..004e1dec 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -98,13 +98,13 @@ private void validateDoExecute(final LockModel lockModel, final Exception except captor.getValue().onResponse(lockModel); // Verify - verify(ip2GeoLockService, lockModel == null ? never() : times(1)).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService, lockModel == null ? never() : times(1)).releaseLock(eq(lockModel)); } else { // Run captor.getValue().onFailure(exception); // Verify - verify(ip2GeoLockService, never()).releaseLock(eq(lockModel), any(ActionListener.class)); + verify(ip2GeoLockService, never()).releaseLock(eq(lockModel)); } } From 75137e84cb917815585acb6422abf5b2cffea6d6 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 17 May 2023 10:50:44 -0700 Subject: [PATCH 36/61] Change index setting from read_only_allow_delete to write (#311) read_only_allow_delete does not block write to an index. The disk-based shard allocator may add and remove this block automatically. Therefore, use index.blocks.write instead. Signed-off-by: Heemin Kim --- .../geospatial/ip2geo/common/GeoIpDataFacade.java | 8 ++------ .../geospatial/ip2geo/common/GeoIpDataFacadeTests.java | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 5584f5eb..a9ffa43d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -70,10 +70,7 @@ public class GeoIpDataFacade { private static final Tuple INDEX_SETTING_REFRESH_INTERVAL = new Tuple<>("index.refresh_interval", -1); private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); - private static final Tuple INDEX_SETTING_READ_ONLY_ALLOW_DELETE = new Tuple<>( - "index.blocks.read_only_allow_delete", - true - ); + private static final Tuple INDEX_SETTING_BLOCKS_WRITE = new Tuple<>("index.blocks.write", true); private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final Client client; @@ -116,8 +113,7 @@ private void freezeIndex(final String indexName) { client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); Map settings = new HashMap<>(); - settings.put(INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v1(), INDEX_SETTING_READ_ONLY_ALLOW_DELETE.v2()); - settings.put(INDEX_SETTING_NUM_OF_REPLICAS.v1(), null); + settings.put(INDEX_SETTING_BLOCKS_WRITE.v1(), INDEX_SETTING_BLOCKS_WRITE.v2()); settings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); client.admin() .indices() diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 19ee2bc7..b11851a9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -210,7 +210,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { UpdateSettingsRequest request = (UpdateSettingsRequest) actionRequest; assertEquals(1, request.indices().length); assertEquals(index, request.indices()[0]); - assertEquals(true, request.settings().getAsBoolean("index.blocks.read_only_allow_delete", false)); + assertEquals(true, request.settings().getAsBoolean("index.blocks.write", false)); assertNull(request.settings().get("index.num_of_replica")); assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); return null; From 94b5bf75a74487bcb172bc7fbc132aebba6759c4 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 19 May 2023 10:07:09 -0700 Subject: [PATCH 37/61] Fix bug in get datasource API and improve memory usage (#313) Signed-off-by: Heemin Kim --- .../action/GetDatasourceTransportAction.java | 10 +++- .../ip2geo/common/GeoIpDataFacade.java | 52 +++++++++--------- .../geospatial/ip2geo/Ip2GeoTestCase.java | 1 + .../GetDatasourceTransportActionTests.java | 54 ++++++++++--------- .../ip2geo/common/GeoIpDataFacadeTests.java | 4 +- 5 files changed, 70 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java index 905d8165..6bac685e 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportAction.java @@ -5,6 +5,7 @@ package org.opensearch.geospatial.ip2geo.action; +import java.util.Collections; import java.util.List; import org.opensearch.OpenSearchException; @@ -12,8 +13,10 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -57,7 +60,8 @@ private boolean shouldGetAllDatasource(final GetDatasourceRequest request) { return request.getNames().length == 0 || (request.getNames().length == 1 && "_all".equals(request.getNames()[0])); } - private ActionListener> newActionListener(final ActionListener listener) { + @VisibleForTesting + protected ActionListener> newActionListener(final ActionListener listener) { return new ActionListener<>() { @Override public void onResponse(final List datasources) { @@ -66,6 +70,10 @@ public void onResponse(final List datasources) { @Override public void onFailure(final Exception e) { + if (e instanceof IndexNotFoundException) { + listener.onResponse(new GetDatasourceResponse(Collections.emptyList())); + return; + } listener.onFailure(e); } }; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index a9ffa43d..4b6d9fe9 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -20,8 +20,10 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -35,6 +37,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.SpecialPermission; import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -51,8 +54,10 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; @@ -187,7 +192,7 @@ protected CSVParser internalGetDatabaseReader(final DatasourceManifest manifest, } /** - * Create a document in json string format to ingest in datasource database index + * Create a document to ingest in datasource database index * * It assumes the first field as ip_range. The rest is added under data field. * @@ -204,31 +209,23 @@ protected CSVParser internalGetDatabaseReader(final DatasourceManifest manifest, * @param fields a list of field name * @param values a list of values * @return Document in json string format + * @throws IOException the exception */ - public String createDocument(final String[] fields, final String[] values) { + public XContentBuilder createDocument(final String[] fields, final String[] values) throws IOException { if (fields.length != values.length) { throw new OpenSearchException("header[{}] and record[{}] length does not match", fields, values); } - StringBuilder sb = new StringBuilder(); - sb.append("{\""); - sb.append(IP_RANGE_FIELD_NAME); - sb.append("\":\""); - sb.append(values[0]); - sb.append("\",\""); - sb.append(DATA_FIELD_NAME); - sb.append("\":{"); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field(IP_RANGE_FIELD_NAME, values[0]); + builder.startObject(DATA_FIELD_NAME); for (int i = 1; i < fields.length; i++) { - if (i != 1) { - sb.append(","); - } - sb.append("\""); - sb.append(fields[i]); - sb.append("\":\""); - sb.append(values[i]); - sb.append("\""); + builder.field(fields[i], values[i]); } - sb.append("}}"); - return sb.toString(); + builder.endObject(); + builder.endObject(); + builder.close(); + return builder; } /** @@ -368,14 +365,20 @@ public void putGeoIpData( @NonNull final Iterator iterator, final int bulkSize, @NonNull final Runnable renewLock - ) { + ) throws IOException { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); final BulkRequest bulkRequest = new BulkRequest(); + Queue requests = new LinkedList<>(); + for (int i = 0; i < bulkSize; i++) { + requests.add(Requests.indexRequest(indexName)); + } while (iterator.hasNext()) { CSVRecord record = iterator.next(); - String document = createDocument(fields, record.values()); - IndexRequest request = Requests.indexRequest(indexName).source(document, XContentType.JSON); - bulkRequest.add(request); + XContentBuilder document = createDocument(fields, record.values()); + IndexRequest indexRequest = (IndexRequest) requests.poll(); + indexRequest.source(document); + indexRequest.id(record.get(0)); + bulkRequest.add(indexRequest); if (iterator.hasNext() == false || bulkRequest.requests().size() == bulkSize) { BulkResponse response = StashedThreadContext.run(client, () -> client.bulk(bulkRequest).actionGet(timeout)); if (response.hasFailures()) { @@ -385,6 +388,7 @@ public void putGeoIpData( response.buildFailureMessage() ); } + requests.addAll(bulkRequest.requests()); bulkRequest.requests().clear(); } renewLock.run(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 931dfdc1..608a957f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -52,6 +52,7 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; +import org.opensearch.geospatial.plugin.GeospatialPlugin; import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; import org.opensearch.jobscheduler.spi.LockModel; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java index 3dd4ddf5..0404da0f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java @@ -5,18 +5,20 @@ package org.opensearch.geospatial.ip2geo.action; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.Before; -import org.mockito.ArgumentCaptor; import org.opensearch.action.ActionListener; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.tasks.Task; public class GetDatasourceTransportActionTests extends Ip2GeoTestCase { @@ -27,7 +29,7 @@ public void init() { action = new GetDatasourceTransportAction(transportService, actionFilters, datasourceFacade); } - public void testDoExecute_whenAll_thenSucceed() throws Exception { + public void testDoExecute_whenAll_thenSucceed() { Task task = mock(Task.class); GetDatasourceRequest request = new GetDatasourceRequest(new String[] { "_all" }); ActionListener listener = mock(ActionListener.class); @@ -36,22 +38,7 @@ public void testDoExecute_whenAll_thenSucceed() throws Exception { action.doExecute(task, request, listener); // Verify - ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getAllDatasources(captor.capture()); - - // Run - List datasources = Arrays.asList(randomDatasource(), randomDatasource()); - captor.getValue().onResponse(datasources); - - // Verify - verify(listener).onResponse(new GetDatasourceResponse(datasources)); - - // Run - RuntimeException exception = new RuntimeException(); - captor.getValue().onFailure(exception); - - // Verify - verify(listener).onFailure(exception); + verify(datasourceFacade).getAllDatasources(any(ActionListener.class)); } public void testDoExecute_whenNames_thenSucceed() { @@ -66,20 +53,37 @@ public void testDoExecute_whenNames_thenSucceed() { action.doExecute(task, request, listener); // Verify - ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getDatasources(eq(datasourceNames), captor.capture()); + verify(datasourceFacade).getDatasources(eq(datasourceNames), any(ActionListener.class)); + } + + public void testNewActionListener_whenOnResponse_thenSucceed() { + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + ActionListener actionListener = mock(ActionListener.class); + + // Run + action.newActionListener(actionListener).onResponse(datasources); + + // Verify + verify(actionListener).onResponse(new GetDatasourceResponse(datasources)); + } + + public void testNewActionListener_whenOnFailureWithNoSuchIndexException_thenEmptyDatasource() { + ActionListener actionListener = mock(ActionListener.class); // Run - captor.getValue().onResponse(datasources); + action.newActionListener(actionListener).onFailure(new IndexNotFoundException("no index")); // Verify - verify(listener).onResponse(new GetDatasourceResponse(datasources)); + verify(actionListener).onResponse(new GetDatasourceResponse(Collections.emptyList())); + } + + public void testNewActionListener_whenOnFailure_thenFails() { + ActionListener actionListener = mock(ActionListener.class); // Run - RuntimeException exception = new RuntimeException(); - captor.getValue().onFailure(exception); + action.newActionListener(actionListener).onFailure(new RuntimeException()); // Verify - verify(listener).onFailure(exception); + verify(actionListener).onFailure(any(RuntimeException.class)); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index b11851a9..cd4c9bad 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -52,6 +52,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.Randomness; +import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesReference; import org.opensearch.geospatial.GeospatialTestHelper; @@ -103,12 +104,13 @@ public void testCreateIndexIfNotExistsWithoutExistingIndex() { verifyingGeoIpDataFacade.createIndexIfNotExists(index); } + @SneakyThrows public void testCreateDocument() { String[] names = { "ip", "country", "city" }; String[] values = { "1.0.0.0/25", "USA", "Seattle" }; assertEquals( "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", - noOpsGeoIpDataFacade.createDocument(names, values) + Strings.toString(noOpsGeoIpDataFacade.createDocument(names, values)) ); } From 34804c9bda69efbb6dba0e713b85f17e3eebf4f1 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 23 May 2023 13:45:53 -0700 Subject: [PATCH 38/61] Change package for Strings.hasText (#314) (#317) Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + .../org/opensearch/geospatial/GeospatialRestTestCase.java | 6 +++--- .../org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 109fc6c6..5b3ac85a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,4 +21,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Exclude lombok generated code from jacoco coverage report ([#268](https://github.com/opensearch-project/geospatial/pull/268)) ### Documentation ### Maintenance +* Change package for Strings.hasText ([#314](https://github.com/opensearch-project/geospatial/pull/314)) ### Refactoring diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 1fc8f3e0..deae4225 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -118,7 +118,7 @@ protected Response createDatasource(final String name, Map prope builder.endObject(); Request request = new Request(PUT, buildDatasourcePath(name)); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(org.opensearch.common.Strings.toString(builder)); return client().performRequest(request); } @@ -178,7 +178,7 @@ protected Response updateDatasource(final String name, Map prope builder.endObject(); Request request = new Request(PUT, String.join(URL_DELIMITER, buildDatasourcePath(name), SETTINGS)); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(org.opensearch.common.Strings.toString(builder)); return client().performRequest(request); } @@ -188,7 +188,7 @@ protected Map simulatePipeline(final String name, List d builder.endObject(); Request request = new Request(GET, String.join(URL_DELIMITER, buildPipelinePath(name), SIMULATE)); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(org.opensearch.common.Strings.toString(builder)); Response response = client().performRequest(request); return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 608a957f..931dfdc1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -52,7 +52,6 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; -import org.opensearch.geospatial.plugin.GeospatialPlugin; import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; import org.opensearch.jobscheduler.spi.LockModel; From 2bec196fbea54e018d1474a8e07a1bd857dbf808 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 24 May 2023 14:46:30 -0700 Subject: [PATCH 39/61] Remove jitter and move index setting from DatasourceFacade to DatasourceExtension (#319) Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceFacade.java | 17 +++-------------- .../ip2geo/jobscheduler/Datasource.java | 18 ------------------ .../jobscheduler/DatasourceExtension.java | 16 ++++++++++++++++ .../ip2geo/jobscheduler/DatasourceTests.java | 9 --------- 4 files changed, 19 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index a14a9c72..166b025a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -12,9 +12,7 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -40,7 +38,6 @@ import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; @@ -62,9 +59,6 @@ @Log4j2 public class DatasourceFacade { private static final Integer MAX_SIZE = 1000; - private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); - private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); - private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); private final Client client; private final ClusterService clusterService; private final ClusterSettings clusterSettings; @@ -76,22 +70,17 @@ public DatasourceFacade(final Client client, final ClusterService clusterService } /** - * Create a datasource index of single shard with auto expand replicas to all nodes + * Create datasource index * - * We want the index to expand to all replica so that datasource query request can be executed locally - * for faster ingestion time. + * @param stepListener setp listener */ public void createIndexIfNotExists(final StepListener stepListener) { if (clusterService.state().metadata().hasIndex(DatasourceExtension.JOB_INDEX_NAME) == true) { stepListener.onResponse(null); return; } - final Map indexSettings = new HashMap<>(); - indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); - indexSettings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); - indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); final CreateIndexRequest createIndexRequest = new CreateIndexRequest(DatasourceExtension.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(indexSettings); + .settings(DatasourceExtension.INDEX_SETTING); StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { @Override public void onResponse(final CreateIndexResponse createIndexResponse) { diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index f27ab286..ac81e563 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -51,9 +51,6 @@ public class Datasource implements Writeable, ScheduledJobParameter { * Prefix of indices having Ip2Geo data */ public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".ip2geo-data"; - private static final long MAX_JITTER_IN_MINUTES = 5; - private static final long ONE_DAY_IN_HOURS = 24; - private static final long ONE_HOUR_IN_MINUTES = 60; /** * Default fields for job scheduling @@ -285,21 +282,6 @@ public Long getLockDurationSeconds() { return Ip2GeoLockService.LOCK_DURATION_IN_SECONDS; } - /** - * Jitter in scheduling a task - * - * We want a job to be delayed randomly with range of (0, 5) minutes for the - * next execution time. - * - * @see ScheduledJobParameter#getJitter() - * - * @return the jitter - */ - @Override - public Double getJitter() { - return MAX_JITTER_IN_MINUTES / ((double) schedule.getInterval() * ONE_DAY_IN_HOURS * ONE_HOUR_IN_MINUTES); - } - /** * Enable auto update of GeoIP data */ diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java index edea50bd..1cf51338 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java @@ -5,6 +5,8 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; +import java.util.Map; + import org.opensearch.jobscheduler.spi.JobSchedulerExtension; import org.opensearch.jobscheduler.spi.ScheduledJobParser; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; @@ -21,6 +23,20 @@ public class DatasourceExtension implements JobSchedulerExtension { * Job index name for a datasource */ public static final String JOB_INDEX_NAME = ".scheduler_geospatial_ip2geo_datasource"; + /** + * Job index setting + * + * We want it to be single shard so that job can be run only in a single node by job scheduler. + * We want it to expand to all replicas so that querying to this index can be done locally to reduce latency. + */ + public static final Map INDEX_SETTING = Map.of( + "index.number_of_shards", + 1, + "index.auto_expand_replicas", + "0-all", + "index.hidden", + true + ); @Override public String getJobType() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 679a043d..b8be8068 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -15,7 +15,6 @@ import java.util.Arrays; import java.util.Locale; -import org.opensearch.common.Randomness; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -77,14 +76,6 @@ public void testGetIndexNameFor() { ); } - public void testGetJitter() { - Datasource datasource = new Datasource(); - datasource.setSchedule(new IntervalSchedule(Instant.now(), Randomness.get().ints(1, 31).findFirst().getAsInt(), ChronoUnit.DAYS)); - long intervalInMinutes = datasource.getSchedule().getInterval() * 60l * 24l; - double sixMinutes = 6; - assertTrue(datasource.getJitter() * intervalInMinutes <= sixMinutes); - } - public void testIsExpired() { Datasource datasource = new Datasource(); // never expire if validForInDays is null From 21ae37aafae91dc5359f96f1ee356129f2f28057 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 24 May 2023 16:46:24 -0700 Subject: [PATCH 40/61] Do not index blank value and do not enrich null property (#320) Signed-off-by: Heemin Kim --- .../ip2geo/common/GeoIpDataFacade.java | 4 ++ .../ip2geo/processor/Ip2GeoProcessor.java | 1 + .../ip2geo/common/GeoIpDataFacadeTests.java | 6 +-- .../processor/Ip2GeoProcessorTests.java | 41 +++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 4b6d9fe9..e9c32417 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -34,6 +34,7 @@ import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; +import org.apache.logging.log4j.util.Strings; import org.opensearch.OpenSearchException; import org.opensearch.SpecialPermission; import org.opensearch.action.ActionListener; @@ -220,6 +221,9 @@ public XContentBuilder createDocument(final String[] fields, final String[] valu builder.field(IP_RANGE_FIELD_NAME, values[0]); builder.startObject(DATA_FIELD_NAME); for (int i = 1; i < fields.length; i++) { + if (Strings.isBlank(values[i])) { + continue; + } builder.field(fields[i], values[i]); } builder.endObject(); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index b2c7e678..7c4f8120 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -198,6 +198,7 @@ private Map filteredGeoData(final Map geoData, f filteredGeoData = properties.stream() .filter(p -> p.equals(PROPERTY_IP) == false) + .filter(p -> geoData.containsKey(p)) .collect(Collectors.toMap(p -> p, p -> geoData.get(p))); if (properties.contains(PROPERTY_IP)) { filteredGeoData.put(PROPERTY_IP, ip); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index cd4c9bad..a5ca25fd 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -105,9 +105,9 @@ public void testCreateIndexIfNotExistsWithoutExistingIndex() { } @SneakyThrows - public void testCreateDocument() { - String[] names = { "ip", "country", "city" }; - String[] values = { "1.0.0.0/25", "USA", "Seattle" }; + public void testCreateDocument_whenBlankValue_thenDoNotAdd() { + String[] names = { "ip", "country", "location", "city" }; + String[] values = { "1.0.0.0/25", "USA", " ", "Seattle" }; assertEquals( "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", Strings.toString(noOpsGeoIpDataFacade.createDocument(names, values)) diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 31451ea8..e83fc764 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.function.BiConsumer; +import lombok.SneakyThrows; + import org.junit.Before; import org.mockito.ArgumentCaptor; import org.opensearch.OpenSearchException; @@ -318,6 +320,45 @@ public void testExecuteInternal() throws Exception { ); } + @SneakyThrows + public void testHandleSingleIp_whenEmptyGeoData_thenTargetFieldShouldNotSet() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + Map ipToGeoData = Collections.emptyMap(); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.handleSingleIp(ip, ipToGeoData, document, handler); + + // Verify + assertFalse(document.hasField(DEFAULT_TARGET_FIELD)); + verify(handler).accept(document, null); + } + + @SneakyThrows + public void testHandleSingleIp_whenNoValueForGivenProperty_thenDoNotAdd() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city", "country"))); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + Map ipToGeoData = Map.of("country", "USA"); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.handleSingleIp(ip, ipToGeoData, document, handler); + + // Verify + assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get("country")); + assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get("city")); + verify(handler).accept(document, null); + } + private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { Datasource datasource = new Datasource(); datasource.setName(datasourceName); From 1de266b1ff8de356230f6cc5688c530327b23612 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 24 May 2023 16:56:48 -0700 Subject: [PATCH 41/61] Move index setting keys to constants (#321) Signed-off-by: Heemin Kim --- .../geospatial/constants/IndexSetting.java | 18 +++++++++ .../ip2geo/common/GeoIpDataFacade.java | 38 ++++++++++--------- .../jobscheduler/DatasourceExtension.java | 13 +++---- 3 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/constants/IndexSetting.java diff --git a/src/main/java/org/opensearch/geospatial/constants/IndexSetting.java b/src/main/java/org/opensearch/geospatial/constants/IndexSetting.java new file mode 100644 index 00000000..0978c411 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/constants/IndexSetting.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.constants; + +/** + * Collection of keys for index setting + */ +public class IndexSetting { + public static final String NUMBER_OF_SHARDS = "index.number_of_shards"; + public static final String NUMBER_OF_REPLICAS = "index.number_of_replicas"; + public static final String REFRESH_INTERVAL = "index.refresh_interval"; + public static final String AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; + public static final String HIDDEN = "index.hidden"; + public static final String BLOCKS_WRITE = "index.blocks.write"; +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index e9c32417..3976ffd0 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -18,7 +18,6 @@ import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -52,7 +51,6 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; @@ -60,6 +58,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.constants.IndexSetting; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.query.QueryBuilders; @@ -71,12 +70,22 @@ public class GeoIpDataFacade { private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; - private static final Tuple INDEX_SETTING_NUM_OF_SHARDS = new Tuple<>("index.number_of_shards", 1); - private static final Tuple INDEX_SETTING_NUM_OF_REPLICAS = new Tuple<>("index.number_of_replicas", 0); - private static final Tuple INDEX_SETTING_REFRESH_INTERVAL = new Tuple<>("index.refresh_interval", -1); - private static final Tuple INDEX_SETTING_AUTO_EXPAND_REPLICAS = new Tuple<>("index.auto_expand_replicas", "0-all"); - private static final Tuple INDEX_SETTING_HIDDEN = new Tuple<>("index.hidden", true); - private static final Tuple INDEX_SETTING_BLOCKS_WRITE = new Tuple<>("index.blocks.write", true); + private static final Map INDEX_SETTING_TO_CREATE = Map.of( + IndexSetting.NUMBER_OF_SHARDS, + 1, + IndexSetting.NUMBER_OF_REPLICAS, + 0, + IndexSetting.REFRESH_INTERVAL, + -1, + IndexSetting.HIDDEN, + true + ); + private static final Map INDEX_SETTING_TO_FREEZE = Map.of( + IndexSetting.AUTO_EXPAND_REPLICAS, + "0-all", + IndexSetting.BLOCKS_WRITE, + true + ); private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final Client client; @@ -101,12 +110,8 @@ public void createIndexIfNotExists(final String indexName) { if (clusterService.state().metadata().hasIndex(indexName) == true) { return; } - final Map indexSettings = new HashMap<>(); - indexSettings.put(INDEX_SETTING_NUM_OF_SHARDS.v1(), INDEX_SETTING_NUM_OF_SHARDS.v2()); - indexSettings.put(INDEX_SETTING_REFRESH_INTERVAL.v1(), INDEX_SETTING_REFRESH_INTERVAL.v2()); - indexSettings.put(INDEX_SETTING_NUM_OF_REPLICAS.v1(), INDEX_SETTING_NUM_OF_REPLICAS.v2()); - indexSettings.put(INDEX_SETTING_HIDDEN.v1(), INDEX_SETTING_HIDDEN.v2()); - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(indexSettings).mapping(getIndexMapping()); + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).settings(INDEX_SETTING_TO_CREATE) + .mapping(getIndexMapping()); StashedThreadContext.run( client, () -> client.admin().indices().create(createIndexRequest).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) @@ -118,13 +123,10 @@ private void freezeIndex(final String indexName) { StashedThreadContext.run(client, () -> { client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); - Map settings = new HashMap<>(); - settings.put(INDEX_SETTING_BLOCKS_WRITE.v1(), INDEX_SETTING_BLOCKS_WRITE.v2()); - settings.put(INDEX_SETTING_AUTO_EXPAND_REPLICAS.v1(), INDEX_SETTING_AUTO_EXPAND_REPLICAS.v2()); client.admin() .indices() .prepareUpdateSettings(indexName) - .setSettings(settings) + .setSettings(INDEX_SETTING_TO_FREEZE) .execute() .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)); }); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java index 1cf51338..ce610ebe 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceExtension.java @@ -5,6 +5,10 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; +import static org.opensearch.geospatial.constants.IndexSetting.AUTO_EXPAND_REPLICAS; +import static org.opensearch.geospatial.constants.IndexSetting.HIDDEN; +import static org.opensearch.geospatial.constants.IndexSetting.NUMBER_OF_SHARDS; + import java.util.Map; import org.opensearch.jobscheduler.spi.JobSchedulerExtension; @@ -29,14 +33,7 @@ public class DatasourceExtension implements JobSchedulerExtension { * We want it to be single shard so that job can be run only in a single node by job scheduler. * We want it to expand to all replicas so that querying to this index can be done locally to reduce latency. */ - public static final Map INDEX_SETTING = Map.of( - "index.number_of_shards", - 1, - "index.auto_expand_replicas", - "0-all", - "index.hidden", - true - ); + public static final Map INDEX_SETTING = Map.of(NUMBER_OF_SHARDS, 1, AUTO_EXPAND_REPLICAS, "0-all", HIDDEN, true); @Override public String getJobType() { From 91b99d800fcda61225200fbd6207a1a2e79194c1 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 25 May 2023 11:32:47 -0700 Subject: [PATCH 42/61] Return null index name for expired data (#322) Return null index name for expired data so that it can be deleted by clean up process. Clean up process exclude current index from deleting. Signed-off-by: Heemin Kim --- .../ip2geo/jobscheduler/Datasource.java | 2 +- .../ip2geo/processor/Ip2GeoProcessor.java | 43 +++++++++---------- .../ip2geo/jobscheduler/DatasourceTests.java | 20 ++++++++- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index ac81e563..2a37eac3 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -307,7 +307,7 @@ public void disable() { * @return Current index name of a datasource */ public String currentIndexName() { - return indexNameFor(database.updatedAt.toEpochMilli()); + return isExpired() ? null : indexNameFor(database.updatedAt.toEpochMilli()); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 7c4f8120..fcb32b56 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -153,11 +153,19 @@ protected void executeInternal( datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { - if (handleInvalidDatasource(ingestDocument, datasource, handler)) { + if (datasource == null) { + handler.accept(null, new IllegalStateException("datasource does not exist")); return; } - geoIpDataFacade.getGeoIpData(datasource.currentIndexName(), ip, new ActionListener<>() { + String indexName = datasource.currentIndexName(); + if (indexName == null) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); + return; + } + + geoIpDataFacade.getGeoIpData(indexName, ip, new ActionListener<>() { @Override public void onResponse(final Map ipToGeoData) { handleSingleIp(ip, ipToGeoData, ingestDocument, handler); @@ -229,12 +237,20 @@ protected void executeInternal( datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { - if (handleInvalidDatasource(ingestDocument, datasource, handler)) { + if (datasource == null) { + handler.accept(null, new IllegalStateException("datasource does not exist")); + return; + } + + String indexName = datasource.currentIndexName(); + if (indexName == null) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); return; } geoIpDataFacade.getGeoIpData( - datasource.currentIndexName(), + indexName, ipList.iterator(), clusterSettings.get(Ip2GeoSettings.MAX_BUNDLE_SIZE), clusterSettings.get(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES), @@ -298,25 +314,6 @@ public void onFailure(final Exception e) { }; } - @VisibleForTesting - protected boolean handleInvalidDatasource( - final IngestDocument ingestDocument, - final Datasource datasource, - final BiConsumer handler - ) { - if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource does not exist")); - return true; - } - - if (datasource.isExpired()) { - ingestDocument.setFieldValue(targetField, DATA_EXPIRED); - handler.accept(ingestDocument, null); - return true; - } - return false; - } - @Override public String getType() { return TYPE; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index b8be8068..49b7961f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -46,7 +46,7 @@ public void testParser() throws Exception { assertTrue(datasource.equals(anotherDatasource)); } - public void testCurrentIndexName() { + public void testCurrentIndexName_whenNotExpired_thenReturnName() { String id = GeospatialTestHelper.randomLowerCaseString(); Instant now = Instant.now(); Datasource datasource = new Datasource(); @@ -54,14 +54,30 @@ public void testCurrentIndexName() { datasource.getDatabase().setProvider("provider"); datasource.getDatabase().setSha256Hash("sha256Hash"); datasource.getDatabase().setUpdatedAt(now); - datasource.getDatabase().setValidForInDays(10l); datasource.getDatabase().setFields(new ArrayList<>()); + assertEquals( String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, now.toEpochMilli()), datasource.currentIndexName() ); } + public void testCurrentIndexName_whenExpired_thenReturnNull() { + String id = GeospatialTestHelper.randomLowerCaseString(); + Instant now = Instant.now(); + Datasource datasource = new Datasource(); + datasource.setName(id); + datasource.getDatabase().setProvider("provider"); + datasource.getDatabase().setSha256Hash("sha256Hash"); + datasource.getDatabase().setUpdatedAt(now); + datasource.getDatabase().setValidForInDays(1l); + datasource.getUpdateStats().setLastSucceededAt(Instant.now().minus(25, ChronoUnit.HOURS)); + datasource.getDatabase().setFields(new ArrayList<>()); + + assertTrue(datasource.isExpired()); + assertNull(datasource.currentIndexName()); + } + public void testGetIndexNameFor() { long updatedAt = randomPositiveLong(); DatasourceManifest manifest = mock(DatasourceManifest.class); From 7a83df42e27e12b1ac0f05f3bcc5a81b07096401 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 May 2023 14:44:54 -0700 Subject: [PATCH 43/61] Add new fields in datasource (#325) Signed-off-by: Heemin Kim --- .../ip2geo/action/GetDatasourceResponse.java | 4 +- .../UpdateDatasourceTransportAction.java | 13 ++- .../ip2geo/jobscheduler/Datasource.java | 79 +++++++++++++++---- .../ip2geo/jobscheduler/DatasourceTask.java | 21 +++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 12 ++- .../action/GetDatasourceResponseTests.java | 4 +- .../PutDatasourceTransportActionTests.java | 2 +- .../UpdateDatasourceTransportActionTests.java | 4 +- 8 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTask.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java index 8e403774..d0dc2dcc 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java @@ -71,11 +71,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(FIELD_NAME_NAME.getPreferredName(), datasource.getName()); builder.field(FIELD_NAME_STATE.getPreferredName(), datasource.getState()); builder.field(FIELD_NAME_ENDPOINT.getPreferredName(), datasource.getEndpoint()); - builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), datasource.getSchedule().getInterval()); + builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), datasource.getUserSchedule().getInterval()); builder.timeField( FIELD_NAME_NEXT_UPDATE_AT.getPreferredName(), FIELD_NAME_NEXT_UPDATE_AT_READABLE.getPreferredName(), - datasource.getSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() + datasource.getUserSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() ); builder.field(FIELD_NAME_DATABASE.getPreferredName(), datasource.getDatabase()); builder.field(FIELD_NAME_UPDATE_STATS.getPreferredName(), datasource.getUpdateStats()); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index f082c2e4..7b2ed0bf 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -105,8 +105,12 @@ private void updateIfChanged(final UpdateDatasourceRequest request, final Dataso } if (isUpdateIntervalChanged(request, datasource)) { - datasource.setSchedule( - new IntervalSchedule(datasource.getSchedule().getStartTime(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS) + datasource.setUserSchedule( + new IntervalSchedule( + datasource.getUserSchedule().getStartTime(), + (int) request.getUpdateInterval().getDays(), + ChronoUnit.DAYS + ) ); isChanged = true; } @@ -163,7 +167,7 @@ private void validateUpdateIntervalIsLessThanValidForInDays(final UpdateDatasour long updateInterval = isUpdateIntervalChanged(request, datasource) ? request.getUpdateInterval().days() - : datasource.getSchedule().getInterval(); + : datasource.getUserSchedule().getInterval(); if (updateInterval >= validForInDays) { throw new InvalidParameterException( @@ -177,6 +181,7 @@ private boolean isEndpointChanged(final UpdateDatasourceRequest request, final D } private boolean isUpdateIntervalChanged(final UpdateDatasourceRequest request, final Datasource datasource) { - return request.getUpdateInterval() != null && (int) request.getUpdateInterval().days() != datasource.getSchedule().getInterval(); + return request.getUpdateInterval() != null + && (int) request.getUpdateInterval().days() != datasource.getUserSchedule().getInterval(); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 2a37eac3..8e0d2040 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -59,7 +59,28 @@ public class Datasource implements Writeable, ScheduledJobParameter { private static final ParseField ENABLED_FIELD = new ParseField("update_enabled"); private static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time"); private static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field"); - private static final ParseField SCHEDULE_FIELD = new ParseField("schedule"); + /** + * Schedule that user set + */ + private static final ParseField USER_SCHEDULE_FIELD = new ParseField("user_schedule"); + /** + * System schedule which will be used by job scheduler + * + * If datasource is going to get expired before next update, we want to run clean up task before the next update + * by changing system schedule. + * + * If datasource is restored from snapshot, we want to run clean up task immediately to handle expired datasource + * by changing system schedule. + * + * For every task run, we revert system schedule back to user schedule. + */ + private static final ParseField SYSTEM_SCHEDULE_FIELD = new ParseField("system_schedule"); + /** + * {@link DatasourceTask} that DatasourceRunner will execute in next run + * + * For every task run, we revert task back to {@link DatasourceTask#ALL} + */ + private static final ParseField TASK_FIELD = new ParseField("task"); private static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time"); private static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field"); @@ -97,10 +118,22 @@ public class Datasource implements Writeable, ScheduledJobParameter { */ private boolean isEnabled; /** - * @param schedule Schedule for a GeoIP data update - * @return Schedule for the job scheduler + * @param userSchedule Schedule that user provided + * @return Schedule that user provided + */ + private IntervalSchedule userSchedule; + + /** + * @param systemSchedule Schedule that job scheduler use + * @return Schedule that job scheduler use */ - private IntervalSchedule schedule; + private IntervalSchedule systemSchedule; + + /** + * @param task Task that {@link DatasourceRunner} will execute + * @return Task that {@link DatasourceRunner} will execute + */ + private DatasourceTask task; /** * Additional variables for datasource @@ -143,18 +176,22 @@ public class Datasource implements Writeable, ScheduledJobParameter { Instant lastUpdateTime = Instant.ofEpochMilli((long) args[1]); Instant enabledTime = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); boolean isEnabled = (boolean) args[3]; - IntervalSchedule schedule = (IntervalSchedule) args[4]; - String endpoint = (String) args[5]; - DatasourceState state = DatasourceState.valueOf((String) args[6]); - List indices = (List) args[7]; - Database database = (Database) args[8]; - UpdateStats updateStats = (UpdateStats) args[9]; + IntervalSchedule userSchedule = (IntervalSchedule) args[4]; + IntervalSchedule systemSchedule = (IntervalSchedule) args[5]; + DatasourceTask task = DatasourceTask.valueOf((String) args[6]); + String endpoint = (String) args[7]; + DatasourceState state = DatasourceState.valueOf((String) args[8]); + List indices = (List) args[9]; + Database database = (Database) args[10]; + UpdateStats updateStats = (UpdateStats) args[11]; Datasource parameter = new Datasource( name, lastUpdateTime, enabledTime, isEnabled, - schedule, + userSchedule, + systemSchedule, + task, endpoint, state, indices, @@ -170,7 +207,9 @@ public class Datasource implements Writeable, ScheduledJobParameter { PARSER.declareLong(ConstructingObjectParser.constructorArg(), LAST_UPDATE_TIME_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ENABLED_TIME_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SCHEDULE_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), USER_SCHEDULE_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SYSTEM_SCHEDULE_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TASK_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), ENDPOINT_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); @@ -190,6 +229,8 @@ public Datasource(final String name, final IntervalSchedule schedule, final Stri null, false, schedule, + schedule, + DatasourceTask.ALL, endpoint, DatasourceState.CREATING, new ArrayList<>(), @@ -203,7 +244,9 @@ public Datasource(final StreamInput in) throws IOException { lastUpdateTime = toInstant(in.readVLong()); enabledTime = toInstant(in.readOptionalVLong()); isEnabled = in.readBoolean(); - schedule = new IntervalSchedule(in); + userSchedule = new IntervalSchedule(in); + systemSchedule = new IntervalSchedule(in); + task = DatasourceTask.valueOf(in.readString()); endpoint = in.readString(); state = DatasourceState.valueOf(in.readString()); indices = in.readStringList(); @@ -217,7 +260,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVLong(lastUpdateTime.toEpochMilli()); out.writeOptionalVLong(enabledTime == null ? null : enabledTime.toEpochMilli()); out.writeBoolean(isEnabled); - schedule.writeTo(out); + userSchedule.writeTo(out); + systemSchedule.writeTo(out); + out.writeString(task.name()); out.writeString(endpoint); out.writeString(state.name()); out.writeStringCollection(indices); @@ -242,7 +287,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa ); } builder.field(ENABLED_FIELD.getPreferredName(), isEnabled); - builder.field(SCHEDULE_FIELD.getPreferredName(), schedule); + builder.field(USER_SCHEDULE_FIELD.getPreferredName(), userSchedule); + builder.field(SYSTEM_SCHEDULE_FIELD.getPreferredName(), systemSchedule); + builder.field(TASK_FIELD.getPreferredName(), task.name()); builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); builder.field(STATE_FIELD.getPreferredName(), state.name()); builder.field(INDICES_FIELD.getPreferredName(), indices); @@ -269,7 +316,7 @@ public Instant getEnabledTime() { @Override public IntervalSchedule getSchedule() { - return schedule; + return systemSchedule; } @Override diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTask.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTask.java new file mode 100644 index 00000000..bfbcb1d2 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTask.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.jobscheduler; + +/** + * Task that {@link DatasourceRunner} will run + */ +public enum DatasourceTask { + /** + * Do everything + */ + ALL, + + /** + * Only delete unused indices + */ + DELETE_UNUSED_INDICES +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 931dfdc1..a5129b48 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -50,6 +50,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; import org.opensearch.ingest.IngestMetadata; @@ -143,6 +144,13 @@ protected DatasourceState randomState() { .get(Randomness.createSecure().nextInt(DatasourceState.values().length - 1)); } + protected DatasourceTask randomTask() { + return Arrays.stream(DatasourceTask.values()) + .sequential() + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(DatasourceTask.values().length - 1)); + } + protected String randomIpAddress() { return String.format( Locale.ROOT, @@ -183,7 +191,9 @@ protected Datasource randomDatasource() { Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setSchedule(new IntervalSchedule(now, Randomness.get().nextInt(28) + 1, ChronoUnit.DAYS)); + datasource.setUserSchedule(new IntervalSchedule(now, Randomness.get().nextInt(28) + 1, ChronoUnit.DAYS)); + datasource.setSystemSchedule(datasource.getUserSchedule()); + datasource.setTask(randomTask()); datasource.setState(randomState()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java index cca230c7..7fa21681 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java @@ -40,7 +40,9 @@ public void testToXContent_whenValidInput_thenSucceed() throws Exception { assertTrue(json.contains(String.format(Locale.ROOT, "\"name\":\"%s\"", datasource.getName()))); assertTrue(json.contains(String.format(Locale.ROOT, "\"state\":\"%s\"", datasource.getState()))); assertTrue(json.contains(String.format(Locale.ROOT, "\"endpoint\":\"%s\"", datasource.getEndpoint()))); - assertTrue(json.contains(String.format(Locale.ROOT, "\"update_interval_in_days\":%d", datasource.getSchedule().getInterval()))); + assertTrue( + json.contains(String.format(Locale.ROOT, "\"update_interval_in_days\":%d", datasource.getUserSchedule().getInterval())) + ); assertTrue(json.contains(String.format(Locale.ROOT, "\"next_update_at_in_epoch_millis\""))); assertTrue(json.contains(String.format(Locale.ROOT, "\"provider\":\"%s\"", datasource.getDatabase().getProvider()))); assertTrue(json.contains(String.format(Locale.ROOT, "\"sha256_hash\":\"%s\"", datasource.getDatabase().getSha256Hash()))); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index 156c9d0d..ff838b84 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -128,7 +128,7 @@ public void testInternalDoExecute_whenValidInput_thenSucceed() { verify(datasourceFacade).putDatasource(datasourceCaptor.capture(), actionListenerCaptor.capture()); assertEquals(request.getName(), datasourceCaptor.getValue().getName()); assertEquals(request.getEndpoint(), datasourceCaptor.getValue().getEndpoint()); - assertEquals(request.getUpdateInterval().days(), datasourceCaptor.getValue().getSchedule().getInterval()); + assertEquals(request.getUpdateInterval().days(), datasourceCaptor.getValue().getUserSchedule().getInterval()); // Run next listener.onResponse actionListenerCaptor.getValue().onResponse(null); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 9364115f..21519e98 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -110,7 +110,7 @@ public void testDoExecute_whenValidInput_thenUpdate() { verify(datasourceFacade).updateDatasource(datasource); verify(datasourceUpdateService).getHeaderFields(request.getEndpoint()); assertEquals(request.getEndpoint(), datasource.getEndpoint()); - assertEquals(request.getUpdateInterval().days(), datasource.getSchedule().getInterval()); + assertEquals(request.getUpdateInterval().days(), datasource.getUserSchedule().getInterval()); verify(listener).onResponse(new AcknowledgedResponse(true)); verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @@ -119,7 +119,7 @@ public void testDoExecute_whenValidInput_thenUpdate() { public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { Datasource datasource = randomDatasource(); UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); - request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval())); + request.setUpdateInterval(TimeValue.timeValueDays(datasource.getUserSchedule().getInterval())); request.setEndpoint(datasource.getEndpoint()); Task task = mock(Task.class); From a52bc8b35d57173282f0beff42e7b3c85742ad5c Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 31 May 2023 15:08:26 -0700 Subject: [PATCH 44/61] Delete index once it is expired (#326) Signed-off-by: Heemin Kim --- .../UpdateDatasourceTransportAction.java | 48 +++++++++----- .../ip2geo/jobscheduler/Datasource.java | 29 ++++++++- .../ip2geo/jobscheduler/DatasourceRunner.java | 31 ++++++++- .../jobscheduler/DatasourceUpdateService.java | 38 +++++++++-- .../geospatial/ip2geo/Ip2GeoTestCase.java | 23 ++++++- .../UpdateDatasourceTransportActionTests.java | 47 ++++++++++++-- .../jobscheduler/DatasourceRunnerTests.java | 65 ++++++++++++++++++- .../ip2geo/jobscheduler/DatasourceTests.java | 53 ++++++++++++++- .../DatasourceUpdateServiceTests.java | 29 +++++++++ 9 files changed, 326 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index 7b2ed0bf..ffdb349d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.net.URL; import java.security.InvalidParameterException; +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Locale; @@ -26,6 +27,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.tasks.Task; @@ -97,21 +99,16 @@ protected void doExecute(final Task task, final UpdateDatasourceRequest request, }, exception -> listener.onFailure(exception))); } - private void updateIfChanged(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { + private void updateIfChanged(final UpdateDatasourceRequest request, final Datasource datasource) { boolean isChanged = false; if (isEndpointChanged(request, datasource)) { datasource.setEndpoint(request.getEndpoint()); isChanged = true; } - - if (isUpdateIntervalChanged(request, datasource)) { - datasource.setUserSchedule( - new IntervalSchedule( - datasource.getUserSchedule().getStartTime(), - (int) request.getUpdateInterval().getDays(), - ChronoUnit.DAYS - ) - ); + if (isUpdateIntervalChanged(request)) { + datasource.setUserSchedule(new IntervalSchedule(Instant.now(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS)); + datasource.setSystemSchedule(datasource.getUserSchedule()); + datasource.setTask(DatasourceTask.ALL); isChanged = true; } @@ -138,6 +135,21 @@ private void updateIfChanged(final UpdateDatasourceRequest request, final Dataso private void validate(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { validateFieldsCompatibility(request, datasource); validateUpdateIntervalIsLessThanValidForInDays(request, datasource); + validateNextUpdateScheduleIsBeforeExpirationDay(request, datasource); + } + + private void validateNextUpdateScheduleIsBeforeExpirationDay(final UpdateDatasourceRequest request, final Datasource datasource) { + if (request.getUpdateInterval() == null) { + return; + } + + IntervalSchedule newSchedule = new IntervalSchedule(Instant.now(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS); + + if (newSchedule.getNextExecutionTime(Instant.now()).isAfter(datasource.expirationDay())) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "datasource will expire at %s with the update interval", datasource.expirationDay().toString()) + ); + } } private void validateFieldsCompatibility(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { @@ -157,7 +169,7 @@ private void validateFieldsCompatibility(final UpdateDatasourceRequest request, private void validateUpdateIntervalIsLessThanValidForInDays(final UpdateDatasourceRequest request, final Datasource datasource) throws IOException { - if (isEndpointChanged(request, datasource) == false && isUpdateIntervalChanged(request, datasource) == false) { + if (isEndpointChanged(request, datasource) == false && isUpdateIntervalChanged(request) == false) { return; } @@ -165,7 +177,7 @@ private void validateUpdateIntervalIsLessThanValidForInDays(final UpdateDatasour ? DatasourceManifest.Builder.build(new URL(request.getEndpoint())).getValidForInDays() : datasource.getDatabase().getValidForInDays(); - long updateInterval = isUpdateIntervalChanged(request, datasource) + long updateInterval = isUpdateIntervalChanged(request) ? request.getUpdateInterval().days() : datasource.getUserSchedule().getInterval(); @@ -180,8 +192,14 @@ private boolean isEndpointChanged(final UpdateDatasourceRequest request, final D return request.getEndpoint() != null && request.getEndpoint().equals(datasource.getEndpoint()) == false; } - private boolean isUpdateIntervalChanged(final UpdateDatasourceRequest request, final Datasource datasource) { - return request.getUpdateInterval() != null - && (int) request.getUpdateInterval().days() != datasource.getUserSchedule().getInterval(); + /** + * Update interval is changed as long as user provide one because + * start time will get updated even if the update interval is same as current one. + * + * @param request the update datasource request + * @return true if update interval is changed, and false otherwise + */ + private boolean isUpdateIntervalChanged(final UpdateDatasourceRequest request) { + return request.getUpdateInterval() != null; } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 8e0d2040..5f1e5a4d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -374,13 +374,38 @@ private String indexNameFor(final long suffix) { /** * Checks if datasource is expired or not * - * @return true if datasource is expired false otherwise + * @return true if datasource is expired, and false otherwise */ public boolean isExpired() { + return willExpire(Instant.now()); + } + + /** + * Checks if datasource will expire at given time + * + * @return true if datasource will expired at given time, and false otherwise + */ + public boolean willExpire(Instant instant) { if (database.validForInDays == null) { return false; } + return instant.isAfter(expirationDay()); + } + + /** + * Day when datasource will expire + * + * @return Day when datasource will expire + */ + public Instant expirationDay() { + if (database.validForInDays == null) { + return Instant.MAX; + } + return lastCheckedAt().plus(database.validForInDays, ChronoUnit.DAYS); + } + + private Instant lastCheckedAt() { Instant lastCheckedAt; if (updateStats.lastSkippedAt == null) { lastCheckedAt = updateStats.lastSucceededAt; @@ -389,7 +414,7 @@ public boolean isExpired() { ? updateStats.lastSkippedAt : updateStats.lastSucceededAt; } - return Instant.now().isAfter(lastCheckedAt.plus(database.validForInDays, ChronoUnit.DAYS)); + return lastCheckedAt; } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index 0ae2953a..8d8938c9 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.extern.log4j.Log4j2; @@ -21,6 +22,7 @@ import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; /** * Datasource update task @@ -29,6 +31,8 @@ */ @Log4j2 public class DatasourceRunner implements ScheduledJobRunner { + private static final int DELETE_INDEX_RETRY_IN_MIN = 15; + private static final int DELETE_INDEX_DELAY_IN_MILLIS = 10000; private static DatasourceRunner INSTANCE; @@ -141,12 +145,37 @@ protected void updateDatasource(final ScheduledJobParameter jobParameter, final try { datasourceUpdateService.deleteUnusedIndices(datasource); - datasourceUpdateService.updateOrCreateGeoIpData(datasource, renewLock); + if (DatasourceTask.DELETE_UNUSED_INDICES.equals(datasource.getTask()) == false) { + datasourceUpdateService.updateOrCreateGeoIpData(datasource, renewLock); + } datasourceUpdateService.deleteUnusedIndices(datasource); } catch (Exception e) { log.error("Failed to update datasource for {}", datasource.getName(), e); datasource.getUpdateStats().setLastFailedAt(Instant.now()); datasourceFacade.updateDatasource(datasource); + } finally { + postProcessing(datasource); + } + } + + private void postProcessing(final Datasource datasource) { + if (datasource.isExpired()) { + // Try to delete again as it could have just been expired + datasourceUpdateService.deleteUnusedIndices(datasource); + datasourceUpdateService.updateDatasource(datasource, datasource.getUserSchedule(), DatasourceTask.ALL); + return; + } + + if (datasource.willExpire(datasource.getUserSchedule().getNextExecutionTime(Instant.now()))) { + IntervalSchedule intervalSchedule = new IntervalSchedule( + datasource.expirationDay(), + DELETE_INDEX_RETRY_IN_MIN, + ChronoUnit.MINUTES, + DELETE_INDEX_DELAY_IN_MILLIS + ); + datasourceUpdateService.updateDatasource(datasource, intervalSchedule, DatasourceTask.DELETE_UNUSED_INDICES); + } else { + datasourceUpdateService.updateDatasource(datasource, datasource.getUserSchedule(), DatasourceTask.ALL); } } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 4d9530db..0e6e16d1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -26,6 +26,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @Log4j2 public class DatasourceUpdateService { @@ -117,23 +118,46 @@ public List getHeaderFields(String manifestUrl) throws IOException { /** * Delete all indices except the one which are being used * - * @param parameter + * @param datasource */ - public void deleteUnusedIndices(final Datasource parameter) { + public void deleteUnusedIndices(final Datasource datasource) { try { - List indicesToDelete = parameter.getIndices() + List indicesToDelete = datasource.getIndices() .stream() - .filter(index -> index.equals(parameter.currentIndexName()) == false) + .filter(index -> index.equals(datasource.currentIndexName()) == false) .collect(Collectors.toList()); List deletedIndices = deleteIndices(indicesToDelete); if (deletedIndices.isEmpty() == false) { - parameter.getIndices().removeAll(deletedIndices); - datasourceFacade.updateDatasource(parameter); + datasource.getIndices().removeAll(deletedIndices); + datasourceFacade.updateDatasource(datasource); } } catch (Exception e) { - log.error("Failed to delete old indices for {}", parameter.getName(), e); + log.error("Failed to delete old indices for {}", datasource.getName(), e); + } + } + + /** + * Update datasource with given systemSchedule and task + * + * @param datasource datasource to update + * @param systemSchedule new system schedule value + * @param task new task value + */ + public void updateDatasource(final Datasource datasource, final IntervalSchedule systemSchedule, final DatasourceTask task) { + boolean updated = false; + if (datasource.getSystemSchedule().equals(systemSchedule) == false) { + datasource.setSystemSchedule(systemSchedule); + updated = true; + } + if (datasource.getTask().equals(task) == false) { + datasource.setTask(task); + updated = true; + } + + if (updated) { + datasourceFacade.updateDatasource(datasource); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index a5129b48..fe3783cc 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -186,12 +186,25 @@ protected long randomPositiveLong() { return value < 0 ? -value : value; } - protected Datasource randomDatasource() { - int validForInDays = Randomness.get().nextInt(30); + /** + * Update interval should be > 0 and < validForInDays. + * For an update test to work, there should be at least one eligible value other than current update interval. + * Therefore, the smallest value for validForInDays is 2. + * Update interval is random value from 1 to validForInDays - 2. + * The new update value will be validForInDays - 1. + */ + protected Datasource randomDatasource(final Instant updateStartTime) { + int validForInDays = 3 + Randomness.get().nextInt(30); Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); Datasource datasource = new Datasource(); datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setUserSchedule(new IntervalSchedule(now, Randomness.get().nextInt(28) + 1, ChronoUnit.DAYS)); + datasource.setUserSchedule( + new IntervalSchedule( + updateStartTime.truncatedTo(ChronoUnit.MILLIS), + 1 + Randomness.get().nextInt(validForInDays - 2), + ChronoUnit.DAYS + ) + ); datasource.setSystemSchedule(datasource.getUserSchedule()); datasource.setTask(randomTask()); datasource.setState(randomState()); @@ -216,6 +229,10 @@ protected Datasource randomDatasource() { return datasource; } + protected Datasource randomDatasource() { + return randomDatasource(Instant.now()); + } + protected LockModel randomLockModel() { LockModel lockModel = new LockModel( GeospatialTestHelper.randomLowerCaseString(), diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 21519e98..3df610e8 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -15,6 +15,8 @@ import static org.mockito.Mockito.when; import java.security.InvalidParameterException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import lombok.SneakyThrows; @@ -25,11 +27,11 @@ import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.Randomness; import org.opensearch.common.unit.TimeValue; import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.tasks.Task; @@ -83,11 +85,12 @@ private void validateDoExecuteWithLockError(final Exception exception) { @SneakyThrows public void testDoExecute_whenValidInput_thenUpdate() { - Datasource datasource = randomDatasource(); + Datasource datasource = randomDatasource(Instant.now().minusSeconds(60)); + datasource.setTask(DatasourceTask.DELETE_UNUSED_INDICES); + Instant originalStartTime = datasource.getSchedule().getStartTime(); UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); request.setEndpoint(sampleManifestUrl()); - // Sample manifest has validForDays of 30. Update interval should be less than that. - request.setUpdateInterval(TimeValue.timeValueDays(Randomness.get().nextInt(29))); + request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval())); Task task = mock(Task.class); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); @@ -113,13 +116,14 @@ public void testDoExecute_whenValidInput_thenUpdate() { assertEquals(request.getUpdateInterval().days(), datasource.getUserSchedule().getInterval()); verify(listener).onResponse(new AcknowledgedResponse(true)); verify(ip2GeoLockService).releaseLock(eq(lockModel)); + assertTrue(originalStartTime.isBefore(datasource.getSchedule().getStartTime())); + assertEquals(DatasourceTask.ALL, datasource.getTask()); } @SneakyThrows public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { Datasource datasource = randomDatasource(); UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); - request.setUpdateInterval(TimeValue.timeValueDays(datasource.getUserSchedule().getInterval())); request.setEndpoint(datasource.getEndpoint()); Task task = mock(Task.class); @@ -204,7 +208,7 @@ public void testDoExecute_whenIncompatibleFields_thenError() { } @SneakyThrows - public void testDoExecute_whenInvalidUpdateInterval_thenError() { + public void testDoExecute_whenLargeUpdateInterval_thenError() { Datasource datasource = randomDatasource(); UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); request.setUpdateInterval(TimeValue.timeValueDays(datasource.getDatabase().getValidForInDays())); @@ -231,4 +235,35 @@ public void testDoExecute_whenInvalidUpdateInterval_thenError() { exceptionCaptor.getValue().getMessage().contains("should be smaller"); verify(ip2GeoLockService).releaseLock(eq(lockModel)); } + + @SneakyThrows + public void testDoExecute_whenExpireWithNewUpdateInterval_thenError() { + Datasource datasource = randomDatasource(); + datasource.getUpdateStats().setLastSkippedAt(null); + datasource.getUpdateStats().setLastSucceededAt(Instant.now().minus(datasource.getDatabase().getValidForInDays(), ChronoUnit.DAYS)); + UpdateDatasourceRequest request = new UpdateDatasourceRequest(datasource.getName()); + request.setUpdateInterval(TimeValue.timeValueDays(1)); + + Task task = mock(Task.class); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + ActionListener listener = mock(ActionListener.class); + LockModel lockModel = randomLockModel(); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(ip2GeoLockService).acquireLock(eq(datasource.getName()), anyLong(), captor.capture()); + + // Run + captor.getValue().onResponse(lockModel); + + // Verify + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener).onFailure(exceptionCaptor.capture()); + assertEquals(IllegalArgumentException.class, exceptionCaptor.getValue().getClass()); + exceptionCaptor.getValue().getMessage().contains("will expire"); + verify(ip2GeoLockService).releaseLock(eq(lockModel)); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 004e1dec..d08dc19a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -17,7 +17,9 @@ import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; import java.io.IOException; +import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import lombok.SneakyThrows; @@ -32,6 +34,7 @@ import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; public class DatasourceRunnerTests extends Ip2GeoTestCase { @Before @@ -138,9 +141,63 @@ public void testUpdateDatasource_whenInvalidState_thenUpdateLastFailedAt() { @SneakyThrows public void testUpdateDatasource_whenValidInput_thenSucceed() { - Datasource datasource = new Datasource(); + Datasource datasource = randomDatasource(); + datasource.setState(DatasourceState.AVAILABLE); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, renewLock); + + // Verify + verify(datasourceUpdateService, times(2)).deleteUnusedIndices(datasource); + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource, renewLock); + verify(datasourceUpdateService).updateDatasource(datasource, datasource.getUserSchedule(), DatasourceTask.ALL); + } + + @SneakyThrows + public void testUpdateDatasource_whenDeleteTask_thenDeleteOnly() { + Datasource datasource = randomDatasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.setTask(DatasourceTask.DELETE_UNUSED_INDICES); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, renewLock); + + // Verify + verify(datasourceUpdateService, times(2)).deleteUnusedIndices(datasource); + verify(datasourceUpdateService, never()).updateOrCreateGeoIpData(datasource, renewLock); + verify(datasourceUpdateService).updateDatasource(datasource, datasource.getUserSchedule(), DatasourceTask.ALL); + } + + @SneakyThrows + public void testUpdateDatasource_whenExpired_thenDeleteIndicesAgain() { + Datasource datasource = randomDatasource(); + datasource.getUpdateStats().setLastSkippedAt(null); + datasource.getUpdateStats() + .setLastSucceededAt(Instant.now().minus(datasource.getDatabase().getValidForInDays() + 1, ChronoUnit.DAYS)); + datasource.setState(DatasourceState.AVAILABLE); + when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + Runnable renewLock = mock(Runnable.class); + + // Run + DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, renewLock); + + // Verify + verify(datasourceUpdateService, times(3)).deleteUnusedIndices(datasource); + verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource, renewLock); + verify(datasourceUpdateService).updateDatasource(datasource, datasource.getUserSchedule(), DatasourceTask.ALL); + } + + @SneakyThrows + public void testUpdateDatasource_whenWillExpire_thenScheduleDeleteTask() { + Datasource datasource = randomDatasource(); + datasource.getUpdateStats().setLastSkippedAt(null); + datasource.getUpdateStats() + .setLastSucceededAt(Instant.now().minus(datasource.getDatabase().getValidForInDays(), ChronoUnit.DAYS).plusSeconds(60)); datasource.setState(DatasourceState.AVAILABLE); - datasource.setName(randomLowerCaseString()); when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); Runnable renewLock = mock(Runnable.class); @@ -150,6 +207,10 @@ public void testUpdateDatasource_whenValidInput_thenSucceed() { // Verify verify(datasourceUpdateService, times(2)).deleteUnusedIndices(datasource); verify(datasourceUpdateService).updateOrCreateGeoIpData(datasource, renewLock); + + ArgumentCaptor captor = ArgumentCaptor.forClass(IntervalSchedule.class); + verify(datasourceUpdateService).updateDatasource(eq(datasource), captor.capture(), eq(DatasourceTask.DELETE_UNUSED_INDICES)); + assertTrue(Duration.between(datasource.expirationDay(), captor.getValue().getNextExecutionTime(Instant.now())).getSeconds() < 30); } @SneakyThrows diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 49b7961f..23981524 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -92,7 +92,7 @@ public void testGetIndexNameFor() { ); } - public void testIsExpired() { + public void testIsExpired_whenCalled_thenExpectedValue() { Datasource datasource = new Datasource(); // never expire if validForInDays is null assertFalse(datasource.isExpired()); @@ -111,6 +111,57 @@ public void testIsExpired() { assertFalse(datasource.isExpired()); } + public void testWillExpired_whenCalled_thenExpectedValue() { + Datasource datasource = new Datasource(); + // never expire if validForInDays is null + assertFalse(datasource.willExpire(Instant.MAX)); + + long validForInDays = 1; + datasource.getDatabase().setValidForInDays(validForInDays); + + // if last skipped date is null, use only last succeeded date to determine + datasource.getUpdateStats().setLastSucceededAt(Instant.now().minus(1, ChronoUnit.DAYS)); + assertTrue( + datasource.willExpire(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS).plusSeconds(1)) + ); + assertFalse(datasource.willExpire(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS))); + + // use the latest date between last skipped date and last succeeded date to determine + datasource.getUpdateStats().setLastSkippedAt(Instant.now()); + assertTrue( + datasource.willExpire(datasource.getUpdateStats().getLastSkippedAt().plus(validForInDays, ChronoUnit.DAYS).plusSeconds(1)) + ); + assertFalse(datasource.willExpire(datasource.getUpdateStats().getLastSkippedAt().plus(validForInDays, ChronoUnit.DAYS))); + + datasource.getUpdateStats().setLastSkippedAt(Instant.now().minus(1, ChronoUnit.HOURS)); + datasource.getUpdateStats().setLastSucceededAt(Instant.now()); + assertTrue( + datasource.willExpire(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS).plusSeconds(1)) + ); + assertFalse(datasource.willExpire(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS))); + } + + public void testExpirationDay_whenCalled_thenExpectedValue() { + Datasource datasource = new Datasource(); + datasource.getDatabase().setValidForInDays(null); + assertEquals(Instant.MAX, datasource.expirationDay()); + + long validForInDays = 1; + datasource.getDatabase().setValidForInDays(validForInDays); + + // if last skipped date is null, use only last succeeded date to determine + datasource.getUpdateStats().setLastSucceededAt(Instant.now().minus(1, ChronoUnit.DAYS)); + assertEquals(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS), datasource.expirationDay()); + + // use the latest date between last skipped date and last succeeded date to determine + datasource.getUpdateStats().setLastSkippedAt(Instant.now()); + assertEquals(datasource.getUpdateStats().getLastSkippedAt().plus(validForInDays, ChronoUnit.DAYS), datasource.expirationDay()); + + datasource.getUpdateStats().setLastSkippedAt(Instant.now().minus(1, ChronoUnit.HOURS)); + datasource.getUpdateStats().setLastSucceededAt(Instant.now()); + assertEquals(datasource.getUpdateStats().getLastSucceededAt().plus(validForInDays, ChronoUnit.DAYS), datasource.expirationDay()); + } + public void testLockDurationSeconds() { Datasource datasource = new Datasource(); assertNotNull(datasource.getLockDurationSeconds()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 108029fe..d8fbc3e0 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -10,6 +10,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -17,6 +18,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -33,6 +35,7 @@ import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @SuppressForbidden(reason = "unit test") public class DatasourceUpdateServiceTests extends Ip2GeoTestCase { @@ -178,6 +181,32 @@ public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { verify(datasourceFacade).updateDatasource(datasource); } + public void testUpdateDatasource_whenNoChange_thenNoUpdate() { + Datasource datasource = randomDatasource(); + + // Run + datasourceUpdateService.updateDatasource(datasource, datasource.getSystemSchedule(), datasource.getTask()); + + // Verify + verify(datasourceFacade, never()).updateDatasource(any()); + } + + public void testUpdateDatasource_whenChange_thenUpdate() { + Datasource datasource = randomDatasource(); + datasource.setTask(DatasourceTask.ALL); + + // Run + datasourceUpdateService.updateDatasource( + datasource, + new IntervalSchedule(Instant.now(), datasource.getSystemSchedule().getInterval() + 1, ChronoUnit.DAYS), + datasource.getTask() + ); + datasourceUpdateService.updateDatasource(datasource, datasource.getSystemSchedule(), DatasourceTask.DELETE_UNUSED_INDICES); + + // Verify + verify(datasourceFacade, times(2)).updateDatasource(any()); + } + @SneakyThrows public void testGetHeaderFields_whenValidInput_thenSucceed() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); From 661fe6a05f497ea78e278f2ad8b50a25557cc10d Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 2 Jun 2023 10:30:58 -0700 Subject: [PATCH 45/61] Add restoring event listener (#328) In the listener, we trigger a geoip data update Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceFacade.java | 30 +++ .../ip2geo/jobscheduler/Datasource.java | 26 ++- .../ip2geo/listener/Ip2GeoListener.java | 112 ++++++++++++ .../geospatial/plugin/GeospatialPlugin.java | 7 + .../ip2geo/common/DatasourceFacadeTests.java | 22 +++ .../ip2geo/jobscheduler/DatasourceTests.java | 28 +++ .../DatasourceUpdateServiceTests.java | 29 +++ .../ip2geo/listener/Ip2GeoListenerTests.java | 173 ++++++++++++++++++ .../plugin/GeospatialPluginTests.java | 8 + 9 files changed, 430 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index 166b025a..6fb5a71d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -26,11 +26,14 @@ import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; import org.opensearch.action.get.MultiGetItemResponse; import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; @@ -133,6 +136,33 @@ public IndexResponse updateDatasource(final Datasource datasource) { }); } + /** + * Update datasources in an index {@code DatasourceExtension.JOB_INDEX_NAME} + * @param datasources the datasources + * @param listener action listener + */ + public void updateDatasource(final List datasources, final ActionListener listener) { + BulkRequest bulkRequest = new BulkRequest(); + datasources.stream().map(datasource -> { + datasource.setLastUpdateTime(Instant.now()); + return datasource; + }).map(this::toIndexRequest).forEach(indexRequest -> bulkRequest.add(indexRequest)); + StashedThreadContext.run(client, () -> client.bulk(bulkRequest, listener)); + } + + private IndexRequest toIndexRequest(Datasource datasource) { + try { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(DatasourceExtension.JOB_INDEX_NAME); + indexRequest.id(datasource.getName()); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + indexRequest.source(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + return indexRequest; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Put datasource in an index {@code DatasourceExtension.JOB_INDEX_NAME} * diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 5f1e5a4d..0f884c32 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -354,7 +354,15 @@ public void disable() { * @return Current index name of a datasource */ public String currentIndexName() { - return isExpired() ? null : indexNameFor(database.updatedAt.toEpochMilli()); + if (isExpired()) { + return null; + } + + if (database.updatedAt == null) { + return null; + } + + return indexNameFor(database.updatedAt.toEpochMilli()); } /** @@ -371,6 +379,14 @@ private String indexNameFor(final long suffix) { return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, name, suffix); } + /** + * Reset database so that it can be updated in next run regardless there is new update or not + */ + public void resetDatabase() { + database.setUpdatedAt(null); + database.setSha256Hash(null); + } + /** * Checks if datasource is expired or not * @@ -537,7 +553,7 @@ public Database(final StreamInput in) throws IOException { public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalString(provider); out.writeOptionalString(sha256Hash); - out.writeOptionalVLong(updatedAt.toEpochMilli()); + out.writeOptionalVLong(updatedAt == null ? null : updatedAt.toEpochMilli()); out.writeOptionalVLong(validForInDays); out.writeOptionalStringCollection(fields); } @@ -640,10 +656,10 @@ public UpdateStats(final StreamInput in) throws IOException { @Override public void writeTo(final StreamOutput out) throws IOException { - out.writeOptionalVLong(lastSucceededAt.toEpochMilli()); + out.writeOptionalVLong(lastSucceededAt == null ? null : lastSucceededAt.toEpochMilli()); out.writeOptionalVLong(lastProcessingTimeInMillis); - out.writeOptionalVLong(lastFailedAt.toEpochMilli()); - out.writeOptionalVLong(lastSkippedAt.toEpochMilli()); + out.writeOptionalVLong(lastFailedAt == null ? null : lastFailedAt.toEpochMilli()); + out.writeOptionalVLong(lastSkippedAt == null ? null : lastSkippedAt.toEpochMilli()); } @Override diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java new file mode 100644 index 00000000..e58ab4b9 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java @@ -0,0 +1,112 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.listener; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.RestoreInProgress; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.inject.Inject; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.threadpool.ThreadPool; + +@Log4j2 +@AllArgsConstructor(onConstructor = @__(@Inject)) +public class Ip2GeoListener extends AbstractLifecycleComponent implements ClusterStateListener { + private static final int SCHEDULE_IN_MIN = 15; + private static final int DELAY_IN_MILLIS = 10000; + private final ClusterService clusterService; + private final ThreadPool threadPool; + private final DatasourceFacade datasourceFacade; + + @Override + public void clusterChanged(final ClusterChangedEvent event) { + if (event.localNodeClusterManager() == false) { + return; + } + + for (RestoreInProgress.Entry entry : event.state().custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)) { + if (RestoreInProgress.State.SUCCESS.equals(entry.state()) == false) { + continue; + } + + if (entry.indices().stream().anyMatch(index -> DatasourceExtension.JOB_INDEX_NAME.equals(index)) == false) { + continue; + } + + threadPool.generic().submit(() -> forceUpdateGeoIpData()); + } + } + + private void forceUpdateGeoIpData() { + datasourceFacade.getAllDatasources(new ActionListener<>() { + @Override + public void onResponse(final List datasources) { + datasources.stream().forEach(Ip2GeoListener.this::scheduleForceUpdate); + datasourceFacade.updateDatasource(datasources, new ActionListener<>() { + @Override + public void onResponse(final BulkResponse bulkItemResponses) { + log.info("Datasources are updated for cleanup"); + } + + @Override + public void onFailure(final Exception e) { + log.error("Failed to update datasource for cleanup after restoring", e); + } + }); + } + + @Override + public void onFailure(final Exception e) { + log.error("Failed to get datasource after restoring", e); + } + }); + } + + /** + * Give a delay so that job scheduler can schedule the job right after the delay. Otherwise, it schedules + * the job after specified update interval. + */ + private void scheduleForceUpdate(Datasource datasource) { + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), SCHEDULE_IN_MIN, ChronoUnit.MINUTES, DELAY_IN_MILLIS); + datasource.resetDatabase(); + datasource.setSystemSchedule(schedule); + datasource.setTask(DatasourceTask.ALL); + } + + @Override + protected void doStart() { + if (DiscoveryNode.isClusterManagerNode(clusterService.getSettings())) { + clusterService.addListener(this); + } + } + + @Override + protected void doStop() { + clusterService.removeListener(this); + } + + @Override + protected void doClose() throws IOException { + + } +} diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index f014256d..5f780045 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -20,6 +20,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.component.LifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; @@ -55,6 +56,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.geospatial.ip2geo.listener.Ip2GeoListener; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; @@ -107,6 +109,11 @@ public Map getProcessors(Processor.Parameters paramet .immutableMap(); } + @Override + public Collection> getGuiceServiceClasses() { + return List.of(Ip2GeoListener.class); + } + @Override public List> getExecutorBuilders(Settings settings) { List> executorBuilders = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index d2ca28bb..2a29afb1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -27,6 +27,7 @@ import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; @@ -311,6 +312,27 @@ public void testGetAllDatasources_whenValidInput_thenSucceed() { assertEquals(datasources, captor.getValue()); } + public void testUpdateDatasource_whenValidInput_thenUpdate() { + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof BulkRequest); + BulkRequest bulkRequest = (BulkRequest) actionRequest; + assertEquals(2, bulkRequest.requests().size()); + for (int i = 0; i < bulkRequest.requests().size(); i++) { + IndexRequest request = (IndexRequest) bulkRequest.requests().get(i); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals(datasources.get(i).getName(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + assertTrue(request.source().utf8ToString().contains(datasources.get(i).getEndpoint())); + } + return null; + }); + + datasourceFacade.updateDatasource(datasources, mock(ActionListener.class)); + } + private SearchHits getMockedSearchHits(List datasources) { SearchHit[] searchHitArray = datasources.stream().map(this::toBytesReference).map(this::toSearchHit).toArray(SearchHit[]::new); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 23981524..1f2210eb 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -78,6 +78,21 @@ public void testCurrentIndexName_whenExpired_thenReturnNull() { assertNull(datasource.currentIndexName()); } + public void testCurrentIndexName_whenDatabaseUpdateDateIsNull_thenReturnNull() { + String id = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource(); + datasource.setName(id); + datasource.getDatabase().setProvider("provider"); + datasource.getDatabase().setSha256Hash("sha256Hash"); + datasource.getDatabase().setUpdatedAt(null); + datasource.getDatabase().setValidForInDays(1l); + datasource.getUpdateStats().setLastSucceededAt(Instant.now()); + datasource.getDatabase().setFields(new ArrayList<>()); + + assertFalse(datasource.isExpired()); + assertNull(datasource.currentIndexName()); + } + public void testGetIndexNameFor() { long updatedAt = randomPositiveLong(); DatasourceManifest manifest = mock(DatasourceManifest.class); @@ -92,6 +107,19 @@ public void testGetIndexNameFor() { ); } + public void testResetDatabase_whenCalled_thenNullifySomeFields() { + Datasource datasource = randomDatasource(); + assertNotNull(datasource.getDatabase().getSha256Hash()); + assertNotNull(datasource.getDatabase().getUpdatedAt()); + + // Run + datasource.resetDatabase(); + + // Verify + assertNull(datasource.getDatabase().getSha256Hash()); + assertNull(datasource.getDatabase().getUpdatedAt()); + } + public void testIsExpired_whenCalled_thenExpectedValue() { Datasource datasource = new Datasource(); // never expire if validForInDays is null diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index d8fbc3e0..7ebab275 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -66,6 +66,35 @@ public void testUpdateOrCreateGeoIpData_whenHashValueIsSame_thenSkipUpdate() { verify(datasourceFacade).updateDatasource(datasource); } + @SneakyThrows + public void testUpdateOrCreateGeoIpData_whenExpired_thenUpdate() { + File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); + DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); + + File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); + when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + + Datasource datasource = new Datasource(); + datasource.setState(DatasourceState.AVAILABLE); + datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash()); + datasource.getDatabase().setValidForInDays(1l); + datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); + datasource.resetDatabase(); + + // Run + datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class)); + + // Verify + verify(geoIpDataFacade).putGeoIpData( + eq(datasource.currentIndexName()), + isA(String[].class), + any(Iterator.class), + anyInt(), + any(Runnable.class) + ); + } + @SneakyThrows public void testUpdateOrCreateGeoIpData_whenInvalidData_thenThrowException() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java new file mode 100644 index 00000000..ff2cd3e3 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java @@ -0,0 +1,173 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.listener; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.action.ActionListener; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.RestoreInProgress; +import org.opensearch.common.settings.Settings; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; +import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; + +public class Ip2GeoListenerTests extends Ip2GeoTestCase { + private Ip2GeoListener ip2GeoListener; + + @Before + public void init() { + ip2GeoListener = new Ip2GeoListener(clusterService, threadPool, datasourceFacade); + } + + public void testDoStart_whenClusterManagerNode_thenAddListener() { + Settings settings = Settings.builder().put("node.roles", "cluster_manager").build(); + when(clusterService.getSettings()).thenReturn(settings); + + // Run + ip2GeoListener.doStart(); + + // Verify + verify(clusterService).addListener(ip2GeoListener); + } + + public void testDoStart_whenNotClusterManagerNode_thenDoNotAddListener() { + Settings settings = Settings.builder().put("node.roles", "data").build(); + when(clusterService.getSettings()).thenReturn(settings); + + // Run + ip2GeoListener.doStart(); + + // Verify + verify(clusterService, never()).addListener(ip2GeoListener); + } + + public void testDoStop_whenCalled_thenRemoveListener() { + // Run + ip2GeoListener.doStop(); + + // Verify + verify(clusterService).removeListener(ip2GeoListener); + } + + public void testClusterChanged_whenNotClusterManagerNode_thenDoNothing() { + ClusterChangedEvent event = mock(ClusterChangedEvent.class); + when(event.localNodeClusterManager()).thenReturn(false); + + // Run + ip2GeoListener.clusterChanged(event); + + // Verify + verify(threadPool, never()).generic(); + } + + public void testClusterChanged_whenNotComplete_thenDoNothing() { + SnapshotId snapshotId = new SnapshotId(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString()); + Snapshot snapshot = new Snapshot(GeospatialTestHelper.randomLowerCaseString(), snapshotId); + RestoreInProgress.Entry entry = new RestoreInProgress.Entry( + GeospatialTestHelper.randomLowerCaseString(), + snapshot, + RestoreInProgress.State.STARTED, + Arrays.asList(DatasourceExtension.JOB_INDEX_NAME), + null + ); + RestoreInProgress restoreInProgress = new RestoreInProgress.Builder().add(entry).build(); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)).thenReturn(restoreInProgress); + ClusterChangedEvent event = mock(ClusterChangedEvent.class); + when(event.localNodeClusterManager()).thenReturn(true); + when(event.state()).thenReturn(clusterState); + + // Run + ip2GeoListener.clusterChanged(event); + + // Verify + verify(threadPool, never()).generic(); + } + + public void testClusterChanged_whenNotDatasourceIndex_thenDoNothing() { + SnapshotId snapshotId = new SnapshotId(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString()); + Snapshot snapshot = new Snapshot(GeospatialTestHelper.randomLowerCaseString(), snapshotId); + RestoreInProgress.Entry entry = new RestoreInProgress.Entry( + GeospatialTestHelper.randomLowerCaseString(), + snapshot, + RestoreInProgress.State.FAILURE, + Arrays.asList(GeospatialTestHelper.randomLowerCaseString()), + null + ); + RestoreInProgress restoreInProgress = new RestoreInProgress.Builder().add(entry).build(); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)).thenReturn(restoreInProgress); + ClusterChangedEvent event = mock(ClusterChangedEvent.class); + when(event.localNodeClusterManager()).thenReturn(true); + when(event.state()).thenReturn(clusterState); + + // Run + ip2GeoListener.clusterChanged(event); + + // Verify + verify(threadPool, never()).generic(); + } + + public void testClusterChanged_whenDatasourceIndexIsRestored_thenUpdate() { + SnapshotId snapshotId = new SnapshotId(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString()); + Snapshot snapshot = new Snapshot(GeospatialTestHelper.randomLowerCaseString(), snapshotId); + RestoreInProgress.Entry entry = new RestoreInProgress.Entry( + GeospatialTestHelper.randomLowerCaseString(), + snapshot, + RestoreInProgress.State.SUCCESS, + Arrays.asList(DatasourceExtension.JOB_INDEX_NAME), + null + ); + RestoreInProgress restoreInProgress = new RestoreInProgress.Builder().add(entry).build(); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)).thenReturn(restoreInProgress); + ClusterChangedEvent event = mock(ClusterChangedEvent.class); + when(event.localNodeClusterManager()).thenReturn(true); + when(event.state()).thenReturn(clusterState); + + // Run + ip2GeoListener.clusterChanged(event); + + // Verify + verify(threadPool).generic(); + ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getAllDatasources(captor.capture()); + + // Run + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + datasources.stream().forEach(datasource -> { datasource.setTask(DatasourceTask.DELETE_UNUSED_INDICES); }); + + captor.getValue().onResponse(datasources); + + // Verify + datasources.stream().forEach(datasource -> { + assertEquals(DatasourceTask.ALL, datasource.getTask()); + assertNull(datasource.getDatabase().getUpdatedAt()); + assertNull(datasource.getDatabase().getSha256Hash()); + assertTrue(datasource.getSystemSchedule().getNextExecutionTime(Instant.now()).isAfter(Instant.now())); + assertTrue(datasource.getSystemSchedule().getNextExecutionTime(Instant.now()).isBefore(Instant.now().plusSeconds(60))); + }); + verify(datasourceFacade).updateDatasource(eq(datasources), any()); + } + +} diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 596bd3f7..430268fb 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -25,6 +25,7 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.LifecycleComponent; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -42,6 +43,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; +import org.opensearch.geospatial.ip2geo.listener.Ip2GeoListener; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.geospatial.stats.upload.RestUploadStatsAction; @@ -159,6 +161,12 @@ public void testCreateComponents() { assertEquals(SUPPORTED_COMPONENTS, registeredComponents); } + public void testGetGuiceServiceClasses() { + GeospatialPlugin plugin = new GeospatialPlugin(); + Collection> classes = List.of(Ip2GeoListener.class); + assertEquals(classes, plugin.getGuiceServiceClasses()); + } + public void testIsAnIngestPlugin() { GeospatialPlugin plugin = new GeospatialPlugin(); assertTrue(plugin instanceof IngestPlugin); From d158239097ca164273de042c04f0b57af6d051fe Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 7 Jun 2023 14:17:02 -0700 Subject: [PATCH 46/61] Reverse forcemerge and refresh order (#331) Otherwise, opensearch does not clear old segment files Signed-off-by: Heemin Kim --- .../opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 3976ffd0..11e499a0 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -121,8 +121,8 @@ public void createIndexIfNotExists(final String indexName) { private void freezeIndex(final String indexName) { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); StashedThreadContext.run(client, () -> { - client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).execute().actionGet(timeout); + client.admin().indices().prepareRefresh(indexName).execute().actionGet(timeout); client.admin() .indices() .prepareUpdateSettings(indexName) From dff82060b73d09fb444757d1609c3f3c35b5f5e9 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 7 Jun 2023 15:14:55 -0700 Subject: [PATCH 47/61] Removed parameter and settings (#332) * Removed first_only parameter * Removed max_concurrency and batch_size setting first_only parameter was added as current geoip processor has it. However, the parameter have no benefit for ip2geo processor as we don't do a sequantial search for array data but use multi search. max_concurrency and batch_size setting is removed as these are only reveal internal implementation and could be a future blocker to improve performance later. Signed-off-by: Heemin Kim --- .../ip2geo/common/GeoIpDataFacade.java | 27 ++---- .../ip2geo/common/Ip2GeoSettings.java | 33 +------ .../jobscheduler/DatasourceUpdateService.java | 9 +- .../ip2geo/processor/Ip2GeoProcessor.java | 49 +++-------- .../geospatial/ip2geo/Ip2GeoTestCase.java | 1 - .../ip2geo/common/GeoIpDataFacadeTests.java | 39 ++------ .../DatasourceUpdateServiceTests.java | 3 - .../processor/Ip2GeoProcessorTests.java | 88 +------------------ 8 files changed, 33 insertions(+), 216 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index 11e499a0..10b2dbda 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -68,6 +68,7 @@ */ @Log4j2 public class GeoIpDataFacade { + public static final int BUNDLE_SIZE = 128; private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; private static final Map INDEX_SETTING_TO_CREATE = Map.of( @@ -279,28 +280,19 @@ public void onFailure(final Exception e) { * * @param indexName the index name * @param ipIterator the iterator of ip addresses - * @param maxBundleSize number of ip address to pass in multi search - * @param maxConcurrentSearches the max concurrent search requests - * @param firstOnly return only the first matching result if true * @param geoIpData collected geo data * @param actionListener the action listener */ public void getGeoIpData( final String indexName, final Iterator ipIterator, - final Integer maxBundleSize, - final Integer maxConcurrentSearches, - final boolean firstOnly, final Map> geoIpData, final ActionListener>> actionListener ) { MultiSearchRequestBuilder mRequestBuilder = client.prepareMultiSearch(); - if (maxConcurrentSearches != 0) { - mRequestBuilder.setMaxConcurrentSearchRequests(maxConcurrentSearches); - } - List ipsToSearch = new ArrayList<>(maxBundleSize); - while (ipIterator.hasNext() && ipsToSearch.size() < maxBundleSize) { + List ipsToSearch = new ArrayList<>(BUNDLE_SIZE); + while (ipIterator.hasNext() && ipsToSearch.size() < BUNDLE_SIZE) { String ip = ipIterator.next(); if (geoIpData.get(ip) == null) { mRequestBuilder.add( @@ -340,13 +332,8 @@ public void onResponse(final MultiSearchResponse items) { ).v2().get(DATA_FIELD_NAME); geoIpData.put(ipsToSearch.get(i), data); - - if (firstOnly) { - actionListener.onResponse(geoIpData); - return; - } } - getGeoIpData(indexName, ipIterator, maxBundleSize, maxConcurrentSearches, firstOnly, geoIpData, actionListener); + getGeoIpData(indexName, ipIterator, geoIpData, actionListener); } @Override @@ -362,20 +349,18 @@ public void onFailure(final Exception e) { * @param indexName Index name to puts the GeoIP data * @param fields Field name matching with data in CSVRecord in order * @param iterator GeoIP data to insert - * @param bulkSize Bulk size of data to process * @param renewLock Runnable to renew lock */ public void putGeoIpData( @NonNull final String indexName, @NonNull final String[] fields, @NonNull final Iterator iterator, - final int bulkSize, @NonNull final Runnable renewLock ) throws IOException { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); final BulkRequest bulkRequest = new BulkRequest(); Queue requests = new LinkedList<>(); - for (int i = 0; i < bulkSize; i++) { + for (int i = 0; i < BUNDLE_SIZE; i++) { requests.add(Requests.indexRequest(indexName)); } while (iterator.hasNext()) { @@ -385,7 +370,7 @@ public void putGeoIpData( indexRequest.source(document); indexRequest.id(record.get(0)); bulkRequest.add(indexRequest); - if (iterator.hasNext() == false || bulkRequest.requests().size() == bulkSize) { + if (iterator.hasNext() == false || bulkRequest.requests().size() == BUNDLE_SIZE) { BulkResponse response = StashedThreadContext.run(client, () -> client.bulk(bulkRequest).actionGet(timeout)); if (response.hasFailures()) { throw new OpenSearchException( diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index b236e95c..12d06b50 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -51,30 +51,6 @@ public class Ip2GeoSettings { Setting.Property.Dynamic ); - /** - * Bulk size for indexing GeoIP data - */ - public static final Setting INDEXING_BULK_SIZE = Setting.intSetting( - "plugins.geospatial.ip2geo.datasource.indexing_bulk_size", - 10000, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Multi search bundle size for GeoIP data - * - * Multi search is used only when a field contains a list of ip addresses. - */ - public static final Setting MAX_BUNDLE_SIZE = Setting.intSetting( - "plugins.geospatial.ip2geo.processor.max_bundle_size", - 100, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - /** * Multi search max concurrent searches * @@ -96,14 +72,7 @@ public class Ip2GeoSettings { * @return a list of all settings for Ip2Geo feature */ public static final List> settings() { - return List.of( - DATASOURCE_ENDPOINT, - DATASOURCE_UPDATE_INTERVAL, - TIMEOUT, - INDEXING_BULK_SIZE, - MAX_BUNDLE_SIZE, - MAX_CONCURRENT_SEARCHES - ); + return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, TIMEOUT, MAX_CONCURRENT_SEARCHES); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 0e6e16d1..9dc94570 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -25,7 +25,6 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @Log4j2 @@ -83,13 +82,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable datasource.getDatabase().getFields().toString() ); } - geoIpDataFacade.putGeoIpData( - indexName, - header, - reader.iterator(), - clusterSettings.get(Ip2GeoSettings.INDEXING_BULK_SIZE), - renewLock - ); + geoIpDataFacade.putGeoIpData(indexName, header, reader.iterator(), renewLock); } Instant endTime = Instant.now(); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index fcb32b56..5ce80a9a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -29,7 +29,6 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.IngestDocument; @@ -49,7 +48,6 @@ public final class Ip2GeoProcessor extends AbstractProcessor { public static final String CONFIG_DATASOURCE = "datasource"; public static final String CONFIG_PROPERTIES = "properties"; public static final String CONFIG_IGNORE_MISSING = "ignore_missing"; - public static final String CONFIG_FIRST_ONLY = "first_only"; private final String field; private final String targetField; @@ -60,7 +58,6 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final String datasourceName; private final Set properties; private final boolean ignoreMissing; - private final boolean firstOnly; private final ClusterSettings clusterSettings; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; @@ -79,7 +76,6 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param datasourceName the datasourceName * @param properties the properties * @param ignoreMissing true if documents with a missing value for the field should be ignored - * @param firstOnly true if only first result should be returned in case of array * @param clusterSettings the cluster settings * @param datasourceFacade the datasource facade * @param geoIpDataFacade the geoip data facade @@ -92,7 +88,6 @@ public Ip2GeoProcessor( final String datasourceName, final Set properties, final boolean ignoreMissing, - final boolean firstOnly, final ClusterSettings clusterSettings, final DatasourceFacade datasourceFacade, final GeoIpDataFacade geoIpDataFacade @@ -103,7 +98,6 @@ public Ip2GeoProcessor( this.datasourceName = datasourceName; this.properties = properties; this.ignoreMissing = ignoreMissing; - this.firstOnly = firstOnly; this.clusterSettings = clusterSettings; this.datasourceFacade = datasourceFacade; this.geoIpDataFacade = geoIpDataFacade; @@ -252,9 +246,6 @@ public void onResponse(final Datasource datasource) { geoIpDataFacade.getGeoIpData( indexName, ipList.iterator(), - clusterSettings.get(Ip2GeoSettings.MAX_BUNDLE_SIZE), - clusterSettings.get(Ip2GeoSettings.MAX_CONCURRENT_SEARCHES), - firstOnly, data, listenerToAppendDataToDocument(data, ipList, ingestDocument, handler) ); @@ -277,33 +268,21 @@ protected ActionListener>> listenerToAppendDataT return new ActionListener<>() { @Override public void onResponse(final Map> response) { - if (firstOnly) { - for (String ipAddr : ipList) { - Map geoData = data.get(ipAddr); - // GeoData for ipAddr won't be null - if (geoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, filteredGeoData(geoData, ipAddr)); - handler.accept(ingestDocument, null); - return; - } - } - } else { - boolean match = false; - List> geoDataList = new ArrayList<>(ipList.size()); - for (String ipAddr : ipList) { - Map geoData = data.get(ipAddr); - // GeoData for ipAddr won't be null - geoDataList.add(geoData.isEmpty() ? null : filteredGeoData(geoData, ipAddr)); - if (geoData.isEmpty() == false) { - match = true; - } - } - if (match) { - ingestDocument.setFieldValue(targetField, geoDataList); - handler.accept(ingestDocument, null); - return; + boolean match = false; + List> geoDataList = new ArrayList<>(ipList.size()); + for (String ipAddr : ipList) { + Map geoData = data.get(ipAddr); + // GeoData for ipAddr won't be null + geoDataList.add(geoData.isEmpty() ? null : filteredGeoData(geoData, ipAddr)); + if (geoData.isEmpty() == false) { + match = true; } } + if (match) { + ingestDocument.setFieldValue(targetField, geoDataList); + handler.accept(ingestDocument, null); + return; + } handler.accept(ingestDocument, null); } @@ -363,7 +342,6 @@ public Ip2GeoProcessor create( String datasourceName = readStringProperty(TYPE, processorTag, config, CONFIG_DATASOURCE); List propertyNames = readOptionalList(TYPE, processorTag, config, CONFIG_PROPERTIES); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, CONFIG_IGNORE_MISSING, false); - boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, CONFIG_FIRST_ONLY, true); // Skip validation for the call by cluster applier service if (Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME) == false) { @@ -378,7 +356,6 @@ public Ip2GeoProcessor create( datasourceName, propertyNames == null ? null : new HashSet<>(propertyNames), ignoreMissing, - firstOnly, ingestService.getClusterService().getClusterSettings(), datasourceFacade, geoIpDataFacade diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index fe3783cc..84609bae 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -258,7 +258,6 @@ protected Ip2GeoProcessor randomIp2GeoProcessor(String datasourceName) { datasourceName, properties, true, - true, clusterSettings, datasourceFacade, geoIpDataFacade diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index a5ca25fd..74d5f189 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade.BUNDLE_SIZE; import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; import java.io.File; @@ -193,7 +194,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { if (actionRequest instanceof BulkRequest) { BulkRequest request = (BulkRequest) actionRequest; - assertEquals(1, request.numberOfActions()); + assertEquals(2, request.numberOfActions()); BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(false); return response; @@ -224,7 +225,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { try (CSVParser csvParser = CSVParser.parse(sampleIp2GeoFile(), StandardCharsets.UTF_8, CSVFormat.RFC4180)) { Iterator iterator = csvParser.iterator(); String[] fields = iterator.next().values(); - verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, 1, renewLock); + verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, renewLock); verify(renewLock, times(2)).run(); } } @@ -261,44 +262,30 @@ public void testGetSingleGeoIpData() { assertEquals("seattle", captor.getValue().get("city")); } - public void testGetMultipleGeoIpDataNoSearchRequired() { + public void testGetGeoIpData_whenAllDataIsGathered_thenNoMoreSearch() { String indexName = GeospatialTestHelper.randomLowerCaseString(); String ip1 = randomIpAddress(); String ip2 = randomIpAddress(); Iterator ipIterator = Arrays.asList(ip1, ip2).iterator(); - int maxBundleSize = 1; - int maxConcurrentSearches = 1; - boolean firstOnly = true; Map> geoData = new HashMap<>(); geoData.put(ip1, Map.of("city", "Seattle")); geoData.put(ip2, Map.of("city", "Hawaii")); ActionListener>> actionListener = mock(ActionListener.class); // Run - verifyingGeoIpDataFacade.getGeoIpData( - indexName, - ipIterator, - maxBundleSize, - maxConcurrentSearches, - firstOnly, - geoData, - actionListener - ); + verifyingGeoIpDataFacade.getGeoIpData(indexName, ipIterator, geoData, actionListener); // Verify verify(actionListener).onResponse(geoData); } - public void testGetMultipleGeoIpData() { + public void testGetGeoIpData_whenCalled_thenGetGeoIpData() { String indexName = GeospatialTestHelper.randomLowerCaseString(); int dataSize = Randomness.get().nextInt(10) + 1; List ips = new ArrayList<>(); for (int i = 0; i < dataSize; i++) { ips.add(randomIpAddress()); } - int maxBundleSize = Randomness.get().nextInt(11) + 1; - int maxConcurrentSearches = 1; - boolean firstOnly = false; Map> geoData = new HashMap<>(); ActionListener>> actionListener = mock(ActionListener.class); @@ -306,8 +293,6 @@ public void testGetMultipleGeoIpData() { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assert actionRequest instanceof MultiSearchRequest; MultiSearchRequest request = (MultiSearchRequest) actionRequest; - assertEquals(maxConcurrentSearches, request.maxConcurrentSearchRequests()); - assertTrue(request.requests().size() == maxBundleSize || request.requests().size() == dataSize % maxBundleSize); for (SearchRequest searchRequest : request.requests()) { assertEquals("_local", searchRequest.preference()); assertEquals(1, searchRequest.source().size()); @@ -341,18 +326,10 @@ public void testGetMultipleGeoIpData() { }); // Run - verifyingGeoIpDataFacade.getGeoIpData( - indexName, - ips.iterator(), - maxBundleSize, - maxConcurrentSearches, - firstOnly, - geoData, - actionListener - ); + verifyingGeoIpDataFacade.getGeoIpData(indexName, ips.iterator(), geoData, actionListener); // Verify - verify(verifyingClient, times((dataSize + maxBundleSize - 1) / maxBundleSize)).execute( + verify(verifyingClient, times((dataSize + BUNDLE_SIZE - 1) / BUNDLE_SIZE)).execute( any(ActionType.class), any(ActionRequest.class), any(ActionListener.class) diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 7ebab275..8dcfbd7a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -6,7 +6,6 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; @@ -90,7 +89,6 @@ public void testUpdateOrCreateGeoIpData_whenExpired_thenUpdate() { eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), - anyInt(), any(Runnable.class) ); } @@ -167,7 +165,6 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), - anyInt(), any(Runnable.class) ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index e83fc764..2a7ce296 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -6,8 +6,6 @@ package org.opensearch.geospatial.ip2geo.processor; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -188,80 +186,9 @@ public void testExecuteNotImplemented() throws Exception { assertTrue(e.getMessage().contains("Not implemented")); } - public void testGenerateDataToAppendWithFirstOnlyOption() throws Exception { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor( - datasourceName, - Map.of("first_only", true, "properties", Arrays.asList(SUPPORTED_FIELDS.get(0))) - ); - List ips = new ArrayList<>(); - Map> data = new HashMap<>(); - for (int i = 0; i < 3; i++) { - String ip = randomIpAddress(); - ips.add(ip); - Map geoData = new HashMap<>(); - for (String field : SUPPORTED_FIELDS) { - geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); - } - data.put(ip, i == 0 ? Collections.emptyMap() : geoData); - } - IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - - // Run - processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); - - // Verify - verify(handler).accept(document, null); - assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).size()); - assertEquals( - data.get(ips.get(1)).get(SUPPORTED_FIELDS.get(0)), - document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get(SUPPORTED_FIELDS.get(0)) - ); - assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get(SUPPORTED_FIELDS.get(1))); - } - - public void testGenerateDataToAppendWithOutFirstOnlyOption() throws Exception { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor( - datasourceName, - Map.of("first_only", false, "properties", Arrays.asList(SUPPORTED_FIELDS.get(0))) - ); - List ips = new ArrayList<>(); - Map> data = new HashMap<>(); - for (int i = 0; i < 3; i++) { - String ip = randomIpAddress(); - ips.add(ip); - Map geoData = new HashMap<>(); - for (String field : SUPPORTED_FIELDS) { - geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); - } - data.put(ip, i == 0 ? Collections.emptyMap() : geoData); - } - IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - - // Run - processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); - - // Verify - verify(handler).accept(document, null); - assertEquals(ips.size(), document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); - for (int i = 0; i < ips.size(); i++) { - if (data.get(ips.get(i)).isEmpty()) { - assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).get(i)); - } else { - Map documentData = (Map) document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).get(i); - assertEquals(1, documentData.size()); - assertEquals(data.get(ips.get(i)).get(SUPPORTED_FIELDS.get(0)), documentData.get(SUPPORTED_FIELDS.get(0))); - assertNull(documentData.get(SUPPORTED_FIELDS.get(1))); - } - } - } - public void testGenerateDataToAppendWithNoData() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("first_only", Randomness.get().nextInt() % 2 == 0)); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); List ips = new ArrayList<>(); Map> data = new HashMap<>(); for (int i = 0; i < 3; i++) { @@ -291,7 +218,8 @@ public void testExecuteInternalNonStringIp() throws Exception { assertTrue(e.getMessage().contains("should only contain strings")); } - public void testExecuteInternal() throws Exception { + @SneakyThrows + public void testExecuteInternal_whenCalled_thenGetDatasourceIsCalled() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); @@ -309,15 +237,7 @@ public void testExecuteInternal() throws Exception { when(datasource.isExpired()).thenReturn(false); when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); captor.getValue().onResponse(datasource); - verify(geoIpDataFacade).getGeoIpData( - anyString(), - any(Iterator.class), - anyInt(), - anyInt(), - anyBoolean(), - anyMap(), - any(ActionListener.class) - ); + verify(geoIpDataFacade).getGeoIpData(anyString(), any(Iterator.class), anyMap(), any(ActionListener.class)); } @SneakyThrows From 860c90292ab26b2eb387497bd291dd31bad8bb0a Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 7 Jun 2023 19:13:03 -0700 Subject: [PATCH 48/61] Add a field in datasource for current index name (#333) Signed-off-by: Heemin Kim --- .../ip2geo/action/GetDatasourceResponse.java | 2 +- .../ip2geo/action/PutDatasourceRequest.java | 6 +- .../action/UpdateDatasourceRequest.java | 4 +- .../ip2geo/jobscheduler/Datasource.java | 46 ++++++++------ .../jobscheduler/DatasourceUpdateService.java | 19 +++--- .../geospatial/ip2geo/Ip2GeoTestCase.java | 1 + .../ip2geo/jobscheduler/DatasourceTests.java | 61 ++++++++----------- .../DatasourceUpdateServiceTests.java | 1 + 8 files changed, 71 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java index d0dc2dcc..c2e3cb0c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java @@ -26,7 +26,7 @@ */ @Getter @Setter -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class GetDatasourceResponse extends ActionResponse implements ToXContentObject { private static final ParseField FIELD_NAME_DATASOURCES = new ParseField("datasources"); private static final ParseField FIELD_NAME_NAME = new ParseField("name"); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 3426008b..9f2335f9 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -18,8 +18,8 @@ import lombok.extern.log4j.Log4j2; import org.opensearch.OpenSearchException; +import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -34,8 +34,8 @@ @Getter @Setter @Log4j2 -@EqualsAndHashCode -public class PutDatasourceRequest extends AcknowledgedRequest { +@EqualsAndHashCode(callSuper = false) +public class PutDatasourceRequest extends ActionRequest { private static final int MAX_DATASOURCE_NAME_BYTES = 255; public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java index 41a7fd90..45f0132e 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java @@ -16,8 +16,8 @@ import lombok.Setter; import lombok.extern.log4j.Log4j2; +import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; @@ -32,7 +32,7 @@ @Setter @Log4j2 @EqualsAndHashCode(callSuper = false) -public class UpdateDatasourceRequest extends AcknowledgedRequest { +public class UpdateDatasourceRequest extends ActionRequest { public static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); private static final int MAX_DATASOURCE_NAME_BYTES = 255; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index 0f884c32..dd63d746 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -89,6 +89,7 @@ public class Datasource implements Writeable, ScheduledJobParameter { */ private static final ParseField ENDPOINT_FIELD = new ParseField("endpoint"); private static final ParseField STATE_FIELD = new ParseField("state"); + private static final ParseField CURRENT_INDEX_FIELD = new ParseField("current_index"); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField DATABASE_FIELD = new ParseField("database"); private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats"); @@ -150,8 +151,14 @@ public class Datasource implements Writeable, ScheduledJobParameter { */ private DatasourceState state; /** - * @param indices A list of indices having GeoIP data - * @return A list of indices having GeoIP data + * @param currentIndex the current index name having GeoIP data + * @return the current index name having GeoIP data + */ + @Getter(AccessLevel.NONE) + private String currentIndex; + /** + * @param indices A list of indices having GeoIP data including currentIndex + * @return A list of indices having GeoIP data including currentIndex */ private List indices; /** @@ -181,9 +188,10 @@ public class Datasource implements Writeable, ScheduledJobParameter { DatasourceTask task = DatasourceTask.valueOf((String) args[6]); String endpoint = (String) args[7]; DatasourceState state = DatasourceState.valueOf((String) args[8]); - List indices = (List) args[9]; - Database database = (Database) args[10]; - UpdateStats updateStats = (UpdateStats) args[11]; + String currentIndex = (String) args[9]; + List indices = (List) args[10]; + Database database = (Database) args[11]; + UpdateStats updateStats = (UpdateStats) args[12]; Datasource parameter = new Datasource( name, lastUpdateTime, @@ -194,6 +202,7 @@ public class Datasource implements Writeable, ScheduledJobParameter { task, endpoint, state, + currentIndex, indices, database, updateStats @@ -212,6 +221,7 @@ public class Datasource implements Writeable, ScheduledJobParameter { PARSER.declareString(ConstructingObjectParser.constructorArg(), TASK_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), ENDPOINT_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CURRENT_INDEX_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), Database.PARSER, DATABASE_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), UpdateStats.PARSER, UPDATE_STATS_FIELD); @@ -233,6 +243,7 @@ public Datasource(final String name, final IntervalSchedule schedule, final Stri DatasourceTask.ALL, endpoint, DatasourceState.CREATING, + null, new ArrayList<>(), new Database(), new UpdateStats() @@ -249,6 +260,7 @@ public Datasource(final StreamInput in) throws IOException { task = DatasourceTask.valueOf(in.readString()); endpoint = in.readString(); state = DatasourceState.valueOf(in.readString()); + currentIndex = in.readOptionalString(); indices = in.readStringList(); database = new Database(in); updateStats = new UpdateStats(in); @@ -265,6 +277,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(task.name()); out.writeString(endpoint); out.writeString(state.name()); + out.writeOptionalString(currentIndex); out.writeStringCollection(indices); database.writeTo(out); updateStats.writeTo(out); @@ -292,6 +305,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(TASK_FIELD.getPreferredName(), task.name()); builder.field(ENDPOINT_FIELD.getPreferredName(), endpoint); builder.field(STATE_FIELD.getPreferredName(), state.name()); + if (currentIndex != null) { + builder.field(CURRENT_INDEX_FIELD.getPreferredName(), currentIndex); + } builder.field(INDICES_FIELD.getPreferredName(), indices); builder.field(DATABASE_FIELD.getPreferredName(), database); builder.field(UPDATE_STATS_FIELD.getPreferredName(), updateStats); @@ -358,25 +374,17 @@ public String currentIndexName() { return null; } - if (database.updatedAt == null) { - return null; - } - - return indexNameFor(database.updatedAt.toEpochMilli()); + return currentIndex; } /** - * Index name for a given manifest + * Index name for a datasource with given suffix * - * @param manifest manifest - * @return Index name for a given manifest + * @param suffix the suffix of a index name + * @return index name for a datasource with given suffix */ - public String indexNameFor(final DatasourceManifest manifest) { - return indexNameFor(manifest.getUpdatedAt()); - } - - private String indexNameFor(final long suffix) { - return String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, name, suffix); + public String newIndexName(final String suffix) { + return String.format(Locale.ROOT, "%s.%s.%s", IP2GEO_DATA_INDEX_NAME_PREFIX, name, suffix); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 9dc94570..579f5bbe 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; @@ -68,7 +69,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable } Instant startTime = Instant.now(); - String indexName = setupIndex(manifest, datasource); + String indexName = setupIndex(datasource); String[] header; List fieldsToStore; try (CSVParser reader = geoIpDataFacade.getDatabaseReader(manifest)) { @@ -86,7 +87,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable } Instant endTime = Instant.now(); - updateDatasourceAsSucceeded(datasource, manifest, fieldsToStore, startTime, endTime); + updateDatasourceAsSucceeded(indexName, datasource, manifest, fieldsToStore, startTime, endTime); } /** @@ -199,16 +200,16 @@ private CSVRecord validateHeader(CSVRecord header) { * * @param manifest the manifest * @param datasource the datasource - * @return - * @throws IOException */ private void updateDatasourceAsSucceeded( + final String newIndexName, final Datasource datasource, final DatasourceManifest manifest, final List fields, final Instant startTime, final Instant endTime - ) throws IOException { + ) { + datasource.setCurrentIndex(newIndexName); datasource.setDatabase(manifest, fields); datasource.getUpdateStats().setLastSucceededAt(endTime); datasource.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); @@ -225,13 +226,11 @@ private void updateDatasourceAsSucceeded( /*** * Setup index to add a new geoip data * - * @param manifest the manifest * @param datasource the datasource - * @return - * @throws IOException + * @return new index name */ - private String setupIndex(final DatasourceManifest manifest, final Datasource datasource) throws IOException { - String indexName = datasource.indexNameFor(manifest); + private String setupIndex(final Datasource datasource) { + String indexName = datasource.newIndexName(UUID.randomUUID().toString()); datasource.getIndices().add(indexName); datasourceFacade.updateDatasource(datasource); geoIpDataFacade.createIndexIfNotExists(indexName); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 84609bae..f8b40232 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -208,6 +208,7 @@ protected Datasource randomDatasource(final Instant updateStartTime) { datasource.setSystemSchedule(datasource.getUserSchedule()); datasource.setTask(randomTask()); datasource.setState(randomState()); + datasource.setCurrentIndex(GeospatialTestHelper.randomLowerCaseString()); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase() diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 1f2210eb..aaa0d29b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -5,8 +5,6 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; import java.time.Instant; @@ -15,20 +13,23 @@ import java.util.Arrays; import java.util.Locale; +import lombok.SneakyThrows; + import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; -import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; public class DatasourceTests extends Ip2GeoTestCase { - public void testParser() throws Exception { + @SneakyThrows + public void testParser_whenAllValueIsFilled_thenSucceed() { String id = GeospatialTestHelper.randomLowerCaseString(); IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); String endpoint = GeospatialTestHelper.randomLowerCaseString(); Datasource datasource = new Datasource(id, schedule, endpoint); datasource.enable(); + datasource.setCurrentIndex(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setFields(Arrays.asList("field1", "field2")); datasource.getDatabase().setProvider("test_provider"); datasource.getDatabase().setUpdatedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); @@ -46,20 +47,31 @@ public void testParser() throws Exception { assertTrue(datasource.equals(anotherDatasource)); } + @SneakyThrows + public void testParser_whenNullForOptionalFields_thenSucceed() { + String id = GeospatialTestHelper.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + String endpoint = GeospatialTestHelper.randomLowerCaseString(); + Datasource datasource = new Datasource(id, schedule, endpoint); + Datasource anotherDatasource = Datasource.PARSER.parse( + createParser(datasource.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + assertTrue(datasource.equals(anotherDatasource)); + } + public void testCurrentIndexName_whenNotExpired_thenReturnName() { String id = GeospatialTestHelper.randomLowerCaseString(); Instant now = Instant.now(); Datasource datasource = new Datasource(); datasource.setName(id); + datasource.setCurrentIndex(datasource.newIndexName(GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase().setProvider("provider"); datasource.getDatabase().setSha256Hash("sha256Hash"); datasource.getDatabase().setUpdatedAt(now); datasource.getDatabase().setFields(new ArrayList<>()); - assertEquals( - String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, now.toEpochMilli()), - datasource.currentIndexName() - ); + assertNotNull(datasource.currentIndexName()); } public void testCurrentIndexName_whenExpired_thenReturnNull() { @@ -67,6 +79,7 @@ public void testCurrentIndexName_whenExpired_thenReturnNull() { Instant now = Instant.now(); Datasource datasource = new Datasource(); datasource.setName(id); + datasource.setCurrentIndex(datasource.newIndexName(GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase().setProvider("provider"); datasource.getDatabase().setSha256Hash("sha256Hash"); datasource.getDatabase().setUpdatedAt(now); @@ -78,33 +91,13 @@ public void testCurrentIndexName_whenExpired_thenReturnNull() { assertNull(datasource.currentIndexName()); } - public void testCurrentIndexName_whenDatabaseUpdateDateIsNull_thenReturnNull() { - String id = GeospatialTestHelper.randomLowerCaseString(); - Datasource datasource = new Datasource(); - datasource.setName(id); - datasource.getDatabase().setProvider("provider"); - datasource.getDatabase().setSha256Hash("sha256Hash"); - datasource.getDatabase().setUpdatedAt(null); - datasource.getDatabase().setValidForInDays(1l); - datasource.getUpdateStats().setLastSucceededAt(Instant.now()); - datasource.getDatabase().setFields(new ArrayList<>()); - - assertFalse(datasource.isExpired()); - assertNull(datasource.currentIndexName()); - } - - public void testGetIndexNameFor() { - long updatedAt = randomPositiveLong(); - DatasourceManifest manifest = mock(DatasourceManifest.class); - when(manifest.getUpdatedAt()).thenReturn(updatedAt); - - String id = GeospatialTestHelper.randomLowerCaseString(); + @SneakyThrows + public void testNewIndexName_whenCalled_thenReturnedExpectedValue() { + String name = GeospatialTestHelper.randomLowerCaseString(); + String suffix = GeospatialTestHelper.randomLowerCaseString(); Datasource datasource = new Datasource(); - datasource.setName(id); - assertEquals( - String.format(Locale.ROOT, "%s.%s.%d", IP2GEO_DATA_INDEX_NAME_PREFIX, id, updatedAt), - datasource.indexNameFor(manifest) - ); + datasource.setName(name); + assertEquals(String.format(Locale.ROOT, "%s.%s.%s", IP2GEO_DATA_INDEX_NAME_PREFIX, name, suffix), datasource.newIndexName(suffix)); } public void testResetDatabase_whenCalled_thenNullifySomeFields() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 8dcfbd7a..f650fc98 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -190,6 +190,7 @@ public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { String lingeringIndex = indexPrefix + now.minusMillis(2).toEpochMilli(); Datasource datasource = new Datasource(); datasource.setName(datasourceName); + datasource.setCurrentIndex(currentIndex); datasource.getIndices().add(currentIndex); datasource.getIndices().add(oldIndex); datasource.getIndices().add(lingeringIndex); From fb1994883387884468b9a73ac7c3ef707eaf8584 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 8 Jun 2023 15:18:23 -0700 Subject: [PATCH 49/61] Delete GeoIP data indices after restoring complete (#334) We don't want to use restored GeoIP data indices. Therefore we delete the indices once restoring process complete. When GeoIP metadata index is restored, we create a new GeoIP data index instead. Signed-off-by: Heemin Kim --- .../DeleteDatasourceTransportAction.java | 5 ++ .../ip2geo/common/DatasourceFacade.java | 10 ---- .../ip2geo/common/GeoIpDataFacade.java | 30 ++++++++--- .../ip2geo/jobscheduler/Datasource.java | 2 +- .../jobscheduler/DatasourceUpdateService.java | 7 +-- .../ip2geo/listener/Ip2GeoListener.java | 17 ++++-- .../geospatial/ip2geo/Ip2GeoTestCase.java | 3 +- .../DeleteDatasourceTransportActionTests.java | 9 +++- .../ip2geo/common/DatasourceFacadeTests.java | 54 +++++++++---------- .../ip2geo/common/GeoIpDataFacadeTests.java | 7 ++- .../DatasourceUpdateServiceTests.java | 3 +- .../ip2geo/listener/Ip2GeoListenerTests.java | 28 +++++++++- 12 files changed, 113 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index a4c6e5c4..a4706834 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -20,6 +20,7 @@ import org.opensearch.geospatial.exceptions.ResourceInUseException; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; @@ -36,6 +37,7 @@ public class DeleteDatasourceTransportAction extends HandledTransportAction indices) { + if (indices == null || indices.isEmpty()) { + return; + } + + Optional invalidIndex = indices.stream() + .filter(index -> index.startsWith(IP2GEO_DATA_INDEX_NAME_PREFIX) == false) + .findAny(); + if (invalidIndex.isPresent()) { throw new OpenSearchException( "the index[{}] is not ip2geo data index which should start with {}", - index, + invalidIndex.get(), IP2GEO_DATA_INDEX_NAME_PREFIX ); } - return StashedThreadContext.run( + + AcknowledgedResponse response = StashedThreadContext.run( client, () -> client.admin() .indices() - .prepareDelete(index) - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .prepareDelete(indices.toArray(new String[0])) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) .execute() .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) ); + + if (response.isAcknowledged() == false) { + throw new OpenSearchException("failed to delete data[{}] in datasource", String.join(",", indices)); + } } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index dd63d746..b3d6b328 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -50,7 +50,7 @@ public class Datasource implements Writeable, ScheduledJobParameter { /** * Prefix of indices having Ip2Geo data */ - public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".ip2geo-data"; + public static final String IP2GEO_DATA_INDEX_NAME_PREFIX = ".geospatial.ip2geo.data"; /** * Default fields for job scheduling diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 579f5bbe..d4e55f79 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -164,11 +164,8 @@ private List deleteIndices(final List indicesToDelete) { } try { - if (geoIpDataFacade.deleteIp2GeoDataIndex(index).isAcknowledged()) { - deletedIndices.add(index); - } else { - log.error("Failed to delete an index [{}]", index); - } + geoIpDataFacade.deleteIp2GeoDataIndex(index); + deletedIndices.add(index); } catch (Exception e) { log.error("Failed to delete an index [{}]", index, e); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java index e58ab4b9..a06bb30c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java @@ -5,10 +5,13 @@ package org.opensearch.geospatial.ip2geo.listener; +import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; + import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -23,6 +26,7 @@ import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; @@ -37,6 +41,7 @@ public class Ip2GeoListener extends AbstractLifecycleComponent implements Cluste private final ClusterService clusterService; private final ThreadPool threadPool; private final DatasourceFacade datasourceFacade; + private final GeoIpDataFacade geoIpDataFacade; @Override public void clusterChanged(final ClusterChangedEvent event) { @@ -49,11 +54,17 @@ public void clusterChanged(final ClusterChangedEvent event) { continue; } - if (entry.indices().stream().anyMatch(index -> DatasourceExtension.JOB_INDEX_NAME.equals(index)) == false) { - continue; + if (entry.indices().stream().anyMatch(index -> DatasourceExtension.JOB_INDEX_NAME.equals(index))) { + threadPool.generic().submit(() -> forceUpdateGeoIpData()); } - threadPool.generic().submit(() -> forceUpdateGeoIpData()); + List ip2GeoDataIndices = entry.indices() + .stream() + .filter(index -> index.startsWith(IP2GEO_DATA_INDEX_NAME_PREFIX)) + .collect(Collectors.toList()); + if (ip2GeoDataIndices.isEmpty() == false) { + threadPool.generic().submit(() -> geoIpDataFacade.deleteIp2GeoDataIndex(ip2GeoDataIndices)); + } } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index f8b40232..01f23e63 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.Locale; import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -208,7 +209,7 @@ protected Datasource randomDatasource(final Instant updateStartTime) { datasource.setSystemSchedule(datasource.getUserSchedule()); datasource.setTask(randomTask()); datasource.setState(randomState()); - datasource.setCurrentIndex(GeospatialTestHelper.randomLowerCaseString()); + datasource.setCurrentIndex(datasource.newIndexName(UUID.randomUUID().toString())); datasource.setIndices(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.setEndpoint(String.format(Locale.ROOT, "https://%s.com/manifest.json", GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase() diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java index 71447454..fa2afef8 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -23,6 +23,8 @@ import org.junit.Before; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mockito; import org.opensearch.OpenSearchException; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; @@ -45,6 +47,7 @@ public void init() { ip2GeoLockService, ingestService, datasourceFacade, + geoIpDataFacade, ip2GeoProcessorFacade ); } @@ -118,7 +121,9 @@ public void testDeleteDatasource_whenSafeToDelete_thenDelete() { // Verify assertEquals(DatasourceState.DELETING, datasource.getState()); verify(datasourceFacade).updateDatasource(datasource); - verify(datasourceFacade).deleteDatasource(datasource); + InOrder inOrder = Mockito.inOrder(geoIpDataFacade, datasourceFacade); + inOrder.verify(geoIpDataFacade).deleteIp2GeoDataIndex(datasource.getIndices()); + inOrder.verify(datasourceFacade).deleteDatasource(datasource); } @SneakyThrows @@ -136,6 +141,7 @@ public void testDeleteDatasource_whenProcessorIsUsingDatasource_thenThrowExcepti // Verify assertEquals(DatasourceState.AVAILABLE, datasource.getState()); verify(datasourceFacade, never()).updateDatasource(datasource); + verify(geoIpDataFacade, never()).deleteIp2GeoDataIndex(datasource.getIndices()); verify(datasourceFacade, never()).deleteDatasource(datasource); } @@ -154,6 +160,7 @@ public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowE // Verify verify(datasourceFacade, times(2)).updateDatasource(datasource); + verify(geoIpDataFacade, never()).deleteIp2GeoDataIndex(datasource.getIndices()); verify(datasourceFacade, never()).deleteDatasource(datasource); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 2a29afb1..43d8727e 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -22,11 +22,11 @@ import org.junit.Before; import org.mockito.ArgumentCaptor; import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.delete.DeleteResponse; @@ -38,9 +38,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; @@ -218,36 +216,36 @@ private Datasource setupClientForGetRequest(final boolean isExist, final Runtime public void testDeleteDatasource_whenValidInput_thenSucceed() { Datasource datasource = randomDatasource(); - verifyingClient.setExecuteVerifier( - (actionResponse, actionRequest) -> { - // Verify - if (actionRequest instanceof DeleteIndexRequest) { - DeleteIndexRequest request = (DeleteIndexRequest) actionRequest; - assertEquals(datasource.getIndices().size(), request.indices().length); - assertEquals(IndicesOptions.LENIENT_EXPAND_OPEN, request.indicesOptions()); - - AcknowledgedResponse response = new AcknowledgedResponse(true); - return response; - } else if (actionRequest instanceof DeleteRequest) { - DeleteRequest request = (DeleteRequest) actionRequest; - assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); - assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); - assertEquals(datasource.getName(), request.id()); - assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); - - DeleteResponse response = mock(DeleteResponse.class); - when(response.status()).thenReturn(RestStatus.OK); - return response; - } else { - throw new RuntimeException("Not expected request type is passed" + actionRequest.getClass()); - } - } - ); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof DeleteRequest); + DeleteRequest request = (DeleteRequest) actionRequest; + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); + assertEquals(datasource.getName(), request.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.OK); + return response; + }); // Run datasourceFacade.deleteDatasource(datasource); } + public void testDeleteDatasource_whenIndexNotFound_thenThrowException() { + Datasource datasource = randomDatasource(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.NOT_FOUND); + return response; + }); + + // Run + expectThrows(ResourceNotFoundException.class, () -> datasourceFacade.deleteDatasource(datasource)); + } + public void testGetDatasources_whenValidInput_thenSucceed() { List datasources = Arrays.asList(randomDatasource(), randomDatasource()); String[] names = datasources.stream().map(Datasource::getName).toArray(String[]::new); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 74d5f189..6ddac0a5 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -51,7 +51,7 @@ import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.Randomness; import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; @@ -168,15 +168,14 @@ public void testInternalGetDatabaseReader_whenCalled_thenSetUserAgent() { verify(connection).addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); } - public void testDeleteIp2GeoDataIndex() { + public void testDeleteIp2GeoDataIndex_whenCalled_thenDeleteIndex() { String index = String.format(Locale.ROOT, "%s.%s", IP2GEO_DATA_INDEX_NAME_PREFIX, GeospatialTestHelper.randomLowerCaseString()); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof DeleteIndexRequest); DeleteIndexRequest request = (DeleteIndexRequest) actionRequest; assertEquals(1, request.indices().length); assertEquals(index, request.indices()[0]); - assertEquals(IndicesOptions.LENIENT_EXPAND_OPEN, request.indicesOptions()); - return null; + return new AcknowledgedResponse(true); }); verifyingGeoIpDataFacade.deleteIp2GeoDataIndex(index); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index f650fc98..b110ddea 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -28,7 +28,6 @@ import org.apache.commons.csv.CSVParser; import org.junit.Before; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.SuppressForbidden; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -199,13 +198,13 @@ public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { when(metadata.hasIndex(currentIndex)).thenReturn(true); when(metadata.hasIndex(oldIndex)).thenReturn(true); when(metadata.hasIndex(lingeringIndex)).thenReturn(false); - when(geoIpDataFacade.deleteIp2GeoDataIndex(any())).thenReturn(new AcknowledgedResponse(true)); datasourceUpdateService.deleteUnusedIndices(datasource); assertEquals(1, datasource.getIndices().size()); assertEquals(currentIndex, datasource.getIndices().get(0)); verify(datasourceFacade).updateDatasource(datasource); + verify(geoIpDataFacade).deleteIp2GeoDataIndex(oldIndex); } public void testUpdateDatasource_whenNoChange_thenNoUpdate() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java index ff2cd3e3..a72a5a1b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java @@ -36,7 +36,7 @@ public class Ip2GeoListenerTests extends Ip2GeoTestCase { @Before public void init() { - ip2GeoListener = new Ip2GeoListener(clusterService, threadPool, datasourceFacade); + ip2GeoListener = new Ip2GeoListener(clusterService, threadPool, datasourceFacade, geoIpDataFacade); } public void testDoStart_whenClusterManagerNode_thenAddListener() { @@ -170,4 +170,30 @@ public void testClusterChanged_whenDatasourceIndexIsRestored_thenUpdate() { verify(datasourceFacade).updateDatasource(eq(datasources), any()); } + public void testClusterChanged_whenGeoIpDataIsRestored_thenDelete() { + Datasource datasource = randomDatasource(); + SnapshotId snapshotId = new SnapshotId(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString()); + Snapshot snapshot = new Snapshot(GeospatialTestHelper.randomLowerCaseString(), snapshotId); + RestoreInProgress.Entry entry = new RestoreInProgress.Entry( + GeospatialTestHelper.randomLowerCaseString(), + snapshot, + RestoreInProgress.State.SUCCESS, + Arrays.asList(datasource.currentIndexName()), + null + ); + RestoreInProgress restoreInProgress = new RestoreInProgress.Builder().add(entry).build(); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)).thenReturn(restoreInProgress); + ClusterChangedEvent event = mock(ClusterChangedEvent.class); + when(event.localNodeClusterManager()).thenReturn(true); + when(event.state()).thenReturn(clusterState); + + // Run + ip2GeoListener.clusterChanged(event); + + // Verify + verify(threadPool).generic(); + verify(geoIpDataFacade).deleteIp2GeoDataIndex(Arrays.asList(datasource.currentIndexName())); + } + } From 10f9107a7e7198cd33f516741b76bb0d46bab29a Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 9 Jun 2023 13:46:31 -0700 Subject: [PATCH 50/61] Use bool query for array form of IPs (#335) Signed-off-by: Heemin Kim --- .../ip2geo/common/GeoIpDataFacade.java | 123 +++++------ .../ip2geo/processor/Ip2GeoProcessor.java | 134 ++++++------ .../ip2geo/common/GeoIpDataFacadeTests.java | 105 +++------ .../ip2geo/processor/Ip2GeoProcessorIT.java | 10 +- .../processor/Ip2GeoProcessorTests.java | 202 ++++++++++-------- 5 files changed, 266 insertions(+), 308 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java index dc3e431e..6b3d71d7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java @@ -16,7 +16,6 @@ import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -44,8 +43,6 @@ import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.search.MultiSearchRequestBuilder; -import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -63,6 +60,7 @@ import org.opensearch.geospatial.constants.IndexSetting; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; /** @@ -255,15 +253,19 @@ public void getGeoIpData(final String indexName, final String ip, final ActionLi .execute(new ActionListener<>() { @Override public void onResponse(final SearchResponse searchResponse) { - if (searchResponse.getHits().getHits().length == 0) { - actionListener.onResponse(Collections.emptyMap()); - } else { - Map geoIpData = (Map) XContentHelper.convertToMap( - searchResponse.getHits().getAt(0).getSourceRef(), - false, - XContentType.JSON - ).v2().get(DATA_FIELD_NAME); - actionListener.onResponse(geoIpData); + try { + if (searchResponse.getHits().getHits().length == 0) { + actionListener.onResponse(Collections.emptyMap()); + } else { + Map geoIpData = (Map) XContentHelper.convertToMap( + searchResponse.getHits().getAt(0).getSourceRef(), + false, + XContentType.JSON + ).v2().get(DATA_FIELD_NAME); + actionListener.onResponse(geoIpData); + } + } catch (Exception e) { + actionListener.onFailure(e); } } @@ -276,73 +278,56 @@ public void onFailure(final Exception e) { } /** - * Query a given index using a given ip address iterator to get geoip data + * Query a given index using a given list of ip addresses to get geoip data * - * This method calls itself recursively until it processes all ip addresses in bulk of {@code bulkSize}. - * - * @param indexName the index name - * @param ipIterator the iterator of ip addresses - * @param geoIpData collected geo data - * @param actionListener the action listener + * @param indexName index + * @param ips list of ip addresses + * @param actionListener action listener */ public void getGeoIpData( final String indexName, - final Iterator ipIterator, - final Map> geoIpData, - final ActionListener>> actionListener + final List ips, + final ActionListener>> actionListener ) { - MultiSearchRequestBuilder mRequestBuilder = client.prepareMultiSearch(); - - List ipsToSearch = new ArrayList<>(BUNDLE_SIZE); - while (ipIterator.hasNext() && ipsToSearch.size() < BUNDLE_SIZE) { - String ip = ipIterator.next(); - if (geoIpData.get(ip) == null) { - mRequestBuilder.add( - client.prepareSearch(indexName) - .setSize(1) - .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) - .setPreference("_local") - .setRequestCache(true) - ); - ipsToSearch.add(ip); - } - } - - if (ipsToSearch.isEmpty()) { - actionListener.onResponse(geoIpData); - return; - } - - StashedThreadContext.run(client, () -> mRequestBuilder.execute(new ActionListener<>() { - @Override - public void onResponse(final MultiSearchResponse items) { - for (int i = 0; i < ipsToSearch.size(); i++) { - if (items.getResponses()[i].isFailure()) { - actionListener.onFailure(items.getResponses()[i].getFailure()); - return; + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + ips.stream().forEach(ip -> boolQueryBuilder.should(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip))); + StashedThreadContext.run( + client, + () -> client.prepareSearch(indexName) + .setSize(ips.size()) + .setQuery(boolQueryBuilder) + .setPreference("_local") + .setRequestCache(true) + .execute(new ActionListener<>() { + @Override + public void onResponse(final SearchResponse searchResponse) { + try { + actionListener.onResponse(toGeoIpDataList(searchResponse)); + } catch (Exception e) { + actionListener.onFailure(e); + } } - if (items.getResponses()[i].getResponse().getHits().getHits().length == 0) { - geoIpData.put(ipsToSearch.get(i), Collections.emptyMap()); - continue; + @Override + public void onFailure(final Exception e) { + actionListener.onFailure(e); } + }) + ); + } - Map data = (Map) XContentHelper.convertToMap( - items.getResponses()[i].getResponse().getHits().getAt(0).getSourceRef(), - false, - XContentType.JSON - ).v2().get(DATA_FIELD_NAME); - - geoIpData.put(ipsToSearch.get(i), data); - } - getGeoIpData(indexName, ipIterator, geoIpData, actionListener); - } + private List> toGeoIpDataList(final SearchResponse searchResponse) { + if (searchResponse.getHits().getHits().length == 0) { + return Collections.emptyList(); + } - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - })); + return Arrays.stream(searchResponse.getHits().getHits()) + .map( + data -> (Map) XContentHelper.convertToMap(data.getSourceRef(), false, XContentType.JSON) + .v2() + .get(DATA_FIELD_NAME) + ) + .collect(Collectors.toList()); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 5ce80a9a..5c5f3280 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -11,10 +11,9 @@ import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; @@ -41,8 +40,6 @@ @Log4j2 public final class Ip2GeoProcessor extends AbstractProcessor { private static final Map DATA_EXPIRED = Map.of("error", "ip2geo_data_expired"); - private static final String PROPERTY_IP = "ip"; - public static final String CONFIG_FIELD = "field"; public static final String CONFIG_TARGET_FIELD = "target_field"; public static final String CONFIG_DATASOURCE = "datasource"; @@ -111,19 +108,28 @@ public Ip2GeoProcessor( */ @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { - Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); + try { + Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); - if (ip == null) { - handler.accept(ingestDocument, null); - return; - } + if (ip == null) { + handler.accept(ingestDocument, null); + return; + } - if (ip instanceof String) { - executeInternal(ingestDocument, handler, (String) ip); - } else if (ip instanceof List) { - executeInternal(ingestDocument, handler, ((List) ip)); - } else { - throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings"); + if (ip instanceof String) { + executeInternal(ingestDocument, handler, (String) ip); + } else if (ip instanceof List) { + executeInternal(ingestDocument, handler, ((List) ip)); + } else { + handler.accept( + null, + new IllegalArgumentException( + String.format(Locale.ROOT, "field [%s] should contain only string or array of strings", field) + ) + ); + } + } catch (Exception e) { + handler.accept(null, e); } } @@ -159,17 +165,11 @@ public void onResponse(final Datasource datasource) { return; } - geoIpDataFacade.getGeoIpData(indexName, ip, new ActionListener<>() { - @Override - public void onResponse(final Map ipToGeoData) { - handleSingleIp(ip, ipToGeoData, ingestDocument, handler); - } - - @Override - public void onFailure(final Exception e) { - handler.accept(null, e); - } - }); + try { + geoIpDataFacade.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); + } catch (Exception e) { + handler.accept(null, e); + } } @Override @@ -180,32 +180,44 @@ public void onFailure(final Exception e) { } @VisibleForTesting - protected void handleSingleIp( - final String ip, - final Map ipToGeoData, + protected ActionListener> getSingleGeoIpDataListener( final IngestDocument ingestDocument, final BiConsumer handler ) { - if (ipToGeoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData, ip)); - } - handler.accept(ingestDocument, null); + return new ActionListener<>() { + @Override + public void onResponse(final Map ipToGeoData) { + try { + if (ipToGeoData.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData)); + } + handler.accept(ingestDocument, null); + } catch (Exception e) { + handler.accept(null, e); + } + } + + @Override + public void onFailure(final Exception e) { + handler.accept(null, e); + } + }; } - private Map filteredGeoData(final Map geoData, final String ip) { - Map filteredGeoData; + private Map filteredGeoData(final Map geoData) { if (properties == null) { return geoData; } - filteredGeoData = properties.stream() - .filter(p -> p.equals(PROPERTY_IP) == false) - .filter(p -> geoData.containsKey(p)) - .collect(Collectors.toMap(p -> p, p -> geoData.get(p))); - if (properties.contains(PROPERTY_IP)) { - filteredGeoData.put(PROPERTY_IP, ip); + return properties.stream().filter(p -> geoData.containsKey(p)).collect(Collectors.toMap(p -> p, p -> geoData.get(p))); + } + + private List> filteredGeoData(final List> geoData) { + if (properties == null) { + return geoData; } - return filteredGeoData; + + return geoData.stream().map(this::filteredGeoData).collect(Collectors.toList()); } /** @@ -221,13 +233,11 @@ protected void executeInternal( final BiConsumer handler, final List ips ) { - Map> data = new HashMap<>(); for (Object ip : ips) { if (ip instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } } - List ipList = (List) ips; datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { @@ -243,12 +253,11 @@ public void onResponse(final Datasource datasource) { return; } - geoIpDataFacade.getGeoIpData( - indexName, - ipList.iterator(), - data, - listenerToAppendDataToDocument(data, ipList, ingestDocument, handler) - ); + try { + geoIpDataFacade.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); + } catch (Exception e) { + handler.accept(null, e); + } } @Override @@ -259,31 +268,21 @@ public void onFailure(final Exception e) { } @VisibleForTesting - protected ActionListener>> listenerToAppendDataToDocument( - final Map> data, - final List ipList, + protected ActionListener>> getMultiGeoIpDataListener( final IngestDocument ingestDocument, final BiConsumer handler ) { return new ActionListener<>() { @Override - public void onResponse(final Map> response) { - boolean match = false; - List> geoDataList = new ArrayList<>(ipList.size()); - for (String ipAddr : ipList) { - Map geoData = data.get(ipAddr); - // GeoData for ipAddr won't be null - geoDataList.add(geoData.isEmpty() ? null : filteredGeoData(geoData, ipAddr)); - if (geoData.isEmpty() == false) { - match = true; + public void onResponse(final List> ipToGeoData) { + try { + if (ipToGeoData.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData)); } - } - if (match) { - ingestDocument.setFieldValue(targetField, geoDataList); handler.accept(ingestDocument, null); - return; + } catch (Exception e) { + handler.accept(null, e); } - handler.accept(ingestDocument, null); } @Override @@ -384,7 +383,6 @@ private void validate(final String processorTag, final String datasourceName, fi // Validate properties are valid. If not add all available properties. final Set availableProperties = new HashSet<>(datasource.getDatabase().getFields()); - availableProperties.add(PROPERTY_IP); for (String fieldName : propertyNames) { if (availableProperties.contains(fieldName) == false) { throw newConfigurationException( diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 6ddac0a5..9e51f23b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -11,7 +11,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade.BUNDLE_SIZE; import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; import java.io.File; @@ -20,9 +19,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -38,8 +35,6 @@ import org.mockito.ArgumentCaptor; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; -import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionType; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; @@ -47,12 +42,9 @@ import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.search.MultiSearchRequest; -import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.Randomness; import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesReference; @@ -229,7 +221,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { } } - public void testGetSingleGeoIpData() { + public void testGetGeoIpData_whenSingleIp_thenSucceed() { String indexName = GeospatialTestHelper.randomLowerCaseString(); String ip = randomIpAddress(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { @@ -261,80 +253,35 @@ public void testGetSingleGeoIpData() { assertEquals("seattle", captor.getValue().get("city")); } - public void testGetGeoIpData_whenAllDataIsGathered_thenNoMoreSearch() { + public void testGetGeoIpData_whenMultiIps_thenSucceed() { String indexName = GeospatialTestHelper.randomLowerCaseString(); - String ip1 = randomIpAddress(); - String ip2 = randomIpAddress(); - Iterator ipIterator = Arrays.asList(ip1, ip2).iterator(); - Map> geoData = new HashMap<>(); - geoData.put(ip1, Map.of("city", "Seattle")); - geoData.put(ip2, Map.of("city", "Hawaii")); - ActionListener>> actionListener = mock(ActionListener.class); - - // Run - verifyingGeoIpDataFacade.getGeoIpData(indexName, ipIterator, geoData, actionListener); - - // Verify - verify(actionListener).onResponse(geoData); - } - - public void testGetGeoIpData_whenCalled_thenGetGeoIpData() { - String indexName = GeospatialTestHelper.randomLowerCaseString(); - int dataSize = Randomness.get().nextInt(10) + 1; - List ips = new ArrayList<>(); - for (int i = 0; i < dataSize; i++) { - ips.add(randomIpAddress()); - } - Map> geoData = new HashMap<>(); - ActionListener>> actionListener = mock(ActionListener.class); - - List cities = new ArrayList<>(); + String ip = randomIpAddress(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { - assert actionRequest instanceof MultiSearchRequest; - MultiSearchRequest request = (MultiSearchRequest) actionRequest; - for (SearchRequest searchRequest : request.requests()) { - assertEquals("_local", searchRequest.preference()); - assertEquals(1, searchRequest.source().size()); - } + assert actionRequest instanceof SearchRequest; + SearchRequest request = (SearchRequest) actionRequest; + assertEquals("_local", request.preference()); + assertEquals(1, request.source().size()); + assertEquals(QueryBuilders.boolQuery().should(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)), request.source().query()); - MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.requests().size()]; - for (int i = 0; i < request.requests().size(); i++) { - String city = GeospatialTestHelper.randomLowerCaseString(); - cities.add(city); - String data = String.format( - Locale.ROOT, - "{\"%s\":\"1.0.0.1/16\",\"%s\":{\"city\":\"%s\"}}", - IP_RANGE_FIELD_NAME, - DATA_FIELD_NAME, - city - ); - SearchHit searchHit = new SearchHit(1); - searchHit.sourceRef(BytesReference.fromByteBuffer(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)))); - SearchHit[] searchHitArray = { searchHit }; - SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); - SearchResponse searchResponse = mock(SearchResponse.class); - when(searchResponse.getHits()).thenReturn(searchHits); - MultiSearchResponse.Item item = mock(MultiSearchResponse.Item.class); - when(item.isFailure()).thenReturn(false); - when(item.getResponse()).thenReturn(searchResponse); - items[i] = item; - } - MultiSearchResponse response = mock(MultiSearchResponse.class); - when(response.getResponses()).thenReturn(items); + String data = String.format( + Locale.ROOT, + "{\"%s\":\"1.0.0.1/16\",\"%s\":{\"city\":\"seattle\"}}", + IP_RANGE_FIELD_NAME, + DATA_FIELD_NAME + ); + SearchHit searchHit = new SearchHit(1); + searchHit.sourceRef(BytesReference.fromByteBuffer(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)))); + SearchHit[] searchHitArray = { searchHit }; + SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); return response; }); - - // Run - verifyingGeoIpDataFacade.getGeoIpData(indexName, ips.iterator(), geoData, actionListener); - - // Verify - verify(verifyingClient, times((dataSize + BUNDLE_SIZE - 1) / BUNDLE_SIZE)).execute( - any(ActionType.class), - any(ActionRequest.class), - any(ActionListener.class) - ); - for (int i = 0; i < dataSize; i++) { - assertEquals(cities.get(i), geoData.get(ips.get(i)).get("city")); - } + ActionListener>> listener = mock(ActionListener.class); + verifyingGeoIpDataFacade.getGeoIpData(indexName, Arrays.asList(ip), listener); + ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class); + verify(listener).onResponse(captor.capture()); + assertEquals("seattle", captor.getValue().get(0).get("city")); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index bf447a5a..aeface41 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -86,7 +86,7 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { Ip2GeoProcessor.CONFIG_TARGET_FIELD, targetField, Ip2GeoProcessor.CONFIG_PROPERTIES, - Arrays.asList(IP, CITY) + Arrays.asList(CITY) ); // Create ip2geo processor @@ -104,10 +104,10 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { // Verify data added to document List> sources = convertToListOfSources(response, targetField); - sources.stream().forEach(source -> { - assertFalse(source.containsKey(COUNTRY)); - assertEquals(sampleData.get(source.get(IP)).get(CITY), source.get(CITY)); - }); + sources.stream().allMatch(source -> source.size() == 1); + List cities = sources.stream().map(value -> value.get(CITY)).collect(Collectors.toList()); + List expectedCities = sampleData.values().stream().map(value -> value.get(CITY)).collect(Collectors.toList()); + assertEquals(expectedCities, cities); // Delete datasource fails when there is a process using it ResponseException deleteException = expectThrows(ResponseException.class, () -> deleteDatasource(datasourceName)); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 2a7ce296..c3dbf765 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -6,18 +6,17 @@ package org.opensearch.geospatial.ip2geo.processor; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; @@ -71,9 +70,9 @@ public void testCreateWithInvalidDatasourceState() { assertTrue(exception.getDetailedMessage().contains("available state")); } - public void testCreateWithInvalidProperties() { + public void testCreateIp2GeoProcessor_whenInvalidProperties_thenException() { Map config = new HashMap<>(); - config.put("properties", Arrays.asList("ip", "invalid_property")); + config.put("properties", Arrays.asList(SUPPORTED_FIELDS.get(0), "invalid_property")); OpenSearchException exception = expectThrows( OpenSearchException.class, () -> createProcessor(GeospatialTestHelper.randomLowerCaseString(), config) @@ -94,25 +93,33 @@ public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { processor.execute(document, handler); } - public void testExecuteWithNoIp() throws Exception { + public void testExecute_whenNoIp_thenException() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Map config = new HashMap<>(); Ip2GeoProcessor processor = createProcessor(datasourceName, config); IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); - BiConsumer handler = (doc, e) -> {}; - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(document, handler)); - assertTrue(exception.getMessage().contains("not present")); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.execute(document, handler); + + // Verify + verify(handler).accept(isNull(), any(IllegalArgumentException.class)); } - public void testExecuteWithNonStringValue() throws Exception { + public void testExecute_whenNonStringValue_thenException() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); Map source = new HashMap<>(); source.put("ip", Randomness.get().nextInt()); IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = (doc, e) -> {}; - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(document, handler)); - assertTrue(exception.getMessage().contains("string")); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.execute(document, handler); + + // Verify + verify(handler).accept(isNull(), any(IllegalArgumentException.class)); } public void testExecuteWithNullDatasource() throws Exception { @@ -133,33 +140,6 @@ public void testExecuteWithExpiredDatasource() throws Exception { getActionListener(Collections.emptyMap(), handler).onResponse(datasource); } - public void testExecute() throws Exception { - Map ip2geoData = new HashMap<>(); - for (String field : SUPPORTED_FIELDS) { - ip2geoData.put(field, GeospatialTestHelper.randomLowerCaseString()); - } - - Datasource datasource = mock(Datasource.class); - when(datasource.isExpired()).thenReturn(false); - when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); - BiConsumer handler = (doc, e) -> { - assertEquals( - ip2geoData.get(SUPPORTED_FIELDS.get(0)), - doc.getFieldValue(DEFAULT_TARGET_FIELD + "." + SUPPORTED_FIELDS.get(0), String.class) - ); - for (int i = 1; i < SUPPORTED_FIELDS.size(); i++) { - assertNull(doc.getFieldValue(DEFAULT_TARGET_FIELD + "." + SUPPORTED_FIELDS.get(i), String.class, true)); - } - assertNull(e); - }; - Map config = Map.of("properties", Arrays.asList(SUPPORTED_FIELDS.get(0))); - getActionListener(config, handler).onResponse(datasource); - - ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(geoIpDataFacade).getGeoIpData(anyString(), anyString(), captor.capture()); - captor.getValue().onResponse(ip2geoData); - } - private ActionListener getActionListener( final Map config, final BiConsumer handler @@ -178,107 +158,155 @@ private ActionListener getActionListener( return captor.getValue(); } - public void testExecuteNotImplemented() throws Exception { + @SneakyThrows + public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - IngestDocument document = new IngestDocument(Collections.emptyMap(), Collections.emptyMap()); - Exception e = expectThrows(IllegalStateException.class, () -> processor.execute(document)); - assertTrue(e.getMessage().contains("Not implemented")); - } + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); - public void testGenerateDataToAppendWithNoData() throws Exception { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - List ips = new ArrayList<>(); - Map> data = new HashMap<>(); - for (int i = 0; i < 3; i++) { - String ip = randomIpAddress(); - ips.add(ip); - data.put(ip, Collections.emptyMap()); - } - IngestDocument document = new IngestDocument(new HashMap<>(), new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); - processor.listenerToAppendDataToDocument(data, ips, document, handler).onResponse(data); - verify(handler).accept(document, null); - Exception e = expectThrows(IllegalArgumentException.class, () -> document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class)); - assertTrue(e.getMessage().contains("not present")); + processor.executeInternal(document, handler, ip); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); + Datasource datasource = mock(Datasource.class); + when(datasource.isExpired()).thenReturn(false); + when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + + captor.getValue().onResponse(datasource); + verify(geoIpDataFacade).getGeoIpData(anyString(), anyString(), any(ActionListener.class)); } - public void testExecuteInternalNonStringIp() throws Exception { + @SneakyThrows + public void testGetSingleGeoIpDataListener_whenNoPropertySet_thenAddAllProperties() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - List ips = Arrays.asList(randomIpAddress(), 1); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - Exception e = expectThrows(IllegalArgumentException.class, () -> processor.executeInternal(document, handler, ips)); - assertTrue(e.getMessage().contains("should only contain strings")); + + Map geoIpData = Map.of("city", "Seattle", "country", "USA"); + // Run + processor.getSingleGeoIpDataListener(document, handler).onResponse(geoIpData); + + // Verify + assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".city", String.class)); + assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD + ".country", String.class)); + verify(handler).accept(document, null); } @SneakyThrows - public void testExecuteInternal_whenCalled_thenGetDatasourceIsCalled() { + public void testGetSingleGeoIpDataListener_whenPropertySet_thenAddOnlyTheProperties() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city"))); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - processor.executeInternal(document, handler, ips); - ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); - Datasource datasource = mock(Datasource.class); - when(datasource.isExpired()).thenReturn(false); - when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); - captor.getValue().onResponse(datasource); - verify(geoIpDataFacade).getGeoIpData(anyString(), any(Iterator.class), anyMap(), any(ActionListener.class)); + Map geoIpData = Map.of("city", "Seattle", "country", "USA"); + // Run + processor.getSingleGeoIpDataListener(document, handler).onResponse(geoIpData); + + // Verify + assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".city", String.class)); + assertFalse(document.hasField(DEFAULT_TARGET_FIELD + ".country")); + verify(handler).accept(document, null); } @SneakyThrows - public void testHandleSingleIp_whenEmptyGeoData_thenTargetFieldShouldNotSet() { + public void testGetMultiGeoIpDataListener_whenNoPropertySet_thenAddAllProperties() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); - Map ipToGeoData = Collections.emptyMap(); IngestDocument document = new IngestDocument(source, new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); + Map geoIpData = Map.of("city", "Seattle", "country", "USA"); // Run - processor.handleSingleIp(ip, ipToGeoData, document, handler); + processor.getMultiGeoIpDataListener(document, handler).onResponse(Arrays.asList(geoIpData)); // Verify - assertFalse(document.hasField(DEFAULT_TARGET_FIELD)); + assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); + assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.city", String.class)); + assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.country", String.class)); verify(handler).accept(document, null); } @SneakyThrows - public void testHandleSingleIp_whenNoValueForGivenProperty_thenDoNotAdd() { + public void testGetMultiGeoIpDataListener_whenPropertySet_thenAddOnlyTheProperties() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city", "country"))); + Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city"))); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); - Map ipToGeoData = Map.of("country", "USA"); IngestDocument document = new IngestDocument(source, new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); + Map geoIpData = Map.of("city", "Seattle", "country", "USA"); // Run - processor.handleSingleIp(ip, ipToGeoData, document, handler); + processor.getMultiGeoIpDataListener(document, handler).onResponse(Arrays.asList(geoIpData)); // Verify - assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get("country")); - assertNull(document.getFieldValue(DEFAULT_TARGET_FIELD, Map.class).get("city")); + assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); + assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.city", String.class)); + assertFalse(document.hasField(DEFAULT_TARGET_FIELD + ".0.country")); verify(handler).accept(document, null); } + public void testExecuteNotImplemented() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + IngestDocument document = new IngestDocument(Collections.emptyMap(), Collections.emptyMap()); + Exception e = expectThrows(IllegalStateException.class, () -> processor.execute(document)); + assertTrue(e.getMessage().contains("Not implemented")); + } + + public void testExecuteInternalNonStringIp() throws Exception { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + List ips = Arrays.asList(randomIpAddress(), 1); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + BiConsumer handler = mock(BiConsumer.class); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.executeInternal(document, handler, ips)); + assertTrue(e.getMessage().contains("should only contain strings")); + } + + @SneakyThrows + public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + BiConsumer handler = mock(BiConsumer.class); + processor.executeInternal(document, handler, ips); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); + Datasource datasource = mock(Datasource.class); + when(datasource.isExpired()).thenReturn(false); + when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + + captor.getValue().onResponse(datasource); + verify(geoIpDataFacade).getGeoIpData(anyString(), anyList(), any(ActionListener.class)); + } + private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { Datasource datasource = new Datasource(); datasource.setName(datasourceName); From c43eae149dcfa0a98141962837712b87ad38c0cb Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 16 Jun 2023 16:23:31 -0700 Subject: [PATCH 51/61] Run update/delete request in a new thread (#337) This is not to block transport thread Signed-off-by: Heemin Kim --- .../DeleteDatasourceTransportAction.java | 20 +++++++++--- .../UpdateDatasourceTransportAction.java | 31 +++++++++++++------ .../DeleteDatasourceTransportActionTests.java | 3 +- .../UpdateDatasourceTransportActionTests.java | 3 +- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index a4706834..f9a13f9e 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -26,6 +26,7 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.ingest.IngestService; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; /** @@ -39,6 +40,7 @@ public class DeleteDatasourceTransportAction extends HandledTransportAction { + try { + deleteDatasource(request.getName()); + lockService.releaseLock(lock); + listener.onResponse(new AcknowledgedResponse(true)); + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + } + }); } catch (Exception e) { lockService.releaseLock(lock); listener.onFailure(e); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index ffdb349d..bc247b70 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -31,6 +31,7 @@ import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; /** @@ -42,6 +43,7 @@ public class UpdateDatasourceTransportAction extends HandledTransportAction { + try { + Datasource datasource = datasourceFacade.getDatasource(request.getName()); + if (datasource == null) { + throw new ResourceNotFoundException("no such datasource exist"); + } + validate(request, datasource); + updateIfChanged(request, datasource); + lockService.releaseLock(lock); + listener.onResponse(new AcknowledgedResponse(true)); + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + } + }); } catch (Exception e) { lockService.releaseLock(lock); listener.onFailure(e); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java index fa2afef8..ce5c91d4 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -48,7 +48,8 @@ public void init() { ingestService, datasourceFacade, geoIpDataFacade, - ip2GeoProcessorFacade + ip2GeoProcessorFacade, + threadPool ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 3df610e8..32272f62 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -45,7 +45,8 @@ public void init() { actionFilters, ip2GeoLockService, datasourceFacade, - datasourceUpdateService + datasourceUpdateService, + threadPool ); } From b505ca3e2d02337f46f4ab9ff56f92b32ff4c206 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 16 Jun 2023 16:23:41 -0700 Subject: [PATCH 52/61] Remove IP2Geo processor validation (#336) Cannot query index to get data to validate IP2Geo processor. Will add validation when we decide to store some of data in cluster state metadata. Signed-off-by: Heemin Kim --- .../ip2geo/processor/Ip2GeoProcessor.java | 72 +++++-------------- .../ip2geo/processor/Ip2GeoProcessorIT.java | 16 ----- .../processor/Ip2GeoProcessorTests.java | 40 ++--------- 3 files changed, 23 insertions(+), 105 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 5c5f3280..f63630a5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -4,8 +4,6 @@ */ package org.opensearch.geospatial.ip2geo.processor; -import static org.opensearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME; -import static org.opensearch.ingest.ConfigurationUtils.newConfigurationException; import static org.opensearch.ingest.ConfigurationUtils.readBooleanProperty; import static org.opensearch.ingest.ConfigurationUtils.readOptionalList; import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; @@ -29,6 +27,7 @@ import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.IngestService; @@ -154,7 +153,12 @@ protected void executeInternal( @Override public void onResponse(final Datasource datasource) { if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource does not exist")); + handler.accept(null, new IllegalStateException("datasource is not available")); + return; + } + + if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { + handler.accept(null, new IllegalStateException("datasource is not in an available state")); return; } @@ -174,6 +178,10 @@ public void onResponse(final Datasource datasource) { @Override public void onFailure(final Exception e) { + if (e instanceof IndexNotFoundException) { + handler.accept(null, new IllegalStateException("datasource is not available")); + return; + } handler.accept(null, e); } }); @@ -241,8 +249,8 @@ protected void executeInternal( datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { @Override public void onResponse(final Datasource datasource) { - if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource does not exist")); + if (datasource == null || DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { + handler.accept(null, new IllegalStateException("datasource is not available")); return; } @@ -262,6 +270,10 @@ public void onResponse(final Datasource datasource) { @Override public void onFailure(final Exception e) { + if (e instanceof IndexNotFoundException) { + handler.accept(null, new IllegalStateException("datasource is not available")); + return; + } handler.accept(null, e); } }); @@ -319,15 +331,8 @@ public Factory(final IngestService ingestService, final DatasourceFacade datasou } /** - * When a user create a processor, this method is called twice. Once to validate the new processor and another - * to apply cluster state change after the processor is added. - * - * The second call is made by ClusterApplierService. Therefore, we cannot access cluster state in the call. - * That means, we cannot even query an index inside the call. - * - * Because the processor is validated in the first call, we skip the validation in the second call. - * - * @see org.opensearch.cluster.service.ClusterApplierService#state() + * Within this method, blocking request cannot be called because this method is executed in a transport thread. + * This means, validation using data in an index won't work. */ @Override public Ip2GeoProcessor create( @@ -342,11 +347,6 @@ public Ip2GeoProcessor create( List propertyNames = readOptionalList(TYPE, processorTag, config, CONFIG_PROPERTIES); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, CONFIG_IGNORE_MISSING, false); - // Skip validation for the call by cluster applier service - if (Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME) == false) { - validate(processorTag, datasourceName, propertyNames); - } - return new Ip2GeoProcessor( processorTag, description, @@ -360,39 +360,5 @@ public Ip2GeoProcessor create( geoIpDataFacade ); } - - private void validate(final String processorTag, final String datasourceName, final List propertyNames) throws IOException { - Datasource datasource = datasourceFacade.getDatasource(datasourceName); - - if (datasource == null) { - throw newConfigurationException(TYPE, processorTag, "datasource", "datasource [" + datasourceName + "] doesn't exist"); - } - - if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { - throw newConfigurationException( - TYPE, - processorTag, - "datasource", - "datasource [" + datasourceName + "] is not in an available state" - ); - } - - if (propertyNames == null) { - return; - } - - // Validate properties are valid. If not add all available properties. - final Set availableProperties = new HashSet<>(datasource.getDatabase().getFields()); - for (String fieldName : propertyNames) { - if (availableProperties.contains(fieldName) == false) { - throw newConfigurationException( - TYPE, - processorTag, - "properties", - "property [" + fieldName + "] is not available in the datasource [" + datasourceName + "]" - ); - } - } - } } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index aeface41..a99570ce 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.time.Duration; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -35,21 +34,6 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase { private static final String IP = "ip"; private static final String SOURCE = "_source"; - @SneakyThrows - public void testCreateIp2GeoProcessor_whenNoSuchDatasourceExist_thenFails() { - String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); - - // Run - ResponseException exception = expectThrows( - ResponseException.class, - () -> createIp2GeoProcessorPipeline(pipelineName, Collections.emptyMap()) - ); - - // Verify - assertTrue(exception.getMessage().contains("doesn't exist")); - assertEquals(RestStatus.BAD_REQUEST.getStatus(), exception.getResponse().getStatusLine().getStatusCode()); - } - @SneakyThrows public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { Ip2GeoDataServer.start(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index c3dbf765..fcf3dcb3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -25,7 +25,6 @@ import org.junit.Before; import org.mockito.ArgumentCaptor; -import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.common.Randomness; import org.opensearch.geospatial.GeospatialTestHelper; @@ -46,40 +45,6 @@ public void init() { factory = new Ip2GeoProcessor.Factory(ingestService, datasourceFacade, geoIpDataFacade); } - public void testCreateWithNoDatasource() { - Map config = new HashMap<>(); - config.put("field", "ip"); - config.put(CONFIG_DATASOURCE_KEY, "no_datasource"); - OpenSearchException exception = expectThrows( - OpenSearchException.class, - () -> factory.create( - Collections.emptyMap(), - GeospatialTestHelper.randomLowerCaseString(), - GeospatialTestHelper.randomLowerCaseString(), - config - ) - ); - assertTrue(exception.getDetailedMessage().contains("doesn't exist")); - } - - public void testCreateWithInvalidDatasourceState() { - Datasource datasource = new Datasource(); - datasource.setName(GeospatialTestHelper.randomLowerCaseString()); - datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); - OpenSearchException exception = expectThrows(OpenSearchException.class, () -> createProcessor(datasource, Collections.emptyMap())); - assertTrue(exception.getDetailedMessage().contains("available state")); - } - - public void testCreateIp2GeoProcessor_whenInvalidProperties_thenException() { - Map config = new HashMap<>(); - config.put("properties", Arrays.asList(SUPPORTED_FIELDS.get(0), "invalid_property")); - OpenSearchException exception = expectThrows( - OpenSearchException.class, - () -> createProcessor(GeospatialTestHelper.randomLowerCaseString(), config) - ); - assertTrue(exception.getDetailedMessage().contains("property")); - } - public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Map config = new HashMap<>(); @@ -125,13 +90,14 @@ public void testExecute_whenNonStringValue_thenException() throws Exception { public void testExecuteWithNullDatasource() throws Exception { BiConsumer handler = (doc, e) -> { assertNull(doc); - assertTrue(e.getMessage().contains("datasource does not exist")); + assertTrue(e.getMessage().contains("datasource is not available")); }; getActionListener(Collections.emptyMap(), handler).onResponse(null); } public void testExecuteWithExpiredDatasource() throws Exception { Datasource datasource = mock(Datasource.class); + when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); when(datasource.isExpired()).thenReturn(true); BiConsumer handler = (doc, e) -> { assertEquals("ip2geo_data_expired", doc.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); @@ -174,6 +140,7 @@ public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); Datasource datasource = mock(Datasource.class); when(datasource.isExpired()).thenReturn(false); + when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); captor.getValue().onResponse(datasource); @@ -301,6 +268,7 @@ public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); Datasource datasource = mock(Datasource.class); when(datasource.isExpired()).thenReturn(false); + when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); captor.getValue().onResponse(datasource); From 8d8b9490f1f3974a9e2f9ce4343bbd9cc73aafb4 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 26 Jun 2023 13:02:51 -0700 Subject: [PATCH 53/61] Acquire lock sychronously (#339) By acquiring lock asychronously, the remaining part of the code is being run by transport thread which does not allow blocking code. We want only single update happen in a node using single thread. However, it cannot be acheived if I acquire lock asynchronously and pass the listener. Signed-off-by: Heemin Kim --- .../ip2geo/common/Ip2GeoLockService.java | 33 +++++++++ .../ip2geo/jobscheduler/DatasourceRunner.java | 30 +++++--- .../ip2geo/common/Ip2GeoLockServiceTests.java | 8 +++ .../jobscheduler/DatasourceRunnerTests.java | 71 +++++++++---------- 4 files changed, 92 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java index 6b3c83b7..7bd03316 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockService.java @@ -8,6 +8,7 @@ import static org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension.JOB_INDEX_NAME; import java.time.Instant; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -56,6 +57,38 @@ public void acquireLock(final String datasourceName, final Long lockDurationSeco lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, datasourceName, listener); } + /** + * Synchronous method of #acquireLock + * + * @param datasourceName datasourceName to acquire lock on + * @param lockDurationSeconds the lock duration in seconds + * @return lock model + */ + public Optional acquireLock(final String datasourceName, final Long lockDurationSeconds) { + AtomicReference lockReference = new AtomicReference(); + CountDownLatch countDownLatch = new CountDownLatch(1); + lockService.acquireLockWithId(JOB_INDEX_NAME, lockDurationSeconds, datasourceName, new ActionListener<>() { + @Override + public void onResponse(final LockModel lockModel) { + lockReference.set(lockModel); + countDownLatch.countDown(); + } + + @Override + public void onFailure(final Exception e) { + lockReference.set(null); + countDownLatch.countDown(); + } + }); + + try { + countDownLatch.await(clusterService.getClusterSettings().get(Ip2GeoSettings.TIMEOUT).getSeconds(), TimeUnit.SECONDS); + return Optional.ofNullable(lockReference.get()); + } catch (InterruptedException e) { + return Optional.empty(); + } + } + /** * Wrapper method of LockService#release * diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index 8d8938c9..b92a58a8 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -8,11 +8,11 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import lombok.extern.log4j.Log4j2; -import org.opensearch.action.ActionListener; import org.opensearch.cluster.service.ClusterService; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; @@ -20,6 +20,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @@ -108,16 +109,23 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC @VisibleForTesting protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParameter) { return () -> { - ip2GeoLockService.acquireLock(jobParameter.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { - if (lock == null) { - return; - } - try { - updateDatasource(jobParameter, ip2GeoLockService.getRenewLockRunnable(new AtomicReference<>(lock))); - } finally { - ip2GeoLockService.releaseLock(lock); - } - }, exception -> { log.error("Failed to acquire lock for job [{}]", jobParameter.getName(), exception); })); + Optional lockModel = ip2GeoLockService.acquireLock( + jobParameter.getName(), + Ip2GeoLockService.LOCK_DURATION_IN_SECONDS + ); + if (lockModel.isEmpty()) { + log.error("Failed to update. Another processor is holding a lock for datasource[{}]", jobParameter.getName()); + return; + } + + LockModel lock = lockModel.get(); + try { + updateDatasource(jobParameter, ip2GeoLockService.getRenewLockRunnable(new AtomicReference<>(lock))); + } catch (Exception e) { + log.error("Failed to update datasource[{}]", jobParameter.getName(), e); + } finally { + ip2GeoLockService.releaseLock(lock); + } }; } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java index d6bfcd13..adcfae7f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java @@ -38,6 +38,14 @@ public void testAcquireLock_whenValidInput_thenSucceed() { noOpsLockService.acquireLock(GeospatialTestHelper.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); } + public void testAcquireLock_whenCalled_thenNotBlocked() { + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertTrue(ip2GeoLockService.acquireLock(null, null).isEmpty()); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } + public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index d08dc19a..a5461f3a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -6,7 +6,6 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -16,16 +15,15 @@ import static org.mockito.internal.verification.VerificationModeFactory.times; import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; -import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Optional; import lombok.SneakyThrows; import org.junit.Before; import org.mockito.ArgumentCaptor; -import org.opensearch.action.ActionListener; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; @@ -44,11 +42,15 @@ public void init() { } public void testRunJob_whenInvalidClass_thenThrowException() { - JobExecutionContext jobExecutionContext = mock(JobExecutionContext.class); + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = randomLowerCaseString(); + String jobId = randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); expectThrows(IllegalStateException.class, () -> DatasourceRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); } + @SneakyThrows public void testRunJob_whenValidInput_thenSucceed() { JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); String jobIndexName = randomLowerCaseString(); @@ -56,59 +58,50 @@ public void testRunJob_whenValidInput_thenSucceed() { JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); Datasource datasource = randomDatasource(); + LockModel lockModel = randomLockModel(); + when(ip2GeoLockService.acquireLock(datasource.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + // Run DatasourceRunner.getJobRunnerInstance().runJob(datasource, jobExecutionContext); // Verify - verify(ip2GeoLockService).acquireLock( - eq(datasource.getName()), - eq(Ip2GeoLockService.LOCK_DURATION_IN_SECONDS), - any(ActionListener.class) - ); + verify(ip2GeoLockService).acquireLock(datasource.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS); + verify(datasourceFacade).getDatasource(datasource.getName()); + verify(ip2GeoLockService).releaseLock(lockModel); } @SneakyThrows - public void testUpdateDatasourceRunner_whenFailedToAcquireLock_thenError() { - validateDoExecute(null, null); - } + public void testUpdateDatasourceRunner_whenExceptionBeforeAcquiringLock_thenNoReleaseLock() { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + when(ip2GeoLockService.acquireLock(jobParameter.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS)).thenThrow( + new RuntimeException() + ); - @SneakyThrows - public void testUpdateDatasourceRunner_whenValidInput_thenSucceed() { - String jobIndexName = GeospatialTestHelper.randomLowerCaseString(); - String jobId = GeospatialTestHelper.randomLowerCaseString(); - LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); - validateDoExecute(lockModel, null); - } + // Run + expectThrows(Exception.class, () -> DatasourceRunner.getJobRunnerInstance().updateDatasourceRunner(jobParameter).run()); - @SneakyThrows - public void testUpdateDatasourceRunner_whenException_thenError() { - validateDoExecute(null, new RuntimeException()); + // Verify + verify(ip2GeoLockService, never()).releaseLock(any()); } - private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { + @SneakyThrows + public void testUpdateDatasourceRunner_whenExceptionAfterAcquiringLock_thenReleaseLock() { ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); when(jobParameter.getName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + LockModel lockModel = randomLockModel(); + when(ip2GeoLockService.acquireLock(jobParameter.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + when(datasourceFacade.getDatasource(jobParameter.getName())).thenThrow(new RuntimeException()); // Run DatasourceRunner.getJobRunnerInstance().updateDatasourceRunner(jobParameter).run(); // Verify - ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(ip2GeoLockService).acquireLock(eq(jobParameter.getName()), anyLong(), captor.capture()); - - if (exception == null) { - // Run - captor.getValue().onResponse(lockModel); - - // Verify - verify(ip2GeoLockService, lockModel == null ? never() : times(1)).releaseLock(eq(lockModel)); - } else { - // Run - captor.getValue().onFailure(exception); - - // Verify - verify(ip2GeoLockService, never()).releaseLock(eq(lockModel)); - } + verify(ip2GeoLockService).releaseLock(any()); } @SneakyThrows From b657fff0e1b4b76d1152f9985590f5613625e91c Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 26 Jun 2023 13:03:06 -0700 Subject: [PATCH 54/61] Added a cache to store datasource metadata (#338) Signed-off-by: Heemin Kim --- .../geospatial/ip2geo/cache/Ip2GeoCache.java | 120 ++++++++++++ .../ip2geo/common/DatasourceFacade.java | 17 ++ .../ip2geo/processor/Ip2GeoProcessor.java | 125 +++++------- .../geospatial/plugin/GeospatialPlugin.java | 31 ++- .../geospatial/ip2geo/Ip2GeoTestCase.java | 6 +- .../ip2geo/cache/Ip2GeoCacheTests.java | 181 ++++++++++++++++++ .../ip2geo/common/DatasourceFacadeTests.java | 27 ++- .../jobscheduler/DatasourceRunnerTests.java | 2 + .../processor/Ip2GeoProcessorTests.java | 100 +++++----- .../plugin/GeospatialPluginTests.java | 44 ++--- 10 files changed, 495 insertions(+), 158 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java b/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java new file mode 100644 index 00000000..a42b31de --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.cache; + +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import lombok.Getter; +import lombok.extern.log4j.Log4j2; + +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.engine.Engine; +import org.opensearch.index.shard.IndexingOperationListener; +import org.opensearch.index.shard.ShardId; + +@Log4j2 +public class Ip2GeoCache implements IndexingOperationListener { + private final DatasourceFacade datasourceFacade; + private Map data; + + public Ip2GeoCache(final DatasourceFacade datasourceFacade) { + this.datasourceFacade = datasourceFacade; + } + + public String getIndexName(final String datasourceName) { + return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getIndexName(); + } + + public boolean isExpired(final String datasourceName) { + return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getExpirationDate().isBefore(Instant.now()); + } + + public boolean has(final String datasourceName) { + return getData().containsKey(datasourceName); + } + + public DatasourceState getState(final String datasourceName) { + return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getState(); + } + + private Map getData() { + if (data != null) { + return data; + } + synchronized (this) { + if (data != null) { + return data; + } + Map tempData = new ConcurrentHashMap<>(); + datasourceFacade.getAllDatasources() + .stream() + .forEach(datasource -> tempData.put(datasource.getName(), new DatasourceMetadata(datasource))); + data = tempData; + return data; + } + } + + private void put(final Datasource datasource) { + DatasourceMetadata metadata = new DatasourceMetadata(datasource); + getData().put(datasource.getName(), metadata); + } + + private void remove(final String datasourceName) { + getData().remove(datasourceName); + } + + @Override + public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) { + if (Engine.Result.Type.FAILURE.equals(result.getResultType())) { + return; + } + + try { + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, index.source().utf8ToString()); + parser.nextToken(); + Datasource datasource = Datasource.PARSER.parse(parser, null); + put(datasource); + } catch (IOException e) { + log.error("IOException occurred updating datasource metadata for datasource {} ", index.id(), e); + } + } + + @Override + public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) { + if (result.getResultType().equals(Engine.Result.Type.FAILURE)) { + return; + } + remove(delete.id()); + } + + @Getter + private static class DatasourceMetadata { + private static DatasourceMetadata EMPTY_METADATA = new DatasourceMetadata(); + private String indexName; + private Instant expirationDate; + private DatasourceState state; + + private DatasourceMetadata() { + expirationDate = Instant.MIN; + } + + public DatasourceMetadata(final Datasource datasource) { + this.indexName = datasource.currentIndexName(); + this.expirationDate = datasource.expirationDay(); + this.state = datasource.getState(); + } + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java index d2f2908e..d0e13cd5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java @@ -297,6 +297,23 @@ public void getAllDatasources(final ActionListener> actionListe ); } + /** + * Get all datasources up to {@code MAX_SIZE} from an index {@code DatasourceExtension.JOB_INDEX_NAME} + */ + public List getAllDatasources() { + SearchResponse response = StashedThreadContext.run( + client, + () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(MAX_SIZE) + .execute() + .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) + ); + + List bytesReferences = toBytesReferences(response); + return bytesReferences.stream().map(bytesRef -> toDatasource(bytesRef)).collect(Collectors.toList()); + } + private ActionListener createGetDataSourceQueryActionLister( final Class response, final ActionListener> actionListener diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index f63630a5..f051ed1c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -17,17 +17,17 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.geospatial.annotation.VisibleForTesting; +import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; -import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.IngestService; @@ -57,6 +57,7 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final ClusterSettings clusterSettings; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; + private final Ip2GeoCache ip2GeoCache; /** * Ip2Geo processor type @@ -75,6 +76,7 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param clusterSettings the cluster settings * @param datasourceFacade the datasource facade * @param geoIpDataFacade the geoip data facade + * @param ip2GeoCache the cache */ public Ip2GeoProcessor( final String tag, @@ -86,7 +88,8 @@ public Ip2GeoProcessor( final boolean ignoreMissing, final ClusterSettings clusterSettings, final DatasourceFacade datasourceFacade, - final GeoIpDataFacade geoIpDataFacade + final GeoIpDataFacade geoIpDataFacade, + final Ip2GeoCache ip2GeoCache ) { super(tag, description); this.field = field; @@ -97,6 +100,7 @@ public Ip2GeoProcessor( this.clusterSettings = clusterSettings; this.datasourceFacade = datasourceFacade; this.geoIpDataFacade = geoIpDataFacade; + this.ip2GeoCache = ip2GeoCache; } /** @@ -149,42 +153,18 @@ protected void executeInternal( final BiConsumer handler, final String ip ) { - datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { - @Override - public void onResponse(final Datasource datasource) { - if (datasource == null) { - handler.accept(null, new IllegalStateException("datasource is not available")); - return; - } - - if (DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { - handler.accept(null, new IllegalStateException("datasource is not in an available state")); - return; - } - - String indexName = datasource.currentIndexName(); - if (indexName == null) { - ingestDocument.setFieldValue(targetField, DATA_EXPIRED); - handler.accept(ingestDocument, null); - return; - } - - try { - geoIpDataFacade.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); - } catch (Exception e) { - handler.accept(null, e); - } - } + validateDatasourceIsInAvailableState(datasourceName); + String indexName = ip2GeoCache.getIndexName(datasourceName); + if (ip2GeoCache.isExpired(datasourceName) || indexName == null) { + handleExpiredData(ingestDocument, handler); + return; + } - @Override - public void onFailure(final Exception e) { - if (e instanceof IndexNotFoundException) { - handler.accept(null, new IllegalStateException("datasource is not available")); - return; - } - handler.accept(null, e); - } - }); + try { + geoIpDataFacade.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); + } catch (Exception e) { + handler.accept(null, e); + } } @VisibleForTesting @@ -228,6 +208,21 @@ private List> filteredGeoData(final List return geoData.stream().map(this::filteredGeoData).collect(Collectors.toList()); } + private void validateDatasourceIsInAvailableState(final String datasourceName) { + if (ip2GeoCache.has(datasourceName) == false) { + throw new IllegalStateException("datasource does not exist"); + } + + if (DatasourceState.AVAILABLE.equals(ip2GeoCache.getState(datasourceName)) == false) { + throw new IllegalStateException("datasource is not in an available state"); + } + } + + private void handleExpiredData(final IngestDocument ingestDocument, final BiConsumer handler) { + ingestDocument.setFieldValue(targetField, DATA_EXPIRED); + handler.accept(ingestDocument, null); + } + /** * Handle multiple ips * @@ -246,37 +241,15 @@ protected void executeInternal( throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } } - datasourceFacade.getDatasource(datasourceName, new ActionListener<>() { - @Override - public void onResponse(final Datasource datasource) { - if (datasource == null || DatasourceState.AVAILABLE.equals(datasource.getState()) == false) { - handler.accept(null, new IllegalStateException("datasource is not available")); - return; - } - String indexName = datasource.currentIndexName(); - if (indexName == null) { - ingestDocument.setFieldValue(targetField, DATA_EXPIRED); - handler.accept(ingestDocument, null); - return; - } - - try { - geoIpDataFacade.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); - } catch (Exception e) { - handler.accept(null, e); - } - } + validateDatasourceIsInAvailableState(datasourceName); + String indexName = ip2GeoCache.getIndexName(datasourceName); + if (ip2GeoCache.isExpired(datasourceName) || indexName == null) { + handleExpiredData(ingestDocument, handler); + return; + } - @Override - public void onFailure(final Exception e) { - if (e instanceof IndexNotFoundException) { - handler.accept(null, new IllegalStateException("datasource is not available")); - return; - } - handler.accept(null, e); - } - }); + geoIpDataFacade.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); } @VisibleForTesting @@ -312,23 +285,12 @@ public String getType() { /** * Ip2Geo processor factory */ + @AllArgsConstructor public static final class Factory implements Processor.Factory { private final IngestService ingestService; private final DatasourceFacade datasourceFacade; private final GeoIpDataFacade geoIpDataFacade; - - /** - * Default constructor - * - * @param ingestService the ingest service - * @param datasourceFacade the datasource facade - * @param geoIpDataFacade the geoip data facade - */ - public Factory(final IngestService ingestService, final DatasourceFacade datasourceFacade, final GeoIpDataFacade geoIpDataFacade) { - this.ingestService = ingestService; - this.datasourceFacade = datasourceFacade; - this.geoIpDataFacade = geoIpDataFacade; - } + private final Ip2GeoCache ip2GeoCache; /** * Within this method, blocking request cannot be called because this method is executed in a transport thread. @@ -357,7 +319,8 @@ public Ip2GeoProcessor create( ignoreMissing, ingestService.getClusterService().getClusterSettings(), datasourceFacade, - geoIpDataFacade + geoIpDataFacade, + ip2GeoCache ); } } diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 5f780045..f94133c2 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -13,6 +13,8 @@ import java.util.Map; import java.util.function.Supplier; +import lombok.extern.log4j.Log4j2; + import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; import org.opensearch.client.Client; @@ -49,11 +51,13 @@ import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceAction; import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceTransportAction; +import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.listener.Ip2GeoListener; @@ -66,6 +70,7 @@ import org.opensearch.geospatial.stats.upload.UploadStats; import org.opensearch.geospatial.stats.upload.UploadStatsAction; import org.opensearch.geospatial.stats.upload.UploadStatsTransportAction; +import org.opensearch.index.IndexModule; import org.opensearch.index.mapper.Mapper; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.ingest.Processor; @@ -87,7 +92,11 @@ * Entry point for Geospatial features. It provides additional Processors, Actions * to interact with Cluster. */ +@Log4j2 public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlugin, MapperPlugin, SearchPlugin, SystemIndexPlugin { + private Ip2GeoCache ip2GeoCache; + private DatasourceFacade datasourceFacade; + private GeoIpDataFacade geoIpDataFacade; @Override public Collection getSystemIndexDescriptors(Settings settings) { @@ -96,19 +105,26 @@ public Collection getSystemIndexDescriptors(Settings sett @Override public Map getProcessors(Processor.Parameters parameters) { + this.datasourceFacade = new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService()); + this.geoIpDataFacade = new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client); + this.ip2GeoCache = new Ip2GeoCache(datasourceFacade); return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) .put( Ip2GeoProcessor.TYPE, - new Ip2GeoProcessor.Factory( - parameters.ingestService, - new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService()), - new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client) - ) + new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceFacade, geoIpDataFacade, ip2GeoCache) ) .immutableMap(); } + @Override + public void onIndexModule(IndexModule indexModule) { + if (DatasourceExtension.JOB_INDEX_NAME.equals(indexModule.getIndex().getName())) { + indexModule.addIndexOperationListener(ip2GeoCache); + log.info("Ip2GeoListener started listening to operations on index {}", DatasourceExtension.JOB_INDEX_NAME); + } + } + @Override public Collection> getGuiceServiceClasses() { return List.of(Ip2GeoListener.class); @@ -140,8 +156,6 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - GeoIpDataFacade geoIpDataFacade = new GeoIpDataFacade(clusterService, client); - DatasourceFacade datasourceFacade = new DatasourceFacade(client, clusterService); DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client); @@ -158,7 +172,8 @@ public Collection createComponents( datasourceFacade, ip2GeoExecutor, geoIpDataFacade, - ip2GeoLockService + ip2GeoLockService, + ip2GeoCache ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 01f23e63..5cf42fa6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -43,6 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; @@ -78,6 +79,8 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { @Mock protected GeoIpDataFacade geoIpDataFacade; @Mock + protected Ip2GeoCache ip2GeoCache; + @Mock protected ClusterState clusterState; @Mock protected Metadata metadata; @@ -262,7 +265,8 @@ protected Ip2GeoProcessor randomIp2GeoProcessor(String datasourceName) { true, clusterSettings, datasourceFacade, - geoIpDataFacade + geoIpDataFacade, + ip2GeoCache ); return ip2GeoProcessor; } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java new file mode 100644 index 00000000..01dc2fc5 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java @@ -0,0 +1,181 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.cache; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.Arrays; + +import lombok.SneakyThrows; + +import org.junit.Before; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.engine.Engine; +import org.opensearch.index.shard.ShardId; + +public class Ip2GeoCacheTests extends Ip2GeoTestCase { + private Ip2GeoCache ip2GeoCache; + + @Before + public void init() { + ip2GeoCache = new Ip2GeoCache(datasourceFacade); + } + + public void testGetIndexName_whenCalled_thenReturnIndexName() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + // Run + String indexName = ip2GeoCache.getIndexName(datasource.getName()); + + // Verify + assertEquals(datasource.currentIndexName(), indexName); + } + + public void testIsExpired_whenExpired_thenReturnTrue() { + Datasource datasource = randomDatasource(); + datasource.getUpdateStats().setLastSucceededAt(Instant.MIN); + datasource.getUpdateStats().setLastSkippedAt(null); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + // Run + boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); + + // Verify + assertTrue(isExpired); + } + + public void testIsExpired_whenNotExpired_thenReturnFalse() { + Datasource datasource = randomDatasource(); + datasource.getUpdateStats().setLastSucceededAt(Instant.now()); + datasource.getUpdateStats().setLastSkippedAt(null); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + // Run + boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); + + // Verify + assertFalse(isExpired); + } + + public void testHas_whenHasDatasource_thenReturnTrue() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + // Run + boolean hasDatasource = ip2GeoCache.has(datasource.getName()); + + // Verify + assertTrue(hasDatasource); + } + + public void testHas_whenNoDatasource_thenReturnFalse() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + // Run + boolean hasDatasource = ip2GeoCache.has(datasourceName); + + // Verify + assertFalse(hasDatasource); + } + + public void testGetState_whenCalled_thenReturnState() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + // Run + DatasourceState state = ip2GeoCache.getState(datasource.getName()); + + // Verify + assertEquals(datasource.getState(), state); + } + + @SneakyThrows + public void testPostIndex_whenFailed_thenNoUpdate() { + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList()); + Datasource datasource = randomDatasource(); + + ShardId shardId = mock(ShardId.class); + Engine.Index index = mock(Engine.Index.class); + BytesReference bytesReference = BytesReference.bytes(datasource.toXContent(XContentFactory.jsonBuilder(), null)); + when(index.source()).thenReturn(bytesReference); + Engine.IndexResult result = mock(Engine.IndexResult.class); + when(result.getResultType()).thenReturn(Engine.Result.Type.FAILURE); + + // Run + ip2GeoCache.postIndex(shardId, index, result); + + // Verify + assertFalse(ip2GeoCache.has(datasource.getName())); + assertTrue(ip2GeoCache.isExpired(datasource.getName())); + assertNull(ip2GeoCache.getIndexName(datasource.getName())); + assertNull(ip2GeoCache.getState(datasource.getName())); + } + + @SneakyThrows + public void testPostIndex_whenSucceed_thenUpdate() { + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList()); + Datasource datasource = randomDatasource(); + + ShardId shardId = mock(ShardId.class); + Engine.Index index = mock(Engine.Index.class); + BytesReference bytesReference = BytesReference.bytes(datasource.toXContent(XContentFactory.jsonBuilder(), null)); + when(index.source()).thenReturn(bytesReference); + Engine.IndexResult result = mock(Engine.IndexResult.class); + when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); + + // Run + ip2GeoCache.postIndex(shardId, index, result); + + // Verify + assertTrue(ip2GeoCache.has(datasource.getName())); + assertFalse(ip2GeoCache.isExpired(datasource.getName())); + assertEquals(datasource.currentIndexName(), ip2GeoCache.getIndexName(datasource.getName())); + assertEquals(datasource.getState(), ip2GeoCache.getState(datasource.getName())); + } + + public void testPostDelete_whenFailed_thenNoUpdate() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + ShardId shardId = mock(ShardId.class); + Engine.Delete index = mock(Engine.Delete.class); + Engine.DeleteResult result = mock(Engine.DeleteResult.class); + when(result.getResultType()).thenReturn(Engine.Result.Type.FAILURE); + + // Run + ip2GeoCache.postDelete(shardId, index, result); + + // Verify + assertTrue(ip2GeoCache.has(datasource.getName())); + } + + public void testPostDelete_whenSucceed_thenUpdate() { + Datasource datasource = randomDatasource(); + when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + + ShardId shardId = mock(ShardId.class); + Engine.Delete index = mock(Engine.Delete.class); + when(index.id()).thenReturn(datasource.getName()); + Engine.DeleteResult result = mock(Engine.DeleteResult.class); + when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); + + // Run + ip2GeoCache.postDelete(shardId, index, result); + + // Verify + assertFalse(ip2GeoCache.has(datasource.getName())); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java index 43d8727e..748e6ab4 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java @@ -282,7 +282,7 @@ public void testGetDatasources_whenValidInput_thenSucceed() { } - public void testGetAllDatasources_whenValidInput_thenSucceed() { + public void testGetAllDatasources_whenAsynchronous_thenSucceed() { List datasources = Arrays.asList(randomDatasource(), randomDatasource()); ActionListener> listener = mock(ActionListener.class); SearchHits searchHits = getMockedSearchHits(datasources); @@ -310,6 +310,31 @@ public void testGetAllDatasources_whenValidInput_thenSucceed() { assertEquals(datasources, captor.getValue()); } + public void testGetAllDatasources_whenSynchronous_thenSucceed() { + List datasources = Arrays.asList(randomDatasource(), randomDatasource()); + SearchHits searchHits = getMockedSearchHits(datasources); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof SearchRequest); + SearchRequest request = (SearchRequest) actionRequest; + assertEquals(1, request.indices().length); + assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.indices()[0]); + assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); + assertEquals(1000, request.source().size()); + + SearchResponse response = mock(SearchResponse.class); + when(response.getHits()).thenReturn(searchHits); + return response; + }); + + // Run + datasourceFacade.getAllDatasources(); + + // Verify + assertEquals(datasources, datasourceFacade.getAllDatasources()); + } + public void testUpdateDatasource_whenValidInput_thenUpdate() { List datasources = Arrays.asList(randomDatasource(), randomDatasource()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index a5461f3a..cc7deb03 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -47,6 +47,8 @@ public void testRunJob_whenInvalidClass_thenThrowException() { String jobId = randomLowerCaseString(); JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + + // Run expectThrows(IllegalStateException.class, () -> DatasourceRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index fcf3dcb3..656375ca 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -8,7 +8,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -42,7 +41,7 @@ public class Ip2GeoProcessorTests extends Ip2GeoTestCase { @Before public void init() { - factory = new Ip2GeoProcessor.Factory(ingestService, datasourceFacade, geoIpDataFacade); + factory = new Ip2GeoProcessor.Factory(ingestService, datasourceFacade, geoIpDataFacade, ip2GeoCache); } public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { @@ -87,41 +86,52 @@ public void testExecute_whenNonStringValue_thenException() throws Exception { verify(handler).accept(isNull(), any(IllegalArgumentException.class)); } - public void testExecuteWithNullDatasource() throws Exception { - BiConsumer handler = (doc, e) -> { - assertNull(doc); - assertTrue(e.getMessage().contains("datasource is not available")); - }; - getActionListener(Collections.emptyMap(), handler).onResponse(null); - } + @SneakyThrows + public void testExecute_whenNoDatasource_thenNotExistError() { + String datasourceName = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - public void testExecuteWithExpiredDatasource() throws Exception { - Datasource datasource = mock(Datasource.class); - when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); - when(datasource.isExpired()).thenReturn(true); - BiConsumer handler = (doc, e) -> { - assertEquals("ip2geo_data_expired", doc.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); - assertNull(e); - }; - getActionListener(Collections.emptyMap(), handler).onResponse(datasource); + Map source = new HashMap<>(); + String ip = randomIpAddress(); + source.put("ip", ip); + IngestDocument document = new IngestDocument(source, new HashMap<>()); + + when(ip2GeoCache.has(datasourceName)).thenReturn(false); + BiConsumer handler = mock(BiConsumer.class); + + // Run + processor.execute(document, handler); + + // Verify + ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); + verify(handler).accept(isNull(), captor.capture()); + captor.getValue().getMessage().contains("not exist"); } - private ActionListener getActionListener( - final Map config, - final BiConsumer handler - ) throws Exception { + @SneakyThrows + public void testExecute_whenExpired_thenExpiredMsg() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, config); + Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); + String index = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(index); + when(ip2GeoCache.has(datasourceName)).thenReturn(true); + when(ip2GeoCache.isExpired(datasourceName)).thenReturn(true); + when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + + BiConsumer handler = mock(BiConsumer.class); + + // Run processor.execute(document, handler); - ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); - return captor.getValue(); + + // Verify + verify(handler).accept(document, null); + assertEquals("ip2geo_data_expired", document.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); } @SneakyThrows @@ -132,18 +142,18 @@ public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { String ip = randomIpAddress(); source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - processor.executeInternal(document, handler, ip); - ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); - Datasource datasource = mock(Datasource.class); - when(datasource.isExpired()).thenReturn(false); - when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); - when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + String indexName = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCache.has(datasourceName)).thenReturn(true); + when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCache.isExpired(datasourceName)).thenReturn(false); + + // Run + processor.executeInternal(document, handler, ip); - captor.getValue().onResponse(datasource); + // Verify verify(geoIpDataFacade).getGeoIpData(anyString(), anyString(), any(ActionListener.class)); } @@ -255,23 +265,23 @@ public void testExecuteInternalNonStringIp() throws Exception { public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); Map source = new HashMap<>(); String ip = randomIpAddress(); source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); - processor.executeInternal(document, handler, ips); + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); - ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getDatasource(eq(datasourceName), captor.capture()); - Datasource datasource = mock(Datasource.class); - when(datasource.isExpired()).thenReturn(false); - when(datasource.getState()).thenReturn(DatasourceState.AVAILABLE); - when(datasource.currentIndexName()).thenReturn(GeospatialTestHelper.randomLowerCaseString()); + String indexName = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCache.has(datasourceName)).thenReturn(true); + when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCache.isExpired(datasourceName)).thenReturn(false); - captor.getValue().onResponse(datasource); + // Run + processor.executeInternal(document, handler, ips); + + // Verify verify(geoIpDataFacade).getGeoIpData(anyString(), anyList(), any(ActionListener.class)); } diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 430268fb..7e2e9e01 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -37,6 +37,7 @@ import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; +import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; @@ -79,7 +80,8 @@ public class GeospatialPluginTests extends OpenSearchTestCase { DatasourceFacade.class, Ip2GeoExecutor.class, GeoIpDataFacade.class, - Ip2GeoLockService.class + Ip2GeoLockService.class, + Ip2GeoCache.class ); @Mock @@ -107,6 +109,7 @@ public class GeospatialPluginTests extends OpenSearchTestCase { private NodeEnvironment nodeEnvironment; private Settings settings; private AutoCloseable openMocks; + private GeospatialPlugin plugin; @Before public void init() { @@ -117,6 +120,9 @@ public void init() { when(clusterService.getSettings()).thenReturn(settings); when(ingestService.getClusterService()).thenReturn(clusterService); nodeEnvironment = null; + plugin = new GeospatialPlugin(); + // Need to call getProcessors to initialize few instances in plugin class + plugin.getProcessors(getProcessorParameter()); } @After @@ -125,7 +131,6 @@ public void close() throws Exception { } public void testSystemIndexDescriptors() { - GeospatialPlugin plugin = new GeospatialPlugin(); Set registeredSystemIndexPatterns = new HashSet<>(); for (SystemIndexDescriptor descriptor : plugin.getSystemIndexDescriptors(Settings.EMPTY)) { registeredSystemIndexPatterns.add(descriptor.getIndexPattern()); @@ -135,12 +140,10 @@ public void testSystemIndexDescriptors() { } public void testExecutorBuilders() { - GeospatialPlugin plugin = new GeospatialPlugin(); assertEquals(1, plugin.getExecutorBuilders(Settings.EMPTY).size()); } public void testCreateComponents() { - GeospatialPlugin plugin = new GeospatialPlugin(); Set registeredComponents = new HashSet<>(); Collection components = plugin.createComponents( client, @@ -162,37 +165,21 @@ public void testCreateComponents() { } public void testGetGuiceServiceClasses() { - GeospatialPlugin plugin = new GeospatialPlugin(); Collection> classes = List.of(Ip2GeoListener.class); assertEquals(classes, plugin.getGuiceServiceClasses()); } public void testIsAnIngestPlugin() { - GeospatialPlugin plugin = new GeospatialPlugin(); assertTrue(plugin instanceof IngestPlugin); } public void testFeatureProcessorIsAdded() { - Processor.Parameters parameters = new Processor.Parameters( - mock(Environment.class), - mock(ScriptService.class), - null, - null, - null, - null, - ingestService, - client, - null - ); - - GeospatialPlugin plugin = new GeospatialPlugin(); - Map processors = plugin.getProcessors(parameters); + Map processors = plugin.getProcessors(getProcessorParameter()); assertTrue(processors.containsKey(FeatureProcessor.TYPE)); assertTrue(processors.get(FeatureProcessor.TYPE) instanceof FeatureProcessor.Factory); } public void testTotalRestHandlers() { - GeospatialPlugin plugin = new GeospatialPlugin(); assertEquals( SUPPORTED_REST_HANDLERS.size(), plugin.getRestHandlers(Settings.EMPTY, null, clusterSettings, null, null, null, null).size() @@ -200,8 +187,21 @@ public void testTotalRestHandlers() { } public void testUploadGeoJSONTransportIsAdded() { - GeospatialPlugin plugin = new GeospatialPlugin(); final List> actions = plugin.getActions(); assertEquals(1, actions.stream().filter(actionHandler -> actionHandler.getAction() instanceof UploadGeoJSONAction).count()); } + + private Processor.Parameters getProcessorParameter() { + return new Processor.Parameters( + mock(Environment.class), + mock(ScriptService.class), + null, + null, + null, + null, + ingestService, + client, + null + ); + } } From 20b3a853c5e0efce3e954658ee045890abfddbb6 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 26 Jun 2023 13:29:14 -0700 Subject: [PATCH 55/61] Changed class name and package (#341) Signed-off-by: Heemin Kim --- .../DeleteDatasourceTransportAction.java | 40 ++++++++--------- .../action/GetDatasourceTransportAction.java | 14 +++--- .../action/PutDatasourceTransportAction.java | 16 +++---- .../UpdateDatasourceTransportAction.java | 14 +++--- .../geospatial/ip2geo/cache/Ip2GeoCache.java | 10 ++--- .../DatasourceDao.java} | 11 ++--- .../GeoIpDataDao.java} | 12 ++--- .../Ip2GeoProcessorDao.java} | 9 ++-- .../ip2geo/jobscheduler/DatasourceRunner.java | 14 +++--- .../jobscheduler/DatasourceUpdateService.java | 36 +++++++-------- .../ip2geo/listener/Ip2GeoListener.java | 14 +++--- .../ip2geo/processor/Ip2GeoProcessor.java | 32 +++++++------- .../geospatial/plugin/GeospatialPlugin.java | 27 +++++------- .../geospatial/ip2geo/Ip2GeoTestCase.java | 16 +++---- .../DeleteDatasourceTransportActionTests.java | 42 +++++++++--------- .../GetDatasourceTransportActionTests.java | 6 +-- .../PutDatasourceTransportActionTests.java | 12 ++--- .../UpdateDatasourceTransportActionTests.java | 20 ++++----- .../ip2geo/cache/Ip2GeoCacheTests.java | 22 +++++----- .../DatasourceDaoTests.java} | 44 +++++++++---------- .../GeoIpDataDaoTests.java} | 35 ++++++++------- .../Ip2GeoProcessorDaoTests.java} | 14 +++--- .../jobscheduler/DatasourceRunnerTests.java | 22 +++++----- .../DatasourceUpdateServiceTests.java | 40 +++++++---------- .../ip2geo/listener/Ip2GeoListenerTests.java | 8 ++-- .../processor/Ip2GeoProcessorTests.java | 8 ++-- .../plugin/GeospatialPluginTests.java | 8 ++-- 27 files changed, 269 insertions(+), 277 deletions(-) rename src/main/java/org/opensearch/geospatial/ip2geo/{common/DatasourceFacade.java => dao/DatasourceDao.java} (97%) rename src/main/java/org/opensearch/geospatial/ip2geo/{common/GeoIpDataFacade.java => dao/GeoIpDataDao.java} (97%) rename src/main/java/org/opensearch/geospatial/ip2geo/{common/Ip2GeoProcessorFacade.java => dao/Ip2GeoProcessorDao.java} (85%) rename src/test/java/org/opensearch/geospatial/ip2geo/{common/DatasourceFacadeTests.java => dao/DatasourceDaoTests.java} (91%) rename src/test/java/org/opensearch/geospatial/ip2geo/{common/GeoIpDataFacadeTests.java => dao/GeoIpDataDaoTests.java} (91%) rename src/test/java/org/opensearch/geospatial/ip2geo/{common/Ip2GeoProcessorFacadeTests.java => dao/Ip2GeoProcessorDaoTests.java} (90%) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java index f9a13f9e..8046f6ff 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportAction.java @@ -18,11 +18,11 @@ import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.exceptions.ResourceInUseException; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoProcessorDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.ingest.IngestService; import org.opensearch.tasks.Task; @@ -37,9 +37,9 @@ public class DeleteDatasourceTransportAction extends HandledTransportAction { - private final DatasourceFacade datasourceFacade; + private final DatasourceDao datasourceDao; /** * Default constructor * @param transportService the transport service * @param actionFilters the action filters - * @param datasourceFacade the datasource facade + * @param datasourceDao the datasource facade */ @Inject public GetDatasourceTransportAction( final TransportService transportService, final ActionFilters actionFilters, - final DatasourceFacade datasourceFacade + final DatasourceDao datasourceDao ) { super(GetDatasourceAction.NAME, transportService, actionFilters, GetDatasourceRequest::new); - this.datasourceFacade = datasourceFacade; + this.datasourceDao = datasourceDao; } @Override protected void doExecute(final Task task, final GetDatasourceRequest request, final ActionListener listener) { if (shouldGetAllDatasource(request)) { // We don't expect too many data sources. Therefore, querying all data sources without pagination should be fine. - datasourceFacade.getAllDatasources(newActionListener(listener)); + datasourceDao.getAllDatasources(newActionListener(listener)); } else { - datasourceFacade.getDatasources(request.getNames(), newActionListener(listener)); + datasourceDao.getDatasources(request.getNames(), newActionListener(listener)); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java index 75b30f29..ed4d1ee7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportAction.java @@ -22,9 +22,9 @@ import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.exceptions.ConcurrentModificationException; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.index.engine.VersionConflictEngineException; @@ -39,7 +39,7 @@ @Log4j2 public class PutDatasourceTransportAction extends HandledTransportAction { private final ThreadPool threadPool; - private final DatasourceFacade datasourceFacade; + private final DatasourceDao datasourceDao; private final DatasourceUpdateService datasourceUpdateService; private final Ip2GeoLockService lockService; @@ -48,7 +48,7 @@ public class PutDatasourceTransportAction extends HandledTransportAction listener ) { StepListener createIndexStep = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(createIndexStep); + datasourceDao.createIndexIfNotExists(createIndexStep); createIndexStep.whenComplete(v -> { Datasource datasource = Datasource.Builder.build(request); - datasourceFacade.putDatasource(datasource, getIndexResponseListener(datasource, lock, listener)); + datasourceDao.putDatasource(datasource, getIndexResponseListener(datasource, lock, listener)); }, exception -> { lockService.releaseLock(lock); listener.onFailure(exception); @@ -165,7 +165,7 @@ private void markDatasourceAsCreateFailed(final Datasource datasource) { datasource.getUpdateStats().setLastFailedAt(Instant.now()); datasource.setState(DatasourceState.CREATE_FAILED); try { - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); } catch (Exception e) { log.error("Failed to mark datasource state as CREATE_FAILED for {}", datasource.getName(), e); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java index bc247b70..c832dc89 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportAction.java @@ -23,9 +23,9 @@ import org.opensearch.common.inject.Inject; import org.opensearch.geospatial.exceptions.ConcurrentModificationException; import org.opensearch.geospatial.exceptions.IncompatibleDatasourceException; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; @@ -41,7 +41,7 @@ public class UpdateDatasourceTransportAction extends HandledTransportAction { private static final long LOCK_DURATION_IN_SECONDS = 300l; private final Ip2GeoLockService lockService; - private final DatasourceFacade datasourceFacade; + private final DatasourceDao datasourceDao; private final DatasourceUpdateService datasourceUpdateService; private final ThreadPool threadPool; @@ -51,7 +51,7 @@ public class UpdateDatasourceTransportAction extends HandledTransportAction { try { - Datasource datasource = datasourceFacade.getDatasource(request.getName()); + Datasource datasource = datasourceDao.getDatasource(request.getName()); if (datasource == null) { throw new ResourceNotFoundException("no such datasource exist"); } @@ -124,7 +124,7 @@ private void updateIfChanged(final UpdateDatasourceRequest request, final Dataso } if (isChanged) { - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); } } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java b/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java index a42b31de..1249655b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java @@ -17,8 +17,8 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexingOperationListener; @@ -26,11 +26,11 @@ @Log4j2 public class Ip2GeoCache implements IndexingOperationListener { - private final DatasourceFacade datasourceFacade; + private final DatasourceDao datasourceDao; private Map data; - public Ip2GeoCache(final DatasourceFacade datasourceFacade) { - this.datasourceFacade = datasourceFacade; + public Ip2GeoCache(final DatasourceDao datasourceDao) { + this.datasourceDao = datasourceDao; } public String getIndexName(final String datasourceName) { @@ -58,7 +58,7 @@ private Map getData() { return data; } Map tempData = new ConcurrentHashMap<>(); - datasourceFacade.getAllDatasources() + datasourceDao.getAllDatasources() .stream() .forEach(datasource -> tempData.put(datasource.getName(), new DatasourceMetadata(datasource))); data = tempData; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java similarity index 97% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java rename to src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java index d0e13cd5..ec8e6f81 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import java.io.BufferedReader; import java.io.IOException; @@ -47,6 +47,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.shared.StashedThreadContext; @@ -56,16 +57,16 @@ import org.opensearch.search.SearchHit; /** - * Facade class for datasource + * Data access object for datasource */ @Log4j2 -public class DatasourceFacade { +public class DatasourceDao { private static final Integer MAX_SIZE = 1000; private final Client client; private final ClusterService clusterService; private final ClusterSettings clusterSettings; - public DatasourceFacade(final Client client, final ClusterService clusterService) { + public DatasourceDao(final Client client, final ClusterService clusterService) { this.client = client; this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); @@ -103,7 +104,7 @@ public void onFailure(final Exception e) { private String getIndexMapping() { try { - try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { + try (InputStream is = DatasourceDao.class.getResourceAsStream("/mappings/ip2geo_datasource.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java similarity index 97% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java rename to src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java index 6b3d71d7..2c0c5d72 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import static org.opensearch.geospatial.ip2geo.jobscheduler.Datasource.IP2GEO_DATA_INDEX_NAME_PREFIX; @@ -58,16 +58,18 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.constants.IndexSetting; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; /** - * Facade class for GeoIp data + * Data access object for GeoIp data */ @Log4j2 -public class GeoIpDataFacade { +public class GeoIpDataDao { public static final int BUNDLE_SIZE = 128; private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; @@ -91,7 +93,7 @@ public class GeoIpDataFacade { private final ClusterSettings clusterSettings; private final Client client; - public GeoIpDataFacade(final ClusterService clusterService, final Client client) { + public GeoIpDataDao(final ClusterService clusterService, final Client client) { this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); this.client = client; @@ -150,7 +152,7 @@ private void freezeIndex(final String indexName) { */ private String getIndexMapping() { try { - try (InputStream is = DatasourceFacade.class.getResourceAsStream("/mappings/ip2geo_geoip.json")) { + try (InputStream is = DatasourceDao.class.getResourceAsStream("/mappings/ip2geo_geoip.json")) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { return reader.lines().map(String::trim).collect(Collectors.joining()); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDao.java similarity index 85% rename from src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java rename to src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDao.java index 0f1b764f..55e1152d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacade.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDao.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import java.util.Collections; import java.util.List; @@ -14,11 +14,14 @@ import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.IngestService; -public class Ip2GeoProcessorFacade { +/** + * Data access object for Ip2Geo processors + */ +public class Ip2GeoProcessorDao { private final IngestService ingestService; @Inject - public Ip2GeoProcessorFacade(final IngestService ingestService) { + public Ip2GeoProcessorDao(final IngestService ingestService) { this.ingestService = ingestService; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java index b92a58a8..fe4aa36b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunner.java @@ -15,10 +15,10 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.geospatial.annotation.VisibleForTesting; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.jobscheduler.spi.JobExecutionContext; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; @@ -57,7 +57,7 @@ public static DatasourceRunner getJobRunnerInstance() { private ClusterService clusterService; private DatasourceUpdateService datasourceUpdateService; private Ip2GeoExecutor ip2GeoExecutor; - private DatasourceFacade datasourceFacade; + private DatasourceDao datasourceDao; private Ip2GeoLockService ip2GeoLockService; private boolean initialized; @@ -72,13 +72,13 @@ public void initialize( final ClusterService clusterService, final DatasourceUpdateService datasourceUpdateService, final Ip2GeoExecutor ip2GeoExecutor, - final DatasourceFacade datasourceFacade, + final DatasourceDao datasourceDao, final Ip2GeoLockService ip2GeoLockService ) { this.clusterService = clusterService; this.datasourceUpdateService = datasourceUpdateService; this.ip2GeoExecutor = ip2GeoExecutor; - this.datasourceFacade = datasourceFacade; + this.datasourceDao = datasourceDao; this.ip2GeoLockService = ip2GeoLockService; this.initialized = true; } @@ -131,7 +131,7 @@ protected Runnable updateDatasourceRunner(final ScheduledJobParameter jobParamet @VisibleForTesting protected void updateDatasource(final ScheduledJobParameter jobParameter, final Runnable renewLock) throws IOException { - Datasource datasource = datasourceFacade.getDatasource(jobParameter.getName()); + Datasource datasource = datasourceDao.getDatasource(jobParameter.getName()); /** * If delete request comes while update task is waiting on a queue for other update tasks to complete, * because update task for this datasource didn't acquire a lock yet, delete request is processed. @@ -147,7 +147,7 @@ protected void updateDatasource(final ScheduledJobParameter jobParameter, final log.error("Invalid datasource state. Expecting {} but received {}", DatasourceState.AVAILABLE, datasource.getState()); datasource.disable(); datasource.getUpdateStats().setLastFailedAt(Instant.now()); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); return; } @@ -160,7 +160,7 @@ protected void updateDatasource(final ScheduledJobParameter jobParameter, final } catch (Exception e) { log.error("Failed to update datasource for {}", datasource.getName(), e); datasource.getUpdateStats().setLastFailedAt(Instant.now()); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); } finally { postProcessing(datasource); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index d4e55f79..f14dd1f1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -22,28 +22,28 @@ import org.opensearch.OpenSearchException; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @Log4j2 public class DatasourceUpdateService { private final ClusterService clusterService; private final ClusterSettings clusterSettings; - private final DatasourceFacade datasourceFacade; - private final GeoIpDataFacade geoIpDataFacade; + private final DatasourceDao datasourceDao; + private final GeoIpDataDao geoIpDataDao; public DatasourceUpdateService( final ClusterService clusterService, - final DatasourceFacade datasourceFacade, - final GeoIpDataFacade geoIpDataFacade + final DatasourceDao datasourceDao, + final GeoIpDataDao geoIpDataDao ) { this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); - this.datasourceFacade = datasourceFacade; - this.geoIpDataFacade = geoIpDataFacade; + this.datasourceDao = datasourceDao; + this.geoIpDataDao = geoIpDataDao; } /** @@ -64,7 +64,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable if (shouldUpdate(datasource, manifest) == false) { log.info("Skipping GeoIP database update. Update is not required for {}", datasource.getName()); datasource.getUpdateStats().setLastSkippedAt(Instant.now()); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); return; } @@ -72,7 +72,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable String indexName = setupIndex(datasource); String[] header; List fieldsToStore; - try (CSVParser reader = geoIpDataFacade.getDatabaseReader(manifest)) { + try (CSVParser reader = geoIpDataDao.getDatabaseReader(manifest)) { CSVRecord headerLine = reader.iterator().next(); header = validateHeader(headerLine).values(); fieldsToStore = Arrays.asList(header).subList(1, header.length); @@ -83,7 +83,7 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable datasource.getDatabase().getFields().toString() ); } - geoIpDataFacade.putGeoIpData(indexName, header, reader.iterator(), renewLock); + geoIpDataDao.putGeoIpData(indexName, header, reader.iterator(), renewLock); } Instant endTime = Instant.now(); @@ -103,7 +103,7 @@ public List getHeaderFields(String manifestUrl) throws IOException { URL url = new URL(manifestUrl); DatasourceManifest manifest = DatasourceManifest.Builder.build(url); - try (CSVParser reader = geoIpDataFacade.getDatabaseReader(manifest)) { + try (CSVParser reader = geoIpDataDao.getDatabaseReader(manifest)) { String[] fields = reader.iterator().next().values(); return Arrays.asList(fields).subList(1, fields.length); } @@ -125,7 +125,7 @@ public void deleteUnusedIndices(final Datasource datasource) { if (deletedIndices.isEmpty() == false) { datasource.getIndices().removeAll(deletedIndices); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); } } catch (Exception e) { log.error("Failed to delete old indices for {}", datasource.getName(), e); @@ -151,7 +151,7 @@ public void updateDatasource(final Datasource datasource, final IntervalSchedule } if (updated) { - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); } } @@ -164,7 +164,7 @@ private List deleteIndices(final List indicesToDelete) { } try { - geoIpDataFacade.deleteIp2GeoDataIndex(index); + geoIpDataDao.deleteIp2GeoDataIndex(index); deletedIndices.add(index); } catch (Exception e) { log.error("Failed to delete an index [{}]", index, e); @@ -212,7 +212,7 @@ private void updateDatasourceAsSucceeded( datasource.getUpdateStats().setLastProcessingTimeInMillis(endTime.toEpochMilli() - startTime.toEpochMilli()); datasource.enable(); datasource.setState(DatasourceState.AVAILABLE); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); log.info( "GeoIP database creation succeeded for {} and took {} seconds", datasource.getName(), @@ -229,8 +229,8 @@ private void updateDatasourceAsSucceeded( private String setupIndex(final Datasource datasource) { String indexName = datasource.newIndexName(UUID.randomUUID().toString()); datasource.getIndices().add(indexName); - datasourceFacade.updateDatasource(datasource); - geoIpDataFacade.createIndexIfNotExists(indexName); + datasourceDao.updateDatasource(datasource); + geoIpDataDao.createIndexIfNotExists(indexName); return indexName; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java index a06bb30c..e6fe98f9 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListener.java @@ -25,8 +25,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; @@ -40,8 +40,8 @@ public class Ip2GeoListener extends AbstractLifecycleComponent implements Cluste private static final int DELAY_IN_MILLIS = 10000; private final ClusterService clusterService; private final ThreadPool threadPool; - private final DatasourceFacade datasourceFacade; - private final GeoIpDataFacade geoIpDataFacade; + private final DatasourceDao datasourceDao; + private final GeoIpDataDao geoIpDataDao; @Override public void clusterChanged(final ClusterChangedEvent event) { @@ -63,17 +63,17 @@ public void clusterChanged(final ClusterChangedEvent event) { .filter(index -> index.startsWith(IP2GEO_DATA_INDEX_NAME_PREFIX)) .collect(Collectors.toList()); if (ip2GeoDataIndices.isEmpty() == false) { - threadPool.generic().submit(() -> geoIpDataFacade.deleteIp2GeoDataIndex(ip2GeoDataIndices)); + threadPool.generic().submit(() -> geoIpDataDao.deleteIp2GeoDataIndex(ip2GeoDataIndices)); } } } private void forceUpdateGeoIpData() { - datasourceFacade.getAllDatasources(new ActionListener<>() { + datasourceDao.getAllDatasources(new ActionListener<>() { @Override public void onResponse(final List datasources) { datasources.stream().forEach(Ip2GeoListener.this::scheduleForceUpdate); - datasourceFacade.updateDatasource(datasources, new ActionListener<>() { + datasourceDao.updateDatasource(datasources, new ActionListener<>() { @Override public void onResponse(final BulkResponse bulkItemResponses) { log.info("Datasources are updated for cleanup"); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index f051ed1c..4c8f404b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -25,9 +25,9 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.IngestService; @@ -55,8 +55,8 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final Set properties; private final boolean ignoreMissing; private final ClusterSettings clusterSettings; - private final DatasourceFacade datasourceFacade; - private final GeoIpDataFacade geoIpDataFacade; + private final DatasourceDao datasourceDao; + private final GeoIpDataDao geoIpDataDao; private final Ip2GeoCache ip2GeoCache; /** @@ -74,8 +74,8 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param properties the properties * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param clusterSettings the cluster settings - * @param datasourceFacade the datasource facade - * @param geoIpDataFacade the geoip data facade + * @param datasourceDao the datasource facade + * @param geoIpDataDao the geoip data facade * @param ip2GeoCache the cache */ public Ip2GeoProcessor( @@ -87,8 +87,8 @@ public Ip2GeoProcessor( final Set properties, final boolean ignoreMissing, final ClusterSettings clusterSettings, - final DatasourceFacade datasourceFacade, - final GeoIpDataFacade geoIpDataFacade, + final DatasourceDao datasourceDao, + final GeoIpDataDao geoIpDataDao, final Ip2GeoCache ip2GeoCache ) { super(tag, description); @@ -98,8 +98,8 @@ public Ip2GeoProcessor( this.properties = properties; this.ignoreMissing = ignoreMissing; this.clusterSettings = clusterSettings; - this.datasourceFacade = datasourceFacade; - this.geoIpDataFacade = geoIpDataFacade; + this.datasourceDao = datasourceDao; + this.geoIpDataDao = geoIpDataDao; this.ip2GeoCache = ip2GeoCache; } @@ -161,7 +161,7 @@ protected void executeInternal( } try { - geoIpDataFacade.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); + geoIpDataDao.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); } catch (Exception e) { handler.accept(null, e); } @@ -249,7 +249,7 @@ protected void executeInternal( return; } - geoIpDataFacade.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); + geoIpDataDao.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); } @VisibleForTesting @@ -288,8 +288,8 @@ public String getType() { @AllArgsConstructor public static final class Factory implements Processor.Factory { private final IngestService ingestService; - private final DatasourceFacade datasourceFacade; - private final GeoIpDataFacade geoIpDataFacade; + private final DatasourceDao datasourceDao; + private final GeoIpDataDao geoIpDataDao; private final Ip2GeoCache ip2GeoCache; /** @@ -318,8 +318,8 @@ public Ip2GeoProcessor create( propertyNames == null ? null : new HashSet<>(propertyNames), ignoreMissing, ingestService.getClusterService().getClusterSettings(), - datasourceFacade, - geoIpDataFacade, + datasourceDao, + geoIpDataDao, ip2GeoCache ); } diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index f94133c2..a5653ad8 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -52,11 +52,11 @@ import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceAction; import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceTransportAction; import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; @@ -95,8 +95,8 @@ @Log4j2 public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlugin, MapperPlugin, SearchPlugin, SystemIndexPlugin { private Ip2GeoCache ip2GeoCache; - private DatasourceFacade datasourceFacade; - private GeoIpDataFacade geoIpDataFacade; + private DatasourceDao datasourceDao; + private GeoIpDataDao geoIpDataDao; @Override public Collection getSystemIndexDescriptors(Settings settings) { @@ -105,15 +105,12 @@ public Collection getSystemIndexDescriptors(Settings sett @Override public Map getProcessors(Processor.Parameters parameters) { - this.datasourceFacade = new DatasourceFacade(parameters.client, parameters.ingestService.getClusterService()); - this.geoIpDataFacade = new GeoIpDataFacade(parameters.ingestService.getClusterService(), parameters.client); - this.ip2GeoCache = new Ip2GeoCache(datasourceFacade); + this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService()); + this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client); + this.ip2GeoCache = new Ip2GeoCache(datasourceDao); return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) - .put( - Ip2GeoProcessor.TYPE, - new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceFacade, geoIpDataFacade, ip2GeoCache) - ) + .put(Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceDao, geoIpDataDao, ip2GeoCache)) .immutableMap(); } @@ -156,7 +153,7 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); + DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao); Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client); /** @@ -164,14 +161,14 @@ public Collection createComponents( * does not use DI but it calls DatasourceExtension#getJobRunner to get DatasourceRunner instance. */ DatasourceRunner.getJobRunnerInstance() - .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade, ip2GeoLockService); + .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceDao, ip2GeoLockService); return List.of( UploadStats.getInstance(), datasourceUpdateService, - datasourceFacade, + datasourceDao, ip2GeoExecutor, - geoIpDataFacade, + geoIpDataDao, ip2GeoLockService, ip2GeoCache ); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 5cf42fa6..cb641d49 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -44,13 +44,13 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; -import org.opensearch.geospatial.ip2geo.common.Ip2GeoProcessorFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoProcessorDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; @@ -73,11 +73,11 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { @Mock protected DatasourceUpdateService datasourceUpdateService; @Mock - protected DatasourceFacade datasourceFacade; + protected DatasourceDao datasourceDao; @Mock protected Ip2GeoExecutor ip2GeoExecutor; @Mock - protected GeoIpDataFacade geoIpDataFacade; + protected GeoIpDataDao geoIpDataDao; @Mock protected Ip2GeoCache ip2GeoCache; @Mock @@ -95,7 +95,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { @Mock protected Ip2GeoLockService ip2GeoLockService; @Mock - protected Ip2GeoProcessorFacade ip2GeoProcessorFacade; + protected Ip2GeoProcessorDao ip2GeoProcessorDao; protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; @@ -264,8 +264,8 @@ protected Ip2GeoProcessor randomIp2GeoProcessor(String datasourceName) { properties, true, clusterSettings, - datasourceFacade, - geoIpDataFacade, + datasourceDao, + geoIpDataDao, ip2GeoCache ); return ip2GeoProcessor; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java index ce5c91d4..3abf3c9d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceTransportActionTests.java @@ -46,9 +46,9 @@ public void init() { actionFilters, ip2GeoLockService, ingestService, - datasourceFacade, - geoIpDataFacade, - ip2GeoProcessorFacade, + datasourceDao, + geoIpDataDao, + ip2GeoProcessorDao, threadPool ); } @@ -74,7 +74,7 @@ public void testDoExecute_whenException_thenError() { private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { Task task = mock(Task.class); Datasource datasource = randomDatasource(); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); DeleteDatasourceRequest request = new DeleteDatasourceRequest(datasource.getName()); ActionListener listener = mock(ActionListener.class); @@ -113,45 +113,43 @@ public void testDeleteDatasource_whenNull_thenThrowException() { @SneakyThrows public void testDeleteDatasource_whenSafeToDelete_thenDelete() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); - when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn(Collections.emptyList()); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); + when(ip2GeoProcessorDao.getProcessors(datasource.getName())).thenReturn(Collections.emptyList()); // Run action.deleteDatasource(datasource.getName()); // Verify assertEquals(DatasourceState.DELETING, datasource.getState()); - verify(datasourceFacade).updateDatasource(datasource); - InOrder inOrder = Mockito.inOrder(geoIpDataFacade, datasourceFacade); - inOrder.verify(geoIpDataFacade).deleteIp2GeoDataIndex(datasource.getIndices()); - inOrder.verify(datasourceFacade).deleteDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); + InOrder inOrder = Mockito.inOrder(geoIpDataDao, datasourceDao); + inOrder.verify(geoIpDataDao).deleteIp2GeoDataIndex(datasource.getIndices()); + inOrder.verify(datasourceDao).deleteDatasource(datasource); } @SneakyThrows public void testDeleteDatasource_whenProcessorIsUsingDatasource_thenThrowException() { Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); - when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn( - Arrays.asList(randomIp2GeoProcessor(datasource.getName())) - ); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); + when(ip2GeoProcessorDao.getProcessors(datasource.getName())).thenReturn(Arrays.asList(randomIp2GeoProcessor(datasource.getName()))); // Run expectThrows(OpenSearchException.class, () -> action.deleteDatasource(datasource.getName())); // Verify assertEquals(DatasourceState.AVAILABLE, datasource.getState()); - verify(datasourceFacade, never()).updateDatasource(datasource); - verify(geoIpDataFacade, never()).deleteIp2GeoDataIndex(datasource.getIndices()); - verify(datasourceFacade, never()).deleteDatasource(datasource); + verify(datasourceDao, never()).updateDatasource(datasource); + verify(geoIpDataDao, never()).deleteIp2GeoDataIndex(datasource.getIndices()); + verify(datasourceDao, never()).deleteDatasource(datasource); } @SneakyThrows public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowException() { Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); - when(ip2GeoProcessorFacade.getProcessors(datasource.getName())).thenReturn( + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); + when(ip2GeoProcessorDao.getProcessors(datasource.getName())).thenReturn( Collections.emptyList(), Arrays.asList(randomIp2GeoProcessor(datasource.getName())) ); @@ -160,8 +158,8 @@ public void testDeleteDatasource_whenProcessorIsCreatedDuringDeletion_thenThrowE expectThrows(OpenSearchException.class, () -> action.deleteDatasource(datasource.getName())); // Verify - verify(datasourceFacade, times(2)).updateDatasource(datasource); - verify(geoIpDataFacade, never()).deleteIp2GeoDataIndex(datasource.getIndices()); - verify(datasourceFacade, never()).deleteDatasource(datasource); + verify(datasourceDao, times(2)).updateDatasource(datasource); + verify(geoIpDataDao, never()).deleteIp2GeoDataIndex(datasource.getIndices()); + verify(datasourceDao, never()).deleteDatasource(datasource); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java index 0404da0f..b7732a31 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java @@ -26,7 +26,7 @@ public class GetDatasourceTransportActionTests extends Ip2GeoTestCase { @Before public void init() { - action = new GetDatasourceTransportAction(transportService, actionFilters, datasourceFacade); + action = new GetDatasourceTransportAction(transportService, actionFilters, datasourceDao); } public void testDoExecute_whenAll_thenSucceed() { @@ -38,7 +38,7 @@ public void testDoExecute_whenAll_thenSucceed() { action.doExecute(task, request, listener); // Verify - verify(datasourceFacade).getAllDatasources(any(ActionListener.class)); + verify(datasourceDao).getAllDatasources(any(ActionListener.class)); } public void testDoExecute_whenNames_thenSucceed() { @@ -53,7 +53,7 @@ public void testDoExecute_whenNames_thenSucceed() { action.doExecute(task, request, listener); // Verify - verify(datasourceFacade).getDatasources(eq(datasourceNames), any(ActionListener.class)); + verify(datasourceDao).getDatasources(eq(datasourceNames), any(ActionListener.class)); } public void testNewActionListener_whenOnResponse_thenSucceed() { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java index ff838b84..ef426cc5 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceTransportActionTests.java @@ -42,7 +42,7 @@ public void init() { transportService, actionFilters, threadPool, - datasourceFacade, + datasourceDao, datasourceUpdateService, ip2GeoLockService ); @@ -74,7 +74,7 @@ private void validateDoExecute(final LockModel lockModel, final Exception before PutDatasourceRequest request = new PutDatasourceRequest(datasource.getName()); ActionListener listener = mock(ActionListener.class); if (after != null) { - doThrow(after).when(datasourceFacade).createIndexIfNotExists(any(StepListener.class)); + doThrow(after).when(datasourceDao).createIndexIfNotExists(any(StepListener.class)); } // Run @@ -118,14 +118,14 @@ public void testInternalDoExecute_whenValidInput_thenSucceed() { // Verify ArgumentCaptor captor = ArgumentCaptor.forClass(StepListener.class); - verify(datasourceFacade).createIndexIfNotExists(captor.capture()); + verify(datasourceDao).createIndexIfNotExists(captor.capture()); // Run captor.getValue().onResponse(null); // Verify ArgumentCaptor datasourceCaptor = ArgumentCaptor.forClass(Datasource.class); ArgumentCaptor actionListenerCaptor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).putDatasource(datasourceCaptor.capture(), actionListenerCaptor.capture()); + verify(datasourceDao).putDatasource(datasourceCaptor.capture(), actionListenerCaptor.capture()); assertEquals(request.getName(), datasourceCaptor.getValue().getName()); assertEquals(request.getEndpoint(), datasourceCaptor.getValue().getEndpoint()); assertEquals(request.getUpdateInterval().days(), datasourceCaptor.getValue().getUserSchedule().getInterval()); @@ -162,7 +162,7 @@ public void testCreateDatasource_whenInvalidState_thenUpdateStateAsFailed() { // Verify assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); assertNotNull(datasource.getUpdateStats().getLastFailedAt()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); verify(datasourceUpdateService, never()).updateOrCreateGeoIpData(any(Datasource.class), any(Runnable.class)); } @@ -177,7 +177,7 @@ public void testCreateDatasource_whenExceptionHappens_thenUpdateStateAsFailed() // Verify assertEquals(DatasourceState.CREATE_FAILED, datasource.getState()); assertNotNull(datasource.getUpdateStats().getLastFailedAt()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); } @SneakyThrows diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java index 32272f62..e0a94a75 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceTransportActionTests.java @@ -44,7 +44,7 @@ public void init() { transportService, actionFilters, ip2GeoLockService, - datasourceFacade, + datasourceDao, datasourceUpdateService, threadPool ); @@ -94,7 +94,7 @@ public void testDoExecute_whenValidInput_thenUpdate() { request.setUpdateInterval(TimeValue.timeValueDays(datasource.getSchedule().getInterval())); Task task = mock(Task.class); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); when(datasourceUpdateService.getHeaderFields(request.getEndpoint())).thenReturn(datasource.getDatabase().getFields()); ActionListener listener = mock(ActionListener.class); LockModel lockModel = randomLockModel(); @@ -110,8 +110,8 @@ public void testDoExecute_whenValidInput_thenUpdate() { captor.getValue().onResponse(lockModel); // Verify - verify(datasourceFacade).getDatasource(datasource.getName()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).getDatasource(datasource.getName()); + verify(datasourceDao).updateDatasource(datasource); verify(datasourceUpdateService).getHeaderFields(request.getEndpoint()); assertEquals(request.getEndpoint(), datasource.getEndpoint()); assertEquals(request.getUpdateInterval().days(), datasource.getUserSchedule().getInterval()); @@ -128,7 +128,7 @@ public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { request.setEndpoint(datasource.getEndpoint()); Task task = mock(Task.class); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); ActionListener listener = mock(ActionListener.class); LockModel lockModel = randomLockModel(); @@ -143,9 +143,9 @@ public void testDoExecute_whenNoChangesInValues_thenNoUpdate() { captor.getValue().onResponse(lockModel); // Verify - verify(datasourceFacade).getDatasource(datasource.getName()); + verify(datasourceDao).getDatasource(datasource.getName()); verify(datasourceUpdateService, never()).getHeaderFields(anyString()); - verify(datasourceFacade, never()).updateDatasource(datasource); + verify(datasourceDao, never()).updateDatasource(datasource); verify(listener).onResponse(new AcknowledgedResponse(true)); verify(ip2GeoLockService).releaseLock(eq(lockModel)); } @@ -184,7 +184,7 @@ public void testDoExecute_whenIncompatibleFields_thenError() { request.setEndpoint(sampleManifestUrl()); Task task = mock(Task.class); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); List newFields = datasource.getDatabase().getFields().subList(0, 0); when(datasourceUpdateService.getHeaderFields(request.getEndpoint())).thenReturn(newFields); ActionListener listener = mock(ActionListener.class); @@ -215,7 +215,7 @@ public void testDoExecute_whenLargeUpdateInterval_thenError() { request.setUpdateInterval(TimeValue.timeValueDays(datasource.getDatabase().getValidForInDays())); Task task = mock(Task.class); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); ActionListener listener = mock(ActionListener.class); LockModel lockModel = randomLockModel(); @@ -246,7 +246,7 @@ public void testDoExecute_whenExpireWithNewUpdateInterval_thenError() { request.setUpdateInterval(TimeValue.timeValueDays(1)); Task task = mock(Task.class); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); ActionListener listener = mock(ActionListener.class); LockModel lockModel = randomLockModel(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java index 01dc2fc5..28549905 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java @@ -28,12 +28,12 @@ public class Ip2GeoCacheTests extends Ip2GeoTestCase { @Before public void init() { - ip2GeoCache = new Ip2GeoCache(datasourceFacade); + ip2GeoCache = new Ip2GeoCache(datasourceDao); } public void testGetIndexName_whenCalled_thenReturnIndexName() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run String indexName = ip2GeoCache.getIndexName(datasource.getName()); @@ -46,7 +46,7 @@ public void testIsExpired_whenExpired_thenReturnTrue() { Datasource datasource = randomDatasource(); datasource.getUpdateStats().setLastSucceededAt(Instant.MIN); datasource.getUpdateStats().setLastSkippedAt(null); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); @@ -59,7 +59,7 @@ public void testIsExpired_whenNotExpired_thenReturnFalse() { Datasource datasource = randomDatasource(); datasource.getUpdateStats().setLastSucceededAt(Instant.now()); datasource.getUpdateStats().setLastSkippedAt(null); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); @@ -70,7 +70,7 @@ public void testIsExpired_whenNotExpired_thenReturnFalse() { public void testHas_whenHasDatasource_thenReturnTrue() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run boolean hasDatasource = ip2GeoCache.has(datasource.getName()); @@ -81,7 +81,7 @@ public void testHas_whenHasDatasource_thenReturnTrue() { public void testHas_whenNoDatasource_thenReturnFalse() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); String datasourceName = GeospatialTestHelper.randomLowerCaseString(); // Run @@ -93,7 +93,7 @@ public void testHas_whenNoDatasource_thenReturnFalse() { public void testGetState_whenCalled_thenReturnState() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run DatasourceState state = ip2GeoCache.getState(datasource.getName()); @@ -104,7 +104,7 @@ public void testGetState_whenCalled_thenReturnState() { @SneakyThrows public void testPostIndex_whenFailed_thenNoUpdate() { - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList()); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList()); Datasource datasource = randomDatasource(); ShardId shardId = mock(ShardId.class); @@ -126,7 +126,7 @@ public void testPostIndex_whenFailed_thenNoUpdate() { @SneakyThrows public void testPostIndex_whenSucceed_thenUpdate() { - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList()); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList()); Datasource datasource = randomDatasource(); ShardId shardId = mock(ShardId.class); @@ -148,7 +148,7 @@ public void testPostIndex_whenSucceed_thenUpdate() { public void testPostDelete_whenFailed_thenNoUpdate() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); ShardId shardId = mock(ShardId.class); Engine.Delete index = mock(Engine.Delete.class); @@ -164,7 +164,7 @@ public void testPostDelete_whenFailed_thenNoUpdate() { public void testPostDelete_whenSucceed_thenUpdate() { Datasource datasource = randomDatasource(); - when(datasourceFacade.getAllDatasources()).thenReturn(Arrays.asList(datasource)); + when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); ShardId shardId = mock(ShardId.class); Engine.Delete index = mock(Engine.Delete.class); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java similarity index 91% rename from src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java rename to src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java index 748e6ab4..09e2dd46 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/DatasourceFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -53,12 +53,12 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -public class DatasourceFacadeTests extends Ip2GeoTestCase { - private DatasourceFacade datasourceFacade; +public class DatasourceDaoTests extends Ip2GeoTestCase { + private DatasourceDao datasourceDao; @Before public void init() { - datasourceFacade = new DatasourceFacade(verifyingClient, clusterService); + datasourceDao = new DatasourceDao(verifyingClient, clusterService); } public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCalled() { @@ -69,7 +69,7 @@ public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCall // Run StepListener stepListener = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(stepListener); + datasourceDao.createIndexIfNotExists(stepListener); // Verify stepListener is called stepListener.result(); @@ -92,7 +92,7 @@ public void testCreateIndexIfNotExists_whenIndexExist_thenCreateRequestIsCalled( // Run StepListener stepListener = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(stepListener); + datasourceDao.createIndexIfNotExists(stepListener); // Verify stepListener is called stepListener.result(); @@ -106,7 +106,7 @@ public void testCreateIndexIfNotExists_whenIndexCreatedAlready_thenExceptionIsIg // Run StepListener stepListener = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(stepListener); + datasourceDao.createIndexIfNotExists(stepListener); // Verify stepListener is called stepListener.result(); @@ -118,7 +118,7 @@ public void testCreateIndexIfNotExists_whenExceptionIsThrown_thenExceptionIsThro // Run StepListener stepListener = new StepListener<>(); - datasourceFacade.createIndexIfNotExists(stepListener); + datasourceDao.createIndexIfNotExists(stepListener); // Verify stepListener is called expectThrows(RuntimeException.class, () -> stepListener.result()); @@ -144,7 +144,7 @@ public void testUpdateDatasource_whenValidInput_thenSucceed() throws Exception { return null; }); - datasourceFacade.updateDatasource(datasource); + datasourceDao.updateDatasource(datasource); assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); } @@ -164,36 +164,36 @@ public void testPutDatasource_whenValidInpu_thenSucceed() { return null; }); - datasourceFacade.putDatasource(datasource, mock(ActionListener.class)); + datasourceDao.putDatasource(datasource, mock(ActionListener.class)); assertTrue(previousTime.isBefore(datasource.getLastUpdateTime())); } public void testGetDatasource_whenException_thenNull() throws Exception { Datasource datasource = setupClientForGetRequest(true, new IndexNotFoundException(DatasourceExtension.JOB_INDEX_NAME)); - assertNull(datasourceFacade.getDatasource(datasource.getName())); + assertNull(datasourceDao.getDatasource(datasource.getName())); } public void testGetDatasource_whenExist_thenReturnDatasource() throws Exception { Datasource datasource = setupClientForGetRequest(true, null); - assertEquals(datasource, datasourceFacade.getDatasource(datasource.getName())); + assertEquals(datasource, datasourceDao.getDatasource(datasource.getName())); } public void testGetDatasource_whenNotExist_thenNull() throws Exception { Datasource datasource = setupClientForGetRequest(false, null); - assertNull(datasourceFacade.getDatasource(datasource.getName())); + assertNull(datasourceDao.getDatasource(datasource.getName())); } public void testGetDatasource_whenExistWithListener_thenListenerIsCalledWithDatasource() { Datasource datasource = setupClientForGetRequest(true, null); ActionListener listener = mock(ActionListener.class); - datasourceFacade.getDatasource(datasource.getName(), listener); + datasourceDao.getDatasource(datasource.getName(), listener); verify(listener).onResponse(eq(datasource)); } public void testGetDatasource_whenNotExistWithListener_thenListenerIsCalledWithNull() { Datasource datasource = setupClientForGetRequest(false, null); ActionListener listener = mock(ActionListener.class); - datasourceFacade.getDatasource(datasource.getName(), listener); + datasourceDao.getDatasource(datasource.getName(), listener); verify(listener).onResponse(null); } @@ -231,7 +231,7 @@ public void testDeleteDatasource_whenValidInput_thenSucceed() { }); // Run - datasourceFacade.deleteDatasource(datasource); + datasourceDao.deleteDatasource(datasource); } public void testDeleteDatasource_whenIndexNotFound_thenThrowException() { @@ -243,7 +243,7 @@ public void testDeleteDatasource_whenIndexNotFound_thenThrowException() { }); // Run - expectThrows(ResourceNotFoundException.class, () -> datasourceFacade.deleteDatasource(datasource)); + expectThrows(ResourceNotFoundException.class, () -> datasourceDao.deleteDatasource(datasource)); } public void testGetDatasources_whenValidInput_thenSucceed() { @@ -273,7 +273,7 @@ public void testGetDatasources_whenValidInput_thenSucceed() { }); // Run - datasourceFacade.getDatasources(names, listener); + datasourceDao.getDatasources(names, listener); // Verify ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); @@ -302,7 +302,7 @@ public void testGetAllDatasources_whenAsynchronous_thenSucceed() { }); // Run - datasourceFacade.getAllDatasources(listener); + datasourceDao.getAllDatasources(listener); // Verify ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); @@ -329,10 +329,10 @@ public void testGetAllDatasources_whenSynchronous_thenSucceed() { }); // Run - datasourceFacade.getAllDatasources(); + datasourceDao.getAllDatasources(); // Verify - assertEquals(datasources, datasourceFacade.getAllDatasources()); + assertEquals(datasources, datasourceDao.getAllDatasources()); } public void testUpdateDatasource_whenValidInput_thenUpdate() { @@ -353,7 +353,7 @@ public void testUpdateDatasource_whenValidInput_thenUpdate() { return null; }); - datasourceFacade.updateDatasource(datasources, mock(ActionListener.class)); + datasourceDao.updateDatasource(datasources, mock(ActionListener.class)); } private SearchHits getMockedSearchHits(List datasources) { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java similarity index 91% rename from src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java rename to src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java index 9e51f23b..13872a27 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -50,29 +50,30 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; +import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.shared.Constants; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @SuppressForbidden(reason = "unit test") -public class GeoIpDataFacadeTests extends Ip2GeoTestCase { +public class GeoIpDataDaoTests extends Ip2GeoTestCase { private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; - private GeoIpDataFacade noOpsGeoIpDataFacade; - private GeoIpDataFacade verifyingGeoIpDataFacade; + private GeoIpDataDao noOpsGeoIpDataDao; + private GeoIpDataDao verifyingGeoIpDataDao; @Before public void init() { - noOpsGeoIpDataFacade = new GeoIpDataFacade(clusterService, client); - verifyingGeoIpDataFacade = new GeoIpDataFacade(clusterService, verifyingClient); + noOpsGeoIpDataDao = new GeoIpDataDao(clusterService, client); + verifyingGeoIpDataDao = new GeoIpDataDao(clusterService, verifyingClient); } public void testCreateIndexIfNotExistsWithExistingIndex() { String index = GeospatialTestHelper.randomLowerCaseString(); when(metadata.hasIndex(index)).thenReturn(true); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); - verifyingGeoIpDataFacade.createIndexIfNotExists(index); + verifyingGeoIpDataDao.createIndexIfNotExists(index); } public void testCreateIndexIfNotExistsWithoutExistingIndex() { @@ -94,7 +95,7 @@ public void testCreateIndexIfNotExistsWithoutExistingIndex() { ); return null; }); - verifyingGeoIpDataFacade.createIndexIfNotExists(index); + verifyingGeoIpDataDao.createIndexIfNotExists(index); } @SneakyThrows @@ -103,7 +104,7 @@ public void testCreateDocument_whenBlankValue_thenDoNotAdd() { String[] values = { "1.0.0.0/25", "USA", " ", "Seattle" }; assertEquals( "{\"_cidr\":\"1.0.0.0/25\",\"_data\":{\"country\":\"USA\",\"city\":\"Seattle\"}}", - Strings.toString(noOpsGeoIpDataFacade.createDocument(names, values)) + Strings.toString(noOpsGeoIpDataDao.createDocument(names, values)) ); } @@ -117,7 +118,7 @@ public void testGetDatabaseReader() throws Exception { Instant.now().toEpochMilli(), "tester" ); - CSVParser parser = noOpsGeoIpDataFacade.getDatabaseReader(manifest); + CSVParser parser = noOpsGeoIpDataDao.getDatabaseReader(manifest); String[] expectedHeader = { "network", "country_name" }; assertArrayEquals(expectedHeader, parser.iterator().next().values()); String[] expectedValues = { "1.0.0.0/24", "Australia" }; @@ -134,7 +135,7 @@ public void testGetDatabaseReaderNoFile() throws Exception { Instant.now().toEpochMilli(), "tester" ); - OpenSearchException exception = expectThrows(OpenSearchException.class, () -> noOpsGeoIpDataFacade.getDatabaseReader(manifest)); + OpenSearchException exception = expectThrows(OpenSearchException.class, () -> noOpsGeoIpDataDao.getDatabaseReader(manifest)); assertTrue(exception.getMessage().contains("does not exist")); } @@ -154,7 +155,7 @@ public void testInternalGetDatabaseReader_whenCalled_thenSetUserAgent() { when(connection.getInputStream()).thenReturn(new FileInputStream(zipFile)); // Run - noOpsGeoIpDataFacade.internalGetDatabaseReader(manifest, connection); + noOpsGeoIpDataDao.internalGetDatabaseReader(manifest, connection); // Verify verify(connection).addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); @@ -169,12 +170,12 @@ public void testDeleteIp2GeoDataIndex_whenCalled_thenDeleteIndex() { assertEquals(index, request.indices()[0]); return new AcknowledgedResponse(true); }); - verifyingGeoIpDataFacade.deleteIp2GeoDataIndex(index); + verifyingGeoIpDataDao.deleteIp2GeoDataIndex(index); } public void testDeleteIp2GeoDataIndexWithNonIp2GeoDataIndex() { String index = GeospatialTestHelper.randomLowerCaseString(); - Exception e = expectThrows(OpenSearchException.class, () -> verifyingGeoIpDataFacade.deleteIp2GeoDataIndex(index)); + Exception e = expectThrows(OpenSearchException.class, () -> verifyingGeoIpDataDao.deleteIp2GeoDataIndex(index)); assertTrue(e.getMessage().contains("not ip2geo data index")); verify(verifyingClient, never()).index(any()); } @@ -216,7 +217,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { try (CSVParser csvParser = CSVParser.parse(sampleIp2GeoFile(), StandardCharsets.UTF_8, CSVFormat.RFC4180)) { Iterator iterator = csvParser.iterator(); String[] fields = iterator.next().values(); - verifyingGeoIpDataFacade.putGeoIpData(index, fields, iterator, renewLock); + verifyingGeoIpDataDao.putGeoIpData(index, fields, iterator, renewLock); verify(renewLock, times(2)).run(); } } @@ -247,7 +248,7 @@ public void testGetGeoIpData_whenSingleIp_thenSucceed() { return response; }); ActionListener> listener = mock(ActionListener.class); - verifyingGeoIpDataFacade.getGeoIpData(indexName, ip, listener); + verifyingGeoIpDataDao.getGeoIpData(indexName, ip, listener); ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); verify(listener).onResponse(captor.capture()); assertEquals("seattle", captor.getValue().get("city")); @@ -279,7 +280,7 @@ public void testGetGeoIpData_whenMultiIps_thenSucceed() { return response; }); ActionListener>> listener = mock(ActionListener.class); - verifyingGeoIpDataFacade.getGeoIpData(indexName, Arrays.asList(ip), listener); + verifyingGeoIpDataDao.getGeoIpData(indexName, Arrays.asList(ip), listener); ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class); verify(listener).onResponse(captor.capture()); assertEquals("seattle", captor.getValue().get(0).get("city")); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java similarity index 90% rename from src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java rename to src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java index 05fed2ca..b298d8ef 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoProcessorFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.common; +package org.opensearch.geospatial.ip2geo.dao; import static org.mockito.Mockito.when; @@ -23,19 +23,19 @@ import org.opensearch.ingest.IngestMetadata; import org.opensearch.ingest.PipelineConfiguration; -public class Ip2GeoProcessorFacadeTests extends Ip2GeoTestCase { - private Ip2GeoProcessorFacade ip2GeoProcessorFacade; +public class Ip2GeoProcessorDaoTests extends Ip2GeoTestCase { + private Ip2GeoProcessorDao ip2GeoProcessorDao; @Before public void init() { - ip2GeoProcessorFacade = new Ip2GeoProcessorFacade(ingestService); + ip2GeoProcessorDao = new Ip2GeoProcessorDao(ingestService); } public void testGetProcessors_whenNullMetadata_thenReturnEmpty() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); when(metadata.custom(IngestMetadata.TYPE)).thenReturn(null); - List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceName); + List ip2GeoProcessorList = ip2GeoProcessorDao.getProcessors(datasourceName); assertTrue(ip2GeoProcessorList.isEmpty()); } @@ -50,7 +50,7 @@ public void testGetProcessors_whenNoProcessorForGivenDatasource_thenReturnEmpty( Ip2GeoProcessor ip2GeoProcessor = randomIp2GeoProcessor(datasourceBeingUsed); when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn(Arrays.asList(ip2GeoProcessor)); - List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceNotBeingUsed); + List ip2GeoProcessorList = ip2GeoProcessorDao.getProcessors(datasourceNotBeingUsed); assertTrue(ip2GeoProcessorList.isEmpty()); } @@ -64,7 +64,7 @@ public void testGetProcessors_whenProcessorsForGivenDatasource_thenReturnProcess Ip2GeoProcessor ip2GeoProcessor = randomIp2GeoProcessor(datasourceName); when(ingestService.getProcessorsInPipeline(pipelineId, Ip2GeoProcessor.class)).thenReturn(Arrays.asList(ip2GeoProcessor)); - List ip2GeoProcessorList = ip2GeoProcessorFacade.getProcessors(datasourceName); + List ip2GeoProcessorList = ip2GeoProcessorDao.getProcessors(datasourceName); assertEquals(1, ip2GeoProcessorList.size()); assertEquals(ip2GeoProcessor.getDatasourceName(), ip2GeoProcessorList.get(0).getDatasourceName()); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index cc7deb03..1800cdb6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -38,7 +38,7 @@ public class DatasourceRunnerTests extends Ip2GeoTestCase { @Before public void init() { DatasourceRunner.getJobRunnerInstance() - .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceFacade, ip2GeoLockService); + .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceDao, ip2GeoLockService); } public void testRunJob_whenInvalidClass_thenThrowException() { @@ -70,7 +70,7 @@ public void testRunJob_whenValidInput_thenSucceed() { // Verify verify(ip2GeoLockService).acquireLock(datasource.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS); - verify(datasourceFacade).getDatasource(datasource.getName()); + verify(datasourceDao).getDatasource(datasource.getName()); verify(ip2GeoLockService).releaseLock(lockModel); } @@ -97,7 +97,7 @@ public void testUpdateDatasourceRunner_whenExceptionAfterAcquiringLock_thenRelea when(ip2GeoLockService.acquireLock(jobParameter.getName(), Ip2GeoLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( Optional.of(lockModel) ); - when(datasourceFacade.getDatasource(jobParameter.getName())).thenThrow(new RuntimeException()); + when(datasourceDao.getDatasource(jobParameter.getName())).thenThrow(new RuntimeException()); // Run DatasourceRunner.getJobRunnerInstance().updateDatasourceRunner(jobParameter).run(); @@ -123,7 +123,7 @@ public void testUpdateDatasource_whenInvalidState_thenUpdateLastFailedAt() { datasource.enable(); datasource.getUpdateStats().setLastFailedAt(null); datasource.setState(randomStateExcept(DatasourceState.AVAILABLE)); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); // Run DatasourceRunner.getJobRunnerInstance().updateDatasource(datasource, mock(Runnable.class)); @@ -131,14 +131,14 @@ public void testUpdateDatasource_whenInvalidState_thenUpdateLastFailedAt() { // Verify assertFalse(datasource.isEnabled()); assertNotNull(datasource.getUpdateStats().getLastFailedAt()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); } @SneakyThrows public void testUpdateDatasource_whenValidInput_thenSucceed() { Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Runnable renewLock = mock(Runnable.class); // Run @@ -155,7 +155,7 @@ public void testUpdateDatasource_whenDeleteTask_thenDeleteOnly() { Datasource datasource = randomDatasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.setTask(DatasourceTask.DELETE_UNUSED_INDICES); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Runnable renewLock = mock(Runnable.class); // Run @@ -174,7 +174,7 @@ public void testUpdateDatasource_whenExpired_thenDeleteIndicesAgain() { datasource.getUpdateStats() .setLastSucceededAt(Instant.now().minus(datasource.getDatabase().getValidForInDays() + 1, ChronoUnit.DAYS)); datasource.setState(DatasourceState.AVAILABLE); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Runnable renewLock = mock(Runnable.class); // Run @@ -193,7 +193,7 @@ public void testUpdateDatasource_whenWillExpire_thenScheduleDeleteTask() { datasource.getUpdateStats() .setLastSucceededAt(Instant.now().minus(datasource.getDatabase().getValidForInDays(), ChronoUnit.DAYS).plusSeconds(60)); datasource.setState(DatasourceState.AVAILABLE); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Runnable renewLock = mock(Runnable.class); // Run @@ -213,7 +213,7 @@ public void testUpdateDatasourceExceptionHandling() { Datasource datasource = new Datasource(); datasource.setName(randomLowerCaseString()); datasource.getUpdateStats().setLastFailedAt(null); - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); doThrow(new RuntimeException("test failure")).when(datasourceUpdateService).deleteUnusedIndices(any()); // Run @@ -221,6 +221,6 @@ public void testUpdateDatasourceExceptionHandling() { // Verify assertNotNull(datasource.getUpdateStats().getLastFailedAt()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index b110ddea..80a3beac 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -41,7 +41,7 @@ public class DatasourceUpdateServiceTests extends Ip2GeoTestCase { @Before public void init() { - datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceFacade, geoIpDataFacade); + datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao); } @SneakyThrows @@ -61,7 +61,7 @@ public void testUpdateOrCreateGeoIpData_whenHashValueIsSame_thenSkipUpdate() { // Verify assertNotNull(datasource.getUpdateStats().getLastSkippedAt()); - verify(datasourceFacade).updateDatasource(datasource); + verify(datasourceDao).updateDatasource(datasource); } @SneakyThrows @@ -70,7 +70,7 @@ public void testUpdateOrCreateGeoIpData_whenExpired_thenUpdate() { DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); @@ -84,12 +84,7 @@ public void testUpdateOrCreateGeoIpData_whenExpired_thenUpdate() { datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class)); // Verify - verify(geoIpDataFacade).putGeoIpData( - eq(datasource.currentIndexName()), - isA(String[].class), - any(Iterator.class), - any(Runnable.class) - ); + verify(geoIpDataDao).putGeoIpData(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class)); } @SneakyThrows @@ -100,7 +95,7 @@ public void testUpdateOrCreateGeoIpData_whenInvalidData_thenThrowException() { File sampleFile = new File( this.getClass().getClassLoader().getResource("ip2geo/sample_invalid_less_than_two_fields.csv").getFile() ); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); @@ -119,7 +114,7 @@ public void testUpdateOrCreateGeoIpData_whenIncompatibleFields_thenThrowExceptio DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); @@ -138,7 +133,7 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { DatasourceManifest manifest = DatasourceManifest.Builder.build(manifestFile.toURI().toURL()); File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); @@ -159,13 +154,8 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { assertEquals(manifest.getValidForInDays(), datasource.getDatabase().getValidForInDays()); assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); assertNotNull(datasource.getUpdateStats().getLastProcessingTimeInMillis()); - verify(datasourceFacade, times(2)).updateDatasource(datasource); - verify(geoIpDataFacade).putGeoIpData( - eq(datasource.currentIndexName()), - isA(String[].class), - any(Iterator.class), - any(Runnable.class) - ); + verify(datasourceDao, times(2)).updateDatasource(datasource); + verify(geoIpDataDao).putGeoIpData(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class)); } @SneakyThrows @@ -173,7 +163,7 @@ public void testGetHeaderFields_whenValidInput_thenReturnCorrectValue() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); // Run assertEquals(Arrays.asList("country_name"), datasourceUpdateService.getHeaderFields(manifestFile.toURI().toURL().toExternalForm())); @@ -203,8 +193,8 @@ public void testDeleteUnusedIndices_whenValidInput_thenSucceed() { assertEquals(1, datasource.getIndices().size()); assertEquals(currentIndex, datasource.getIndices().get(0)); - verify(datasourceFacade).updateDatasource(datasource); - verify(geoIpDataFacade).deleteIp2GeoDataIndex(oldIndex); + verify(datasourceDao).updateDatasource(datasource); + verify(geoIpDataDao).deleteIp2GeoDataIndex(oldIndex); } public void testUpdateDatasource_whenNoChange_thenNoUpdate() { @@ -214,7 +204,7 @@ public void testUpdateDatasource_whenNoChange_thenNoUpdate() { datasourceUpdateService.updateDatasource(datasource, datasource.getSystemSchedule(), datasource.getTask()); // Verify - verify(datasourceFacade, never()).updateDatasource(any()); + verify(datasourceDao, never()).updateDatasource(any()); } public void testUpdateDatasource_whenChange_thenUpdate() { @@ -230,14 +220,14 @@ public void testUpdateDatasource_whenChange_thenUpdate() { datasourceUpdateService.updateDatasource(datasource, datasource.getSystemSchedule(), DatasourceTask.DELETE_UNUSED_INDICES); // Verify - verify(datasourceFacade, times(2)).updateDatasource(any()); + verify(datasourceDao, times(2)).updateDatasource(any()); } @SneakyThrows public void testGetHeaderFields_whenValidInput_thenSucceed() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); - when(geoIpDataFacade.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); // Run List fields = datasourceUpdateService.getHeaderFields(manifestFile.toURI().toURL().toExternalForm()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java index a72a5a1b..d31f38bc 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/listener/Ip2GeoListenerTests.java @@ -36,7 +36,7 @@ public class Ip2GeoListenerTests extends Ip2GeoTestCase { @Before public void init() { - ip2GeoListener = new Ip2GeoListener(clusterService, threadPool, datasourceFacade, geoIpDataFacade); + ip2GeoListener = new Ip2GeoListener(clusterService, threadPool, datasourceDao, geoIpDataDao); } public void testDoStart_whenClusterManagerNode_thenAddListener() { @@ -151,7 +151,7 @@ public void testClusterChanged_whenDatasourceIndexIsRestored_thenUpdate() { // Verify verify(threadPool).generic(); ArgumentCaptor>> captor = ArgumentCaptor.forClass(ActionListener.class); - verify(datasourceFacade).getAllDatasources(captor.capture()); + verify(datasourceDao).getAllDatasources(captor.capture()); // Run List datasources = Arrays.asList(randomDatasource(), randomDatasource()); @@ -167,7 +167,7 @@ public void testClusterChanged_whenDatasourceIndexIsRestored_thenUpdate() { assertTrue(datasource.getSystemSchedule().getNextExecutionTime(Instant.now()).isAfter(Instant.now())); assertTrue(datasource.getSystemSchedule().getNextExecutionTime(Instant.now()).isBefore(Instant.now().plusSeconds(60))); }); - verify(datasourceFacade).updateDatasource(eq(datasources), any()); + verify(datasourceDao).updateDatasource(eq(datasources), any()); } public void testClusterChanged_whenGeoIpDataIsRestored_thenDelete() { @@ -193,7 +193,7 @@ public void testClusterChanged_whenGeoIpDataIsRestored_thenDelete() { // Verify verify(threadPool).generic(); - verify(geoIpDataFacade).deleteIp2GeoDataIndex(Arrays.asList(datasource.currentIndexName())); + verify(geoIpDataDao).deleteIp2GeoDataIndex(Arrays.asList(datasource.currentIndexName())); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 656375ca..aa3b52aa 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -41,7 +41,7 @@ public class Ip2GeoProcessorTests extends Ip2GeoTestCase { @Before public void init() { - factory = new Ip2GeoProcessor.Factory(ingestService, datasourceFacade, geoIpDataFacade, ip2GeoCache); + factory = new Ip2GeoProcessor.Factory(ingestService, datasourceDao, geoIpDataDao, ip2GeoCache); } public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { @@ -154,7 +154,7 @@ public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { processor.executeInternal(document, handler, ip); // Verify - verify(geoIpDataFacade).getGeoIpData(anyString(), anyString(), any(ActionListener.class)); + verify(geoIpDataDao).getGeoIpData(anyString(), anyString(), any(ActionListener.class)); } @SneakyThrows @@ -282,7 +282,7 @@ public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { processor.executeInternal(document, handler, ips); // Verify - verify(geoIpDataFacade).getGeoIpData(anyString(), anyList(), any(ActionListener.class)); + verify(geoIpDataDao).getGeoIpData(anyString(), anyList(), any(ActionListener.class)); } private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { @@ -294,7 +294,7 @@ private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { - when(datasourceFacade.getDatasource(datasource.getName())).thenReturn(datasource); + when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Map baseConfig = new HashMap<>(); baseConfig.put(CONFIG_FIELD_KEY, "ip"); baseConfig.put(CONFIG_DATASOURCE_KEY, datasource.getName()); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 7e2e9e01..92e5b63a 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -38,11 +38,11 @@ import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; -import org.opensearch.geospatial.ip2geo.common.DatasourceFacade; -import org.opensearch.geospatial.ip2geo.common.GeoIpDataFacade; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; +import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.listener.Ip2GeoListener; import org.opensearch.geospatial.processor.FeatureProcessor; @@ -77,9 +77,9 @@ public class GeospatialPluginTests extends OpenSearchTestCase { private final Set SUPPORTED_COMPONENTS = Set.of( UploadStats.class, DatasourceUpdateService.class, - DatasourceFacade.class, + DatasourceDao.class, Ip2GeoExecutor.class, - GeoIpDataFacade.class, + GeoIpDataDao.class, Ip2GeoLockService.class, Ip2GeoCache.class ); From a45760f00a2b1ce4fe6e4e831626f2c6f3bf0540 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 26 Jun 2023 13:59:56 -0700 Subject: [PATCH 56/61] Refactoring of code (#342) 1. Changed class name from Ip2GeoCache to Ip2GeoCachedDao 2. Moved the Ip2GeoCachedDao from cache to dao package Signed-off-by: Heemin Kim --- .../Ip2GeoCachedDao.java} | 10 ++-- .../ip2geo/processor/Ip2GeoProcessor.java | 26 +++++----- .../geospatial/plugin/GeospatialPlugin.java | 12 ++--- .../geospatial/ip2geo/Ip2GeoTestCase.java | 6 +-- .../Ip2GeoCachedDaoTests.java} | 48 +++++++++---------- .../processor/Ip2GeoProcessorTests.java | 28 +++++------ .../plugin/GeospatialPluginTests.java | 4 +- 7 files changed, 68 insertions(+), 66 deletions(-) rename src/main/java/org/opensearch/geospatial/ip2geo/{cache/Ip2GeoCache.java => dao/Ip2GeoCachedDao.java} (93%) rename src/test/java/org/opensearch/geospatial/ip2geo/{cache/Ip2GeoCacheTests.java => dao/Ip2GeoCachedDaoTests.java} (76%) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java similarity index 93% rename from src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java rename to src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java index 1249655b..cd645cd1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCache.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.cache; +package org.opensearch.geospatial.ip2geo.dao; import java.io.IOException; import java.time.Instant; @@ -18,18 +18,20 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.ip2geo.common.DatasourceState; -import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.ShardId; +/** + * Data access object for Datasource and GeoIP data with added caching layer + */ @Log4j2 -public class Ip2GeoCache implements IndexingOperationListener { +public class Ip2GeoCachedDao implements IndexingOperationListener { private final DatasourceDao datasourceDao; private Map data; - public Ip2GeoCache(final DatasourceDao datasourceDao) { + public Ip2GeoCachedDao(final DatasourceDao datasourceDao) { this.datasourceDao = datasourceDao; } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 4c8f404b..356869fe 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -24,10 +24,10 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.geospatial.annotation.VisibleForTesting; -import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.IngestService; @@ -57,7 +57,7 @@ public final class Ip2GeoProcessor extends AbstractProcessor { private final ClusterSettings clusterSettings; private final DatasourceDao datasourceDao; private final GeoIpDataDao geoIpDataDao; - private final Ip2GeoCache ip2GeoCache; + private final Ip2GeoCachedDao ip2GeoCachedDao; /** * Ip2Geo processor type @@ -76,7 +76,7 @@ public final class Ip2GeoProcessor extends AbstractProcessor { * @param clusterSettings the cluster settings * @param datasourceDao the datasource facade * @param geoIpDataDao the geoip data facade - * @param ip2GeoCache the cache + * @param ip2GeoCachedDao the cache */ public Ip2GeoProcessor( final String tag, @@ -89,7 +89,7 @@ public Ip2GeoProcessor( final ClusterSettings clusterSettings, final DatasourceDao datasourceDao, final GeoIpDataDao geoIpDataDao, - final Ip2GeoCache ip2GeoCache + final Ip2GeoCachedDao ip2GeoCachedDao ) { super(tag, description); this.field = field; @@ -100,7 +100,7 @@ public Ip2GeoProcessor( this.clusterSettings = clusterSettings; this.datasourceDao = datasourceDao; this.geoIpDataDao = geoIpDataDao; - this.ip2GeoCache = ip2GeoCache; + this.ip2GeoCachedDao = ip2GeoCachedDao; } /** @@ -154,8 +154,8 @@ protected void executeInternal( final String ip ) { validateDatasourceIsInAvailableState(datasourceName); - String indexName = ip2GeoCache.getIndexName(datasourceName); - if (ip2GeoCache.isExpired(datasourceName) || indexName == null) { + String indexName = ip2GeoCachedDao.getIndexName(datasourceName); + if (ip2GeoCachedDao.isExpired(datasourceName) || indexName == null) { handleExpiredData(ingestDocument, handler); return; } @@ -209,11 +209,11 @@ private List> filteredGeoData(final List } private void validateDatasourceIsInAvailableState(final String datasourceName) { - if (ip2GeoCache.has(datasourceName) == false) { + if (ip2GeoCachedDao.has(datasourceName) == false) { throw new IllegalStateException("datasource does not exist"); } - if (DatasourceState.AVAILABLE.equals(ip2GeoCache.getState(datasourceName)) == false) { + if (DatasourceState.AVAILABLE.equals(ip2GeoCachedDao.getState(datasourceName)) == false) { throw new IllegalStateException("datasource is not in an available state"); } } @@ -243,8 +243,8 @@ protected void executeInternal( } validateDatasourceIsInAvailableState(datasourceName); - String indexName = ip2GeoCache.getIndexName(datasourceName); - if (ip2GeoCache.isExpired(datasourceName) || indexName == null) { + String indexName = ip2GeoCachedDao.getIndexName(datasourceName); + if (ip2GeoCachedDao.isExpired(datasourceName) || indexName == null) { handleExpiredData(ingestDocument, handler); return; } @@ -290,7 +290,7 @@ public static final class Factory implements Processor.Factory { private final IngestService ingestService; private final DatasourceDao datasourceDao; private final GeoIpDataDao geoIpDataDao; - private final Ip2GeoCache ip2GeoCache; + private final Ip2GeoCachedDao ip2GeoCachedDao; /** * Within this method, blocking request cannot be called because this method is executed in a transport thread. @@ -320,7 +320,7 @@ public Ip2GeoProcessor create( ingestService.getClusterService().getClusterSettings(), datasourceDao, geoIpDataDao, - ip2GeoCache + ip2GeoCachedDao ); } } diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index a5653ad8..0eca5df7 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -51,12 +51,12 @@ import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceAction; import org.opensearch.geospatial.ip2geo.action.UpdateDatasourceTransportAction; -import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceExtension; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceRunner; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; @@ -94,7 +94,7 @@ */ @Log4j2 public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlugin, MapperPlugin, SearchPlugin, SystemIndexPlugin { - private Ip2GeoCache ip2GeoCache; + private Ip2GeoCachedDao ip2GeoCachedDao; private DatasourceDao datasourceDao; private GeoIpDataDao geoIpDataDao; @@ -107,17 +107,17 @@ public Collection getSystemIndexDescriptors(Settings sett public Map getProcessors(Processor.Parameters parameters) { this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService()); this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client); - this.ip2GeoCache = new Ip2GeoCache(datasourceDao); + this.ip2GeoCachedDao = new Ip2GeoCachedDao(datasourceDao); return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) - .put(Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceDao, geoIpDataDao, ip2GeoCache)) + .put(Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceDao, geoIpDataDao, ip2GeoCachedDao)) .immutableMap(); } @Override public void onIndexModule(IndexModule indexModule) { if (DatasourceExtension.JOB_INDEX_NAME.equals(indexModule.getIndex().getName())) { - indexModule.addIndexOperationListener(ip2GeoCache); + indexModule.addIndexOperationListener(ip2GeoCachedDao); log.info("Ip2GeoListener started listening to operations on index {}", DatasourceExtension.JOB_INDEX_NAME); } } @@ -170,7 +170,7 @@ public Collection createComponents( ip2GeoExecutor, geoIpDataDao, ip2GeoLockService, - ip2GeoCache + ip2GeoCachedDao ); } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index cb641d49..e59e2c35 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -43,13 +43,13 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.geospatial.GeospatialTestHelper; -import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; import org.opensearch.geospatial.ip2geo.dao.Ip2GeoProcessorDao; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceTask; @@ -79,7 +79,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { @Mock protected GeoIpDataDao geoIpDataDao; @Mock - protected Ip2GeoCache ip2GeoCache; + protected Ip2GeoCachedDao ip2GeoCachedDao; @Mock protected ClusterState clusterState; @Mock @@ -266,7 +266,7 @@ protected Ip2GeoProcessor randomIp2GeoProcessor(String datasourceName) { clusterSettings, datasourceDao, geoIpDataDao, - ip2GeoCache + ip2GeoCachedDao ); return ip2GeoProcessor; } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java similarity index 76% rename from src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java rename to src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java index 28549905..5aba0f62 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/cache/Ip2GeoCacheTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.geospatial.ip2geo.cache; +package org.opensearch.geospatial.ip2geo.dao; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -23,12 +23,12 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.ShardId; -public class Ip2GeoCacheTests extends Ip2GeoTestCase { - private Ip2GeoCache ip2GeoCache; +public class Ip2GeoCachedDaoTests extends Ip2GeoTestCase { + private Ip2GeoCachedDao ip2GeoCachedDao; @Before public void init() { - ip2GeoCache = new Ip2GeoCache(datasourceDao); + ip2GeoCachedDao = new Ip2GeoCachedDao(datasourceDao); } public void testGetIndexName_whenCalled_thenReturnIndexName() { @@ -36,7 +36,7 @@ public void testGetIndexName_whenCalled_thenReturnIndexName() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run - String indexName = ip2GeoCache.getIndexName(datasource.getName()); + String indexName = ip2GeoCachedDao.getIndexName(datasource.getName()); // Verify assertEquals(datasource.currentIndexName(), indexName); @@ -49,7 +49,7 @@ public void testIsExpired_whenExpired_thenReturnTrue() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run - boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); + boolean isExpired = ip2GeoCachedDao.isExpired(datasource.getName()); // Verify assertTrue(isExpired); @@ -62,7 +62,7 @@ public void testIsExpired_whenNotExpired_thenReturnFalse() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run - boolean isExpired = ip2GeoCache.isExpired(datasource.getName()); + boolean isExpired = ip2GeoCachedDao.isExpired(datasource.getName()); // Verify assertFalse(isExpired); @@ -73,7 +73,7 @@ public void testHas_whenHasDatasource_thenReturnTrue() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run - boolean hasDatasource = ip2GeoCache.has(datasource.getName()); + boolean hasDatasource = ip2GeoCachedDao.has(datasource.getName()); // Verify assertTrue(hasDatasource); @@ -85,7 +85,7 @@ public void testHas_whenNoDatasource_thenReturnFalse() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); // Run - boolean hasDatasource = ip2GeoCache.has(datasourceName); + boolean hasDatasource = ip2GeoCachedDao.has(datasourceName); // Verify assertFalse(hasDatasource); @@ -96,7 +96,7 @@ public void testGetState_whenCalled_thenReturnState() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList(datasource)); // Run - DatasourceState state = ip2GeoCache.getState(datasource.getName()); + DatasourceState state = ip2GeoCachedDao.getState(datasource.getName()); // Verify assertEquals(datasource.getState(), state); @@ -115,13 +115,13 @@ public void testPostIndex_whenFailed_thenNoUpdate() { when(result.getResultType()).thenReturn(Engine.Result.Type.FAILURE); // Run - ip2GeoCache.postIndex(shardId, index, result); + ip2GeoCachedDao.postIndex(shardId, index, result); // Verify - assertFalse(ip2GeoCache.has(datasource.getName())); - assertTrue(ip2GeoCache.isExpired(datasource.getName())); - assertNull(ip2GeoCache.getIndexName(datasource.getName())); - assertNull(ip2GeoCache.getState(datasource.getName())); + assertFalse(ip2GeoCachedDao.has(datasource.getName())); + assertTrue(ip2GeoCachedDao.isExpired(datasource.getName())); + assertNull(ip2GeoCachedDao.getIndexName(datasource.getName())); + assertNull(ip2GeoCachedDao.getState(datasource.getName())); } @SneakyThrows @@ -137,13 +137,13 @@ public void testPostIndex_whenSucceed_thenUpdate() { when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); // Run - ip2GeoCache.postIndex(shardId, index, result); + ip2GeoCachedDao.postIndex(shardId, index, result); // Verify - assertTrue(ip2GeoCache.has(datasource.getName())); - assertFalse(ip2GeoCache.isExpired(datasource.getName())); - assertEquals(datasource.currentIndexName(), ip2GeoCache.getIndexName(datasource.getName())); - assertEquals(datasource.getState(), ip2GeoCache.getState(datasource.getName())); + assertTrue(ip2GeoCachedDao.has(datasource.getName())); + assertFalse(ip2GeoCachedDao.isExpired(datasource.getName())); + assertEquals(datasource.currentIndexName(), ip2GeoCachedDao.getIndexName(datasource.getName())); + assertEquals(datasource.getState(), ip2GeoCachedDao.getState(datasource.getName())); } public void testPostDelete_whenFailed_thenNoUpdate() { @@ -156,10 +156,10 @@ public void testPostDelete_whenFailed_thenNoUpdate() { when(result.getResultType()).thenReturn(Engine.Result.Type.FAILURE); // Run - ip2GeoCache.postDelete(shardId, index, result); + ip2GeoCachedDao.postDelete(shardId, index, result); // Verify - assertTrue(ip2GeoCache.has(datasource.getName())); + assertTrue(ip2GeoCachedDao.has(datasource.getName())); } public void testPostDelete_whenSucceed_thenUpdate() { @@ -173,9 +173,9 @@ public void testPostDelete_whenSucceed_thenUpdate() { when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); // Run - ip2GeoCache.postDelete(shardId, index, result); + ip2GeoCachedDao.postDelete(shardId, index, result); // Verify - assertFalse(ip2GeoCache.has(datasource.getName())); + assertFalse(ip2GeoCachedDao.has(datasource.getName())); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index aa3b52aa..700fe56b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -41,7 +41,7 @@ public class Ip2GeoProcessorTests extends Ip2GeoTestCase { @Before public void init() { - factory = new Ip2GeoProcessor.Factory(ingestService, datasourceDao, geoIpDataDao, ip2GeoCache); + factory = new Ip2GeoProcessor.Factory(ingestService, datasourceDao, geoIpDataDao, ip2GeoCachedDao); } public void testExecuteWithNoIpAndIgnoreMissing() throws Exception { @@ -96,7 +96,7 @@ public void testExecute_whenNoDatasource_thenNotExistError() { source.put("ip", ip); IngestDocument document = new IngestDocument(source, new HashMap<>()); - when(ip2GeoCache.has(datasourceName)).thenReturn(false); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(false); BiConsumer handler = mock(BiConsumer.class); // Run @@ -119,10 +119,10 @@ public void testExecute_whenExpired_thenExpiredMsg() { IngestDocument document = new IngestDocument(source, new HashMap<>()); String index = GeospatialTestHelper.randomLowerCaseString(); - when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(index); - when(ip2GeoCache.has(datasourceName)).thenReturn(true); - when(ip2GeoCache.isExpired(datasourceName)).thenReturn(true); - when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(index); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); BiConsumer handler = mock(BiConsumer.class); @@ -145,10 +145,10 @@ public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { BiConsumer handler = mock(BiConsumer.class); String indexName = GeospatialTestHelper.randomLowerCaseString(); - when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(indexName); - when(ip2GeoCache.has(datasourceName)).thenReturn(true); - when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); - when(ip2GeoCache.isExpired(datasourceName)).thenReturn(false); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); // Run processor.executeInternal(document, handler, ip); @@ -273,10 +273,10 @@ public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); String indexName = GeospatialTestHelper.randomLowerCaseString(); - when(ip2GeoCache.getIndexName(datasourceName)).thenReturn(indexName); - when(ip2GeoCache.has(datasourceName)).thenReturn(true); - when(ip2GeoCache.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); - when(ip2GeoCache.isExpired(datasourceName)).thenReturn(false); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); // Run processor.executeInternal(document, handler, ips); diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 92e5b63a..d4d5db55 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -37,12 +37,12 @@ import org.opensearch.geospatial.ip2geo.action.RestGetDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestPutDatasourceHandler; import org.opensearch.geospatial.ip2geo.action.RestUpdateDatasourceHandler; -import org.opensearch.geospatial.ip2geo.cache.Ip2GeoCache; import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; +import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; import org.opensearch.geospatial.ip2geo.jobscheduler.DatasourceUpdateService; import org.opensearch.geospatial.ip2geo.listener.Ip2GeoListener; import org.opensearch.geospatial.processor.FeatureProcessor; @@ -81,7 +81,7 @@ public class GeospatialPluginTests extends OpenSearchTestCase { Ip2GeoExecutor.class, GeoIpDataDao.class, Ip2GeoLockService.class, - Ip2GeoCache.class + Ip2GeoCachedDao.class ); @Mock From dd3f3936b3d5d5554f31bd579a9c94d1a175b21d Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 26 Jun 2023 15:00:40 -0700 Subject: [PATCH 57/61] Add geo data cache (#340) Signed-off-by: Heemin Kim --- .../ip2geo/dao/Ip2GeoCachedDao.java | 69 +++++++++++++++++++ .../ip2geo/dao/Ip2GeoCachedDaoTests.java | 59 ++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java index cd645cd1..23e98279 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java @@ -7,16 +7,24 @@ import java.io.IOException; import java.time.Instant; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.extern.log4j.Log4j2; +import org.opensearch.common.cache.Cache; +import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.index.engine.Engine; @@ -119,4 +127,65 @@ public DatasourceMetadata(final Datasource datasource) { this.state = datasource.getState(); } } + + /** + * Cache to hold geo data + * + * GeoData in an index in immutable. Therefore, invalidation is not needed. + */ + @VisibleForTesting + protected static class GeoDataCache { + private Cache> cache; + + public GeoDataCache(final long maxSize) { + if (maxSize < 0) { + throw new IllegalArgumentException("ip2geo max cache size must be 0 or greater"); + } + this.cache = CacheBuilder.>builder().setMaximumWeight(maxSize).build(); + } + + public Map putIfAbsent( + final String indexName, + final String ip, + final Function> retrieveFunction + ) throws ExecutionException { + CacheKey cacheKey = new CacheKey(indexName, ip); + return cache.computeIfAbsent(cacheKey, key -> retrieveFunction.apply(key.ip)); + } + + public Map get(final String indexName, final String ip) { + return cache.get(new CacheKey(indexName, ip)); + } + + /** + * Create a new cache with give size and replace existing cache + * + * Try to populate the existing value from previous cache to the new cache in best effort + * + * @param maxSize + */ + public void updateMaxSize(final long maxSize) { + if (maxSize < 0) { + throw new IllegalArgumentException("ip2geo max cache size must be 0 or greater"); + } + Cache> temp = CacheBuilder.>builder() + .setMaximumWeight(maxSize) + .build(); + int count = 0; + Iterator it = cache.keys().iterator(); + while (it.hasNext() && count < maxSize) { + CacheKey key = it.next(); + temp.put(key, cache.get(key)); + count++; + } + cache = temp; + } + + @AllArgsConstructor + @EqualsAndHashCode + private static class CacheKey { + private final String indexName; + private final String ip; + } + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java index 5aba0f62..4906fba9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java @@ -9,12 +9,16 @@ import static org.mockito.Mockito.when; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import lombok.SneakyThrows; import org.junit.Before; import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -178,4 +182,59 @@ public void testPostDelete_whenSucceed_thenUpdate() { // Verify assertFalse(ip2GeoCachedDao.has(datasource.getName())); } + + @SneakyThrows + public void testUpdateMaxSize_whenBiggerSize_thenContainsAllData() { + int cacheSize = 10; + String datasource = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoCachedDao.GeoDataCache geoDataCache = new Ip2GeoCachedDao.GeoDataCache(cacheSize); + List ips = new ArrayList<>(cacheSize); + for (int i = 0; i < cacheSize; i++) { + String ip = NetworkAddress.format(randomIp(false)); + ips.add(ip); + geoDataCache.putIfAbsent(datasource, ip, addr -> Collections.emptyMap()); + } + + // Verify all data exist in the cache + assertTrue(ips.stream().allMatch(ip -> geoDataCache.get(datasource, ip) != null)); + + // Update cache size + int newCacheSize = 15; + geoDataCache.updateMaxSize(newCacheSize); + + // Verify all data exist in the cache + assertTrue(ips.stream().allMatch(ip -> geoDataCache.get(datasource, ip) != null)); + + // Add (newCacheSize - cacheSize + 1) data and the first data should not be available in the cache + for (int i = 0; i < newCacheSize - cacheSize + 1; i++) { + geoDataCache.putIfAbsent(datasource, NetworkAddress.format(randomIp(false)), addr -> Collections.emptyMap()); + } + assertNull(geoDataCache.get(datasource, ips.get(0))); + } + + @SneakyThrows + public void testUpdateMaxSize_whenSmallerSize_thenContainsPartialData() { + int cacheSize = 10; + String datasource = GeospatialTestHelper.randomLowerCaseString(); + Ip2GeoCachedDao.GeoDataCache geoDataCache = new Ip2GeoCachedDao.GeoDataCache(cacheSize); + List ips = new ArrayList<>(cacheSize); + for (int i = 0; i < cacheSize; i++) { + String ip = NetworkAddress.format(randomIp(false)); + ips.add(ip); + geoDataCache.putIfAbsent(datasource, ip, addr -> Collections.emptyMap()); + } + + // Verify all data exist in the cache + assertTrue(ips.stream().allMatch(ip -> geoDataCache.get(datasource, ip) != null)); + + // Update cache size + int newCacheSize = 5; + geoDataCache.updateMaxSize(newCacheSize); + + // Verify the last (cacheSize - newCacheSize) data is available in the cache + List deleted = ips.subList(0, ips.size() - newCacheSize); + List retained = ips.subList(ips.size() - newCacheSize, ips.size()); + assertTrue(deleted.stream().allMatch(ip -> geoDataCache.get(datasource, ip) == null)); + assertTrue(retained.stream().allMatch(ip -> geoDataCache.get(datasource, ip) != null)); + } } From 418c3bc50c522ad047b656bf2086526398f5e80a Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 29 Jun 2023 09:11:45 -0700 Subject: [PATCH 58/61] Add cache layer to reduce GeoIp data retrieval latency (#343) Signed-off-by: Heemin Kim --- .../ip2geo/action/GetDatasourceRequest.java | 7 +- .../ip2geo/common/Ip2GeoSettings.java | 15 +- .../geospatial/ip2geo/dao/GeoIpDataDao.java | 90 +------ .../ip2geo/dao/Ip2GeoCachedDao.java | 63 +++-- .../ip2geo/processor/Ip2GeoProcessor.java | 83 ++----- .../geospatial/plugin/GeospatialPlugin.java | 2 +- .../action/DeleteDatasourceRequestTests.java | 23 ++ .../action/GetDatasourceRequestTests.java | 12 + .../GetDatasourceTransportActionTests.java | 13 ++ .../ip2geo/dao/GeoIpDataDaoTests.java | 56 ++--- .../ip2geo/dao/Ip2GeoCachedDaoTests.java | 27 ++- .../jobscheduler/DatasourceRunnerTests.java | 4 + .../processor/Ip2GeoProcessorTests.java | 221 +++++++++--------- 13 files changed, 293 insertions(+), 323 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java index aa32c8f7..889d7899 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java @@ -10,7 +10,6 @@ import lombok.Getter; import lombok.Setter; -import org.opensearch.OpenSearchException; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.StreamInput; @@ -52,10 +51,12 @@ public GetDatasourceRequest(final StreamInput in) throws IOException { @Override public ActionRequestValidationException validate() { + ActionRequestValidationException errors = null; if (names == null) { - throw new OpenSearchException("names should not be null"); + errors = new ActionRequestValidationException(); + errors.addValidationError("names should not be null"); } - return null; + return errors; } @Override diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index 12d06b50..0c95fc92 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -52,16 +52,11 @@ public class Ip2GeoSettings { ); /** - * Multi search max concurrent searches - * - * Multi search is used only when a field contains a list of ip addresses. - * - * When the value is 0, it will use default value which will be decided - * based on node count and search thread pool size. + * Max size for geo data cache */ - public static final Setting MAX_CONCURRENT_SEARCHES = Setting.intSetting( - "plugins.geospatial.ip2geo.processor.max_concurrent_searches", - 0, + public static final Setting CACHE_SIZE = Setting.longSetting( + "plugins.geospatial.ip2geo.processor.cache_size", + 1000, 0, Setting.Property.NodeScope, Setting.Property.Dynamic @@ -72,7 +67,7 @@ public class Ip2GeoSettings { * @return a list of all settings for Ip2Geo feature */ public static final List> settings() { - return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, TIMEOUT, MAX_CONCURRENT_SEARCHES); + return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, TIMEOUT, CACHE_SIZE); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java index 2c0c5d72..a930781c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.util.Strings; import org.opensearch.OpenSearchException; import org.opensearch.SpecialPermission; -import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.bulk.BulkRequest; @@ -62,7 +61,6 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; -import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; /** @@ -242,94 +240,26 @@ public XContentBuilder createDocument(final String[] fields, final String[] valu * * @param indexName index * @param ip ip address - * @param actionListener action listener + * @return geoIP data */ - public void getGeoIpData(final String indexName, final String ip, final ActionListener> actionListener) { - StashedThreadContext.run( + public Map getGeoIpData(final String indexName, final String ip) { + SearchResponse response = StashedThreadContext.run( client, () -> client.prepareSearch(indexName) .setSize(1) .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) .setPreference("_local") .setRequestCache(true) - .execute(new ActionListener<>() { - @Override - public void onResponse(final SearchResponse searchResponse) { - try { - if (searchResponse.getHits().getHits().length == 0) { - actionListener.onResponse(Collections.emptyMap()); - } else { - Map geoIpData = (Map) XContentHelper.convertToMap( - searchResponse.getHits().getAt(0).getSourceRef(), - false, - XContentType.JSON - ).v2().get(DATA_FIELD_NAME); - actionListener.onResponse(geoIpData); - } - } catch (Exception e) { - actionListener.onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - }) + .get(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) ); - } - /** - * Query a given index using a given list of ip addresses to get geoip data - * - * @param indexName index - * @param ips list of ip addresses - * @param actionListener action listener - */ - public void getGeoIpData( - final String indexName, - final List ips, - final ActionListener>> actionListener - ) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - ips.stream().forEach(ip -> boolQueryBuilder.should(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip))); - StashedThreadContext.run( - client, - () -> client.prepareSearch(indexName) - .setSize(ips.size()) - .setQuery(boolQueryBuilder) - .setPreference("_local") - .setRequestCache(true) - .execute(new ActionListener<>() { - @Override - public void onResponse(final SearchResponse searchResponse) { - try { - actionListener.onResponse(toGeoIpDataList(searchResponse)); - } catch (Exception e) { - actionListener.onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - }) - ); - } - - private List> toGeoIpDataList(final SearchResponse searchResponse) { - if (searchResponse.getHits().getHits().length == 0) { - return Collections.emptyList(); + if (response.getHits().getHits().length == 0) { + return Collections.emptyMap(); + } else { + return (Map) XContentHelper.convertToMap(response.getHits().getAt(0).getSourceRef(), false, XContentType.JSON) + .v2() + .get(DATA_FIELD_NAME); } - - return Arrays.stream(searchResponse.getHits().getHits()) - .map( - data -> (Map) XContentHelper.convertToMap(data.getSourceRef(), false, XContentType.JSON) - .v2() - .get(DATA_FIELD_NAME) - ) - .collect(Collectors.toList()); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java index 23e98279..dfe44ceb 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java @@ -18,6 +18,7 @@ import lombok.Getter; import lombok.extern.log4j.Log4j2; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.cache.Cache; import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -26,63 +27,89 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.ShardId; /** * Data access object for Datasource and GeoIP data with added caching layer + * + * Ip2GeoCachedDao has a memory cache to store Datasource and GeoIP data. To fully utilize the cache, + * do not create multiple Ip2GeoCachedDao. Ip2GeoCachedDao instance is bound to guice so that you can use + * it through injection. + * + * All IP2Geo processors share single Ip2GeoCachedDao instance. */ @Log4j2 public class Ip2GeoCachedDao implements IndexingOperationListener { private final DatasourceDao datasourceDao; - private Map data; + private final GeoIpDataDao geoIpDataDao; + private final GeoDataCache geoDataCache; + private Map metadata; - public Ip2GeoCachedDao(final DatasourceDao datasourceDao) { + public Ip2GeoCachedDao(final ClusterService clusterService, final DatasourceDao datasourceDao, final GeoIpDataDao geoIpDataDao) { this.datasourceDao = datasourceDao; + this.geoIpDataDao = geoIpDataDao; + this.geoDataCache = new GeoDataCache(clusterService.getClusterSettings().get(Ip2GeoSettings.CACHE_SIZE)); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(Ip2GeoSettings.CACHE_SIZE, setting -> this.geoDataCache.updateMaxSize(setting.longValue())); } public String getIndexName(final String datasourceName) { - return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getIndexName(); + return getMetadata().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getIndexName(); } public boolean isExpired(final String datasourceName) { - return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getExpirationDate().isBefore(Instant.now()); + return getMetadata().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getExpirationDate().isBefore(Instant.now()); } public boolean has(final String datasourceName) { - return getData().containsKey(datasourceName); + return getMetadata().containsKey(datasourceName); } public DatasourceState getState(final String datasourceName) { - return getData().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getState(); + return getMetadata().getOrDefault(datasourceName, DatasourceMetadata.EMPTY_METADATA).getState(); } - private Map getData() { - if (data != null) { - return data; + public Map getGeoData(final String indexName, final String ip) { + try { + return geoDataCache.putIfAbsent(indexName, ip, addr -> geoIpDataDao.getGeoIpData(indexName, ip)); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + + private Map getMetadata() { + if (metadata != null) { + return metadata; } synchronized (this) { - if (data != null) { - return data; + if (metadata != null) { + return metadata; } Map tempData = new ConcurrentHashMap<>(); - datasourceDao.getAllDatasources() - .stream() - .forEach(datasource -> tempData.put(datasource.getName(), new DatasourceMetadata(datasource))); - data = tempData; - return data; + try { + datasourceDao.getAllDatasources() + .stream() + .forEach(datasource -> tempData.put(datasource.getName(), new DatasourceMetadata(datasource))); + } catch (IndexNotFoundException e) { + log.debug("Datasource has never been created"); + } + metadata = tempData; + return metadata; } } private void put(final Datasource datasource) { DatasourceMetadata metadata = new DatasourceMetadata(datasource); - getData().put(datasource.getName(), metadata); + getMetadata().put(datasource.getName(), metadata); } private void remove(final String datasourceName) { - getData().remove(datasourceName); + getMetadata().remove(datasourceName); } @Override diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java index 356869fe..56100c0b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessor.java @@ -21,9 +21,7 @@ import lombok.Getter; import lombok.extern.log4j.Log4j2; -import org.opensearch.action.ActionListener; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; @@ -147,8 +145,7 @@ public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException("Not implemented"); } - @VisibleForTesting - protected void executeInternal( + private void executeInternal( final IngestDocument ingestDocument, final BiConsumer handler, final String ip @@ -160,36 +157,11 @@ protected void executeInternal( return; } - try { - geoIpDataDao.getGeoIpData(indexName, ip, getSingleGeoIpDataListener(ingestDocument, handler)); - } catch (Exception e) { - handler.accept(null, e); + Map geoData = ip2GeoCachedDao.getGeoData(indexName, ip); + if (geoData.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, filteredGeoData(geoData)); } - } - - @VisibleForTesting - protected ActionListener> getSingleGeoIpDataListener( - final IngestDocument ingestDocument, - final BiConsumer handler - ) { - return new ActionListener<>() { - @Override - public void onResponse(final Map ipToGeoData) { - try { - if (ipToGeoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData)); - } - handler.accept(ingestDocument, null); - } catch (Exception e) { - handler.accept(null, e); - } - } - - @Override - public void onFailure(final Exception e) { - handler.accept(null, e); - } - }; + handler.accept(ingestDocument, null); } private Map filteredGeoData(final Map geoData) { @@ -200,14 +172,6 @@ private Map filteredGeoData(final Map geoData) { return properties.stream().filter(p -> geoData.containsKey(p)).collect(Collectors.toMap(p -> p, p -> geoData.get(p))); } - private List> filteredGeoData(final List> geoData) { - if (properties == null) { - return geoData; - } - - return geoData.stream().map(this::filteredGeoData).collect(Collectors.toList()); - } - private void validateDatasourceIsInAvailableState(final String datasourceName) { if (ip2GeoCachedDao.has(datasourceName) == false) { throw new IllegalStateException("datasource does not exist"); @@ -230,8 +194,7 @@ private void handleExpiredData(final IngestDocument ingestDocument, final BiCons * @param handler the handler * @param ips the ip list */ - @VisibleForTesting - protected void executeInternal( + private void executeInternal( final IngestDocument ingestDocument, final BiConsumer handler, final List ips @@ -249,32 +212,16 @@ protected void executeInternal( return; } - geoIpDataDao.getGeoIpData(indexName, (List) ips, getMultiGeoIpDataListener(ingestDocument, handler)); - } - - @VisibleForTesting - protected ActionListener>> getMultiGeoIpDataListener( - final IngestDocument ingestDocument, - final BiConsumer handler - ) { - return new ActionListener<>() { - @Override - public void onResponse(final List> ipToGeoData) { - try { - if (ipToGeoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, filteredGeoData(ipToGeoData)); - } - handler.accept(ingestDocument, null); - } catch (Exception e) { - handler.accept(null, e); - } - } + List> geoDataList = ips.stream() + .map(ip -> ip2GeoCachedDao.getGeoData(indexName, (String) ip)) + .filter(geoData -> geoData.isEmpty() == false) + .map(this::filteredGeoData) + .collect(Collectors.toList()); - @Override - public void onFailure(final Exception e) { - handler.accept(null, e); - } - }; + if (geoDataList.isEmpty() == false) { + ingestDocument.setFieldValue(targetField, geoDataList); + } + handler.accept(ingestDocument, null); } @Override diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index 0eca5df7..de69daa6 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -107,7 +107,7 @@ public Collection getSystemIndexDescriptors(Settings sett public Map getProcessors(Processor.Parameters parameters) { this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService()); this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client); - this.ip2GeoCachedDao = new Ip2GeoCachedDao(datasourceDao); + this.ip2GeoCachedDao = new Ip2GeoCachedDao(parameters.ingestService.getClusterService(), datasourceDao, geoIpDataDao); return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) .put(Ip2GeoProcessor.TYPE, new Ip2GeoProcessor.Factory(parameters.ingestService, datasourceDao, geoIpDataDao, ip2GeoCachedDao)) diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java index 8bd84924..f5925135 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java @@ -7,6 +7,7 @@ import lombok.SneakyThrows; +import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.geospatial.GeospatialTestHelper; @@ -27,4 +28,26 @@ public void testStreamInOut_whenValidInput_thenSucceed() { // Verify assertEquals(request.getName(), copiedRequest.getName()); } + + public void testValidate_whenNull_thenError() { + DeleteDatasourceRequest request = new DeleteDatasourceRequest((String) null); + + // Run + ActionRequestValidationException error = request.validate(); + + // Verify + assertNotNull(error.validationErrors()); + assertFalse(error.validationErrors().isEmpty()); + } + + public void testValidate_whenBlank_thenError() { + DeleteDatasourceRequest request = new DeleteDatasourceRequest(" "); + + // Run + ActionRequestValidationException error = request.validate(); + + // Verify + assertNotNull(error.validationErrors()); + assertFalse(error.validationErrors().isEmpty()); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java index 09f1b67c..6b61e6e2 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java @@ -5,6 +5,7 @@ package org.opensearch.geospatial.ip2geo.action; +import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.geospatial.GeospatialTestHelper; @@ -40,4 +41,15 @@ public void testStreamInOut_whenNames_thenSucceed() throws Exception { // Verify assertArrayEquals(request.getNames(), copiedRequest.getNames()); } + + public void testValidate_whenNull_thenError() { + GetDatasourceRequest request = new GetDatasourceRequest((String[]) null); + + // Run + ActionRequestValidationException error = request.validate(); + + // Verify + assertNotNull(error.validationErrors()); + assertFalse(error.validationErrors().isEmpty()); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java index b7732a31..581fd3b8 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceTransportActionTests.java @@ -15,6 +15,7 @@ import java.util.List; import org.junit.Before; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; @@ -56,6 +57,18 @@ public void testDoExecute_whenNames_thenSucceed() { verify(datasourceDao).getDatasources(eq(datasourceNames), any(ActionListener.class)); } + public void testDoExecute_whenNull_thenException() { + Task task = mock(Task.class); + GetDatasourceRequest request = new GetDatasourceRequest((String[]) null); + ActionListener listener = mock(ActionListener.class); + + // Run + Exception exception = expectThrows(OpenSearchException.class, () -> action.doExecute(task, request, listener)); + + // Verify + assertTrue(exception.getMessage().contains("should not be null")); + } + public void testNewActionListener_whenOnResponse_thenSucceed() { List datasources = Arrays.asList(randomDatasource(), randomDatasource()); ActionListener actionListener = mock(ActionListener.class); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java index 13872a27..45380aa3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java @@ -19,9 +19,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.Arrays; import java.util.Iterator; -import java.util.List; import java.util.Locale; import java.util.Map; @@ -32,9 +30,7 @@ import org.apache.commons.csv.CSVRecord; import org.apache.lucene.search.TotalHits; import org.junit.Before; -import org.mockito.ArgumentCaptor; import org.opensearch.OpenSearchException; -import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; @@ -108,6 +104,18 @@ public void testCreateDocument_whenBlankValue_thenDoNotAdd() { ); } + @SneakyThrows + public void testCreateDocument_whenFieldsAndValuesLengthDoesNotMatch_thenThrowException() { + String[] names = { "ip", "country", "location", "city" }; + String[] values = { "1.0.0.0/25", "USA", " " }; + + // Run + Exception e = expectThrows(OpenSearchException.class, () -> noOpsGeoIpDataDao.createDocument(names, values)); + + // Verify + assertTrue(e.getMessage().contains("does not match")); + } + public void testGetDatabaseReader() throws Exception { File zipFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.zip").getFile()); DatasourceManifest manifest = new DatasourceManifest( @@ -222,7 +230,7 @@ public void testPutGeoIpData_whenValidInput_thenSucceed() { } } - public void testGetGeoIpData_whenSingleIp_thenSucceed() { + public void testGetGeoIpData_whenDataExist_thenReturnTheData() { String indexName = GeospatialTestHelper.randomLowerCaseString(); String ip = randomIpAddress(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { @@ -247,14 +255,15 @@ public void testGetGeoIpData_whenSingleIp_thenSucceed() { when(response.getHits()).thenReturn(searchHits); return response; }); - ActionListener> listener = mock(ActionListener.class); - verifyingGeoIpDataDao.getGeoIpData(indexName, ip, listener); - ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); - verify(listener).onResponse(captor.capture()); - assertEquals("seattle", captor.getValue().get("city")); + + // Run + Map geoData = verifyingGeoIpDataDao.getGeoIpData(indexName, ip); + + // Verify + assertEquals("seattle", geoData.get("city")); } - public void testGetGeoIpData_whenMultiIps_thenSucceed() { + public void testGetGeoIpData_whenNoData_thenReturnEmpty() { String indexName = GeospatialTestHelper.randomLowerCaseString(); String ip = randomIpAddress(); verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { @@ -262,27 +271,20 @@ public void testGetGeoIpData_whenMultiIps_thenSucceed() { SearchRequest request = (SearchRequest) actionRequest; assertEquals("_local", request.preference()); assertEquals(1, request.source().size()); - assertEquals(QueryBuilders.boolQuery().should(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)), request.source().query()); + assertEquals(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip), request.source().query()); - String data = String.format( - Locale.ROOT, - "{\"%s\":\"1.0.0.1/16\",\"%s\":{\"city\":\"seattle\"}}", - IP_RANGE_FIELD_NAME, - DATA_FIELD_NAME - ); - SearchHit searchHit = new SearchHit(1); - searchHit.sourceRef(BytesReference.fromByteBuffer(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)))); - SearchHit[] searchHitArray = { searchHit }; - SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(1l, TotalHits.Relation.EQUAL_TO), 1); + SearchHit[] searchHitArray = {}; + SearchHits searchHits = new SearchHits(searchHitArray, new TotalHits(0l, TotalHits.Relation.EQUAL_TO), 0); SearchResponse response = mock(SearchResponse.class); when(response.getHits()).thenReturn(searchHits); return response; }); - ActionListener>> listener = mock(ActionListener.class); - verifyingGeoIpDataDao.getGeoIpData(indexName, Arrays.asList(ip), listener); - ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class); - verify(listener).onResponse(captor.capture()); - assertEquals("seattle", captor.getValue().get(0).get("city")); + + // Run + Map geoData = verifyingGeoIpDataDao.getGeoIpData(indexName, ip); + + // Verify + assertTrue(geoData.isEmpty()); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java index 4906fba9..c02a9cd1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import lombok.SneakyThrows; @@ -24,6 +25,7 @@ import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.ShardId; @@ -32,7 +34,7 @@ public class Ip2GeoCachedDaoTests extends Ip2GeoTestCase { @Before public void init() { - ip2GeoCachedDao = new Ip2GeoCachedDao(datasourceDao); + ip2GeoCachedDao = new Ip2GeoCachedDao(clusterService, datasourceDao, geoIpDataDao); } public void testGetIndexName_whenCalled_thenReturnIndexName() { @@ -46,6 +48,16 @@ public void testGetIndexName_whenCalled_thenReturnIndexName() { assertEquals(datasource.currentIndexName(), indexName); } + public void testGetIndexName_whenIndexNotFound_thenReturnNull() { + when(datasourceDao.getAllDatasources()).thenThrow(new IndexNotFoundException("not found")); + + // Run + String indexName = ip2GeoCachedDao.getIndexName(GeospatialTestHelper.randomLowerCaseString()); + + // Verify + assertNull(indexName); + } + public void testIsExpired_whenExpired_thenReturnTrue() { Datasource datasource = randomDatasource(); datasource.getUpdateStats().setLastSucceededAt(Instant.MIN); @@ -106,6 +118,19 @@ public void testGetState_whenCalled_thenReturnState() { assertEquals(datasource.getState(), state); } + public void testGetGeoData_whenCalled_thenReturnGeoData() { + Datasource datasource = randomDatasource(); + String ip = NetworkAddress.format(randomIp(false)); + Map expectedGeoData = Map.of("city", "Seattle"); + when(geoIpDataDao.getGeoIpData(datasource.currentIndexName(), ip)).thenReturn(expectedGeoData); + + // Run + Map geoData = ip2GeoCachedDao.getGeoData(datasource.currentIndexName(), ip); + + // Verify + assertEquals(expectedGeoData, geoData); + } + @SneakyThrows public void testPostIndex_whenFailed_thenNoUpdate() { when(datasourceDao.getAllDatasources()).thenReturn(Arrays.asList()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java index 1800cdb6..d4c460c4 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceRunnerTests.java @@ -41,6 +41,10 @@ public void init() { .initialize(clusterService, datasourceUpdateService, ip2GeoExecutor, datasourceDao, ip2GeoLockService); } + public void testGetJobRunnerInstance_whenCalledAgain_thenReturnSameInstance() { + assertTrue(DatasourceRunner.getJobRunnerInstance() == DatasourceRunner.getJobRunnerInstance()); + } + public void testRunJob_whenInvalidClass_thenThrowException() { JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); String jobIndexName = randomLowerCaseString(); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java index 700fe56b..82233c66 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorTests.java @@ -6,10 +6,10 @@ package org.opensearch.geospatial.ip2geo.processor; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -24,7 +24,6 @@ import org.junit.Before; import org.mockito.ArgumentCaptor; -import org.opensearch.action.ActionListener; import org.opensearch.common.Randomness; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -34,8 +33,6 @@ public class Ip2GeoProcessorTests extends Ip2GeoTestCase { private static final String DEFAULT_TARGET_FIELD = "ip2geo"; - private static final String CONFIG_DATASOURCE_KEY = "datasource"; - private static final String CONFIG_FIELD_KEY = "field"; private static final List SUPPORTED_FIELDS = Arrays.asList("city", "country"); private Ip2GeoProcessor.Factory factory; @@ -112,134 +109,136 @@ public void testExecute_whenNoDatasource_thenNotExistError() { public void testExecute_whenExpired_thenExpiredMsg() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); + BiConsumer handler = mock(BiConsumer.class); - Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); - - String index = GeospatialTestHelper.randomLowerCaseString(); - when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(index); + String indexName = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); - when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(true); when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(true); + Map geoData = Map.of("city", "Seattle", "country", "USA"); + when(ip2GeoCachedDao.getGeoData(eq(indexName), any())).thenReturn(geoData); - BiConsumer handler = mock(BiConsumer.class); + // Run for single ip + String ip = randomIpAddress(); + IngestDocument documentWithIp = createDocument(ip); + processor.execute(documentWithIp, handler); - // Run - processor.execute(document, handler); + // Verify + verify(handler).accept(documentWithIp, null); + assertEquals("ip2geo_data_expired", documentWithIp.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); + + // Run for multi ips + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + IngestDocument documentWithIps = createDocument(ips); + processor.execute(documentWithIps, handler); // Verify - verify(handler).accept(document, null); - assertEquals("ip2geo_data_expired", document.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); + verify(handler).accept(documentWithIps, null); + assertEquals("ip2geo_data_expired", documentWithIp.getFieldValue(DEFAULT_TARGET_FIELD + ".error", String.class)); } @SneakyThrows - public void testExecuteInternal_whenSingleIp_thenGetDatasourceIsCalled() { + public void testExecute_whenNotAvailable_thenException() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); String indexName = GeospatialTestHelper.randomLowerCaseString(); when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); - when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.CREATE_FAILED); when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); + Map geoData = Map.of("city", "Seattle", "country", "USA"); + when(ip2GeoCachedDao.getGeoData(eq(indexName), any())).thenReturn(geoData); - // Run - processor.executeInternal(document, handler, ip); + // Run for single ip + String ip = randomIpAddress(); + IngestDocument documentWithIp = createDocument(ip); + processor.execute(documentWithIp, handler); + + // Run for multi ips + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + IngestDocument documentWithIps = createDocument(ips); + processor.execute(documentWithIps, handler); // Verify - verify(geoIpDataDao).getGeoIpData(anyString(), anyString(), any(ActionListener.class)); + ArgumentCaptor captor = ArgumentCaptor.forClass(IllegalStateException.class); + verify(handler, times(2)).accept(isNull(), captor.capture()); + assertTrue(captor.getAllValues().stream().allMatch(e -> e.getMessage().contains("not in an available state"))); } @SneakyThrows - public void testGetSingleGeoIpDataListener_whenNoPropertySet_thenAddAllProperties() { + public void testExecute_whenCalled_thenGeoIpDataIsAdded() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); - Map geoIpData = Map.of("city", "Seattle", "country", "USA"); - // Run - processor.getSingleGeoIpDataListener(document, handler).onResponse(geoIpData); - - // Verify - assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".city", String.class)); - assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD + ".country", String.class)); - verify(handler).accept(document, null); - } + String indexName = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); + Map geoData = Map.of("city", "Seattle", "country", "USA"); + when(ip2GeoCachedDao.getGeoData(eq(indexName), any())).thenReturn(geoData); - @SneakyThrows - public void testGetSingleGeoIpDataListener_whenPropertySet_thenAddOnlyTheProperties() { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city"))); - Map source = new HashMap<>(); + // Run for single ip String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); + IngestDocument documentWithIp = createDocument(ip); + processor.execute(documentWithIp, handler); - Map geoIpData = Map.of("city", "Seattle", "country", "USA"); - // Run - processor.getSingleGeoIpDataListener(document, handler).onResponse(geoIpData); + // Verify + assertEquals(geoData.get("city"), documentWithIp.getFieldValue("ip2geo.city", String.class)); + assertEquals(geoData.get("country"), documentWithIp.getFieldValue("ip2geo.country", String.class)); + + // Run for multi ips + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + IngestDocument documentWithIps = createDocument(ips); + processor.execute(documentWithIps, handler); // Verify - assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".city", String.class)); - assertFalse(document.hasField(DEFAULT_TARGET_FIELD + ".country")); - verify(handler).accept(document, null); + assertEquals(2, documentWithIps.getFieldValue("ip2geo", List.class).size()); + Map addedValue = (Map) documentWithIps.getFieldValue("ip2geo", List.class).get(0); + assertEquals(geoData.get("city"), addedValue.get("city")); + assertEquals(geoData.get("country"), addedValue.get("country")); } @SneakyThrows - public void testGetMultiGeoIpDataListener_whenNoPropertySet_thenAddAllProperties() { + public void testExecute_whenPropertiesSet_thenFilteredGeoIpDataIsAdded() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); + Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of(Ip2GeoProcessor.CONFIG_PROPERTIES, Arrays.asList("country"))); BiConsumer handler = mock(BiConsumer.class); - Map geoIpData = Map.of("city", "Seattle", "country", "USA"); - // Run - processor.getMultiGeoIpDataListener(document, handler).onResponse(Arrays.asList(geoIpData)); - - // Verify - assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); - assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.city", String.class)); - assertEquals("USA", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.country", String.class)); - verify(handler).accept(document, null); - } + String indexName = GeospatialTestHelper.randomLowerCaseString(); + when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); + when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); + when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); + when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); + Map geoData = Map.of("city", "Seattle", "country", "USA"); + when(ip2GeoCachedDao.getGeoData(eq(indexName), any())).thenReturn(geoData); - @SneakyThrows - public void testGetMultiGeoIpDataListener_whenPropertySet_thenAddOnlyTheProperties() { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Map.of("properties", Arrays.asList("city"))); - Map source = new HashMap<>(); + // Run for single ip String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); - BiConsumer handler = mock(BiConsumer.class); + IngestDocument documentWithIp = createDocument(ip); + processor.execute(documentWithIp, handler); - Map geoIpData = Map.of("city", "Seattle", "country", "USA"); - // Run - processor.getMultiGeoIpDataListener(document, handler).onResponse(Arrays.asList(geoIpData)); + // Verify + assertFalse(documentWithIp.hasField("ip2geo.city")); + assertEquals(geoData.get("country"), documentWithIp.getFieldValue("ip2geo.country", String.class)); + + // Run for multi ips + List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); + IngestDocument documentWithIps = createDocument(ips); + processor.execute(documentWithIps, handler); // Verify - assertEquals(1, document.getFieldValue(DEFAULT_TARGET_FIELD, List.class).size()); - assertEquals("Seattle", document.getFieldValue(DEFAULT_TARGET_FIELD + ".0.city", String.class)); - assertFalse(document.hasField(DEFAULT_TARGET_FIELD + ".0.country")); - verify(handler).accept(document, null); + assertEquals(2, documentWithIps.getFieldValue("ip2geo", List.class).size()); + Map addedValue = (Map) documentWithIps.getFieldValue("ip2geo", List.class).get(0); + assertFalse(addedValue.containsKey("city")); + assertEquals(geoData.get("country"), addedValue.get("country")); } - public void testExecuteNotImplemented() throws Exception { + public void testExecute_whenNoHandler_thenException() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); IngestDocument document = new IngestDocument(Collections.emptyMap(), Collections.emptyMap()); @@ -247,42 +246,22 @@ public void testExecuteNotImplemented() throws Exception { assertTrue(e.getMessage().contains("Not implemented")); } - public void testExecuteInternalNonStringIp() throws Exception { + public void testExecute_whenContainsNonString_thenException() throws Exception { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); List ips = Arrays.asList(randomIpAddress(), 1); Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); - IngestDocument document = new IngestDocument(source, new HashMap<>()); - - BiConsumer handler = mock(BiConsumer.class); - Exception e = expectThrows(IllegalArgumentException.class, () -> processor.executeInternal(document, handler, ips)); - assertTrue(e.getMessage().contains("should only contain strings")); - } - - @SneakyThrows - public void testExecuteInternal_whenMultiIps_thenGetDatasourceIsCalled() { - String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - Ip2GeoProcessor processor = createProcessor(datasourceName, Collections.emptyMap()); - Map source = new HashMap<>(); - String ip = randomIpAddress(); - source.put("ip", ip); + source.put("ip", ips); IngestDocument document = new IngestDocument(source, new HashMap<>()); BiConsumer handler = mock(BiConsumer.class); - List ips = Arrays.asList(randomIpAddress(), randomIpAddress()); - - String indexName = GeospatialTestHelper.randomLowerCaseString(); - when(ip2GeoCachedDao.getIndexName(datasourceName)).thenReturn(indexName); - when(ip2GeoCachedDao.has(datasourceName)).thenReturn(true); - when(ip2GeoCachedDao.getState(datasourceName)).thenReturn(DatasourceState.AVAILABLE); - when(ip2GeoCachedDao.isExpired(datasourceName)).thenReturn(false); // Run - processor.executeInternal(document, handler, ips); + processor.execute(document, handler); // Verify - verify(geoIpDataDao).getGeoIpData(anyString(), anyList(), any(ActionListener.class)); + ArgumentCaptor captor = ArgumentCaptor.forClass(IllegalArgumentException.class); + verify(handler).accept(isNull(), captor.capture()); + assertTrue(captor.getValue().getMessage().contains("should only contain strings")); } private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { @@ -296,8 +275,8 @@ private Ip2GeoProcessor createProcessor(final String datasourceName, final Map config) throws Exception { when(datasourceDao.getDatasource(datasource.getName())).thenReturn(datasource); Map baseConfig = new HashMap<>(); - baseConfig.put(CONFIG_FIELD_KEY, "ip"); - baseConfig.put(CONFIG_DATASOURCE_KEY, datasource.getName()); + baseConfig.put(Ip2GeoProcessor.CONFIG_FIELD, "ip"); + baseConfig.put(Ip2GeoProcessor.CONFIG_DATASOURCE, datasource.getName()); baseConfig.putAll(config); return factory.create( @@ -307,4 +286,16 @@ private Ip2GeoProcessor createProcessor(final Datasource datasource, final Map source = new HashMap<>(); + source.put("ip", ip); + return new IngestDocument(source, new HashMap<>()); + } + + private IngestDocument createDocument(List ips) { + Map source = new HashMap<>(); + source.put("ip", ips); + return new IngestDocument(source, new HashMap<>()); + } } From 04e6d6de1d0f700b4f45e1402af10ae4e95d22da Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 30 Jun 2023 10:00:35 -0700 Subject: [PATCH 59/61] Use _primary in query preference and few changes (#347) 1. Use _primary preference to get datasource metadata so that it can read the latest data. RefreshPolicy.IMMEDIATE won't refresh replica shards immediately according to #346 2. Update datasource metadata index mapping 3. Move batch size from static value to setting Signed-off-by: Heemin Kim --- .../ip2geo/common/Ip2GeoSettings.java | 13 +- .../geospatial/ip2geo/dao/DatasourceDao.java | 9 +- .../geospatial/ip2geo/dao/GeoIpDataDao.java | 10 +- .../resources/mappings/ip2geo_datasource.json | 149 ++++++++++++------ .../ip2geo/dao/DatasourceDaoTests.java | 5 + .../ip2geo/dao/GeoIpDataDaoTests.java | 5 +- 6 files changed, 135 insertions(+), 56 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index 0c95fc92..16aba0e1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -40,6 +40,17 @@ public class Ip2GeoSettings { Setting.Property.Dynamic ); + /** + * Bulk size for indexing GeoIP data + */ + public static final Setting BATCH_SIZE = Setting.intSetting( + "plugins.geospatial.ip2geo.datasource.batch_size", + 10000, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * Timeout value for Ip2Geo processor */ @@ -67,7 +78,7 @@ public class Ip2GeoSettings { * @return a list of all settings for Ip2Geo feature */ public static final List> settings() { - return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, TIMEOUT, CACHE_SIZE); + return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java index ec8e6f81..fd6ad995 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java @@ -38,6 +38,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; @@ -156,6 +157,7 @@ private IndexRequest toIndexRequest(Datasource datasource) { indexRequest.index(DatasourceExtension.JOB_INDEX_NAME); indexRequest.id(datasource.getName()); indexRequest.opType(DocWriteRequest.OpType.INDEX); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); indexRequest.source(datasource.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); return indexRequest; } catch (IOException e) { @@ -215,7 +217,7 @@ public void deleteDatasource(final Datasource datasource) { * @throws IOException exception */ public Datasource getDatasource(final String name) throws IOException { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name).preference(Preference.PRIMARY.type()); GetResponse response; try { response = StashedThreadContext.run(client, () -> client.get(request).actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT))); @@ -242,7 +244,7 @@ public Datasource getDatasource(final String name) throws IOException { * @param actionListener the action listener */ public void getDatasource(final String name, final ActionListener actionListener) { - GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name); + GetRequest request = new GetRequest(DatasourceExtension.JOB_INDEX_NAME, name).preference(Preference.PRIMARY.type()); StashedThreadContext.run(client, () -> client.get(request, new ActionListener<>() { @Override public void onResponse(final GetResponse response) { @@ -280,6 +282,7 @@ public void getDatasources(final String[] names, final ActionListener client.prepareMultiGet() .add(DatasourceExtension.JOB_INDEX_NAME, names) + .setPreference(Preference.PRIMARY.type()) .execute(createGetDataSourceQueryActionLister(MultiGetResponse.class, actionListener)) ); } @@ -293,6 +296,7 @@ public void getAllDatasources(final ActionListener> actionListe client, () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) .setQuery(QueryBuilders.matchAllQuery()) + .setPreference(Preference.PRIMARY.type()) .setSize(MAX_SIZE) .execute(createGetDataSourceQueryActionLister(SearchResponse.class, actionListener)) ); @@ -306,6 +310,7 @@ public List getAllDatasources() { client, () -> client.prepareSearch(DatasourceExtension.JOB_INDEX_NAME) .setQuery(QueryBuilders.matchAllQuery()) + .setPreference(Preference.PRIMARY.type()) .setSize(MAX_SIZE) .execute() .actionGet(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java index a930781c..a538e813 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java @@ -47,6 +47,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.Requests; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.ClusterSettings; @@ -68,7 +69,6 @@ */ @Log4j2 public class GeoIpDataDao { - public static final int BUNDLE_SIZE = 128; private static final String IP_RANGE_FIELD_NAME = "_cidr"; private static final String DATA_FIELD_NAME = "_data"; private static final Map INDEX_SETTING_TO_CREATE = Map.of( @@ -248,7 +248,7 @@ public Map getGeoIpData(final String indexName, final String ip) () -> client.prepareSearch(indexName) .setSize(1) .setQuery(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip)) - .setPreference("_local") + .setPreference(Preference.LOCAL.type()) .setRequestCache(true) .get(clusterSettings.get(Ip2GeoSettings.TIMEOUT)) ); @@ -277,9 +277,10 @@ public void putGeoIpData( @NonNull final Runnable renewLock ) throws IOException { TimeValue timeout = clusterSettings.get(Ip2GeoSettings.TIMEOUT); + Integer batchSize = clusterSettings.get(Ip2GeoSettings.BATCH_SIZE); final BulkRequest bulkRequest = new BulkRequest(); Queue requests = new LinkedList<>(); - for (int i = 0; i < BUNDLE_SIZE; i++) { + for (int i = 0; i < batchSize; i++) { requests.add(Requests.indexRequest(indexName)); } while (iterator.hasNext()) { @@ -289,7 +290,7 @@ public void putGeoIpData( indexRequest.source(document); indexRequest.id(record.get(0)); bulkRequest.add(indexRequest); - if (iterator.hasNext() == false || bulkRequest.requests().size() == BUNDLE_SIZE) { + if (iterator.hasNext() == false || bulkRequest.requests().size() == batchSize) { BulkResponse response = StashedThreadContext.run(client, () -> client.bulk(bulkRequest).actionGet(timeout)); if (response.hasFailures()) { throw new OpenSearchException( @@ -304,6 +305,7 @@ public void putGeoIpData( renewLock.run(); } freezeIndex(indexName); + } public void deleteIp2GeoDataIndex(final String index) { diff --git a/src/main/resources/mappings/ip2geo_datasource.json b/src/main/resources/mappings/ip2geo_datasource.json index 3f3d5aa1..567052d6 100644 --- a/src/main/resources/mappings/ip2geo_datasource.json +++ b/src/main/resources/mappings/ip2geo_datasource.json @@ -1,75 +1,130 @@ { - "properties" : { - "database" : { - "properties" : { - "fields" : { - "type" : "text" + "properties": { + "database": { + "properties": { + "fields": { + "type": "text" }, - "sha256_hash" : { - "type" : "text" + "provider": { + "type": "text" }, - "provider" : { - "type" : "text" + "sha256_hash": { + "type": "text" }, - "updated_at_in_epoch_millis" : { - "type" : "long" + "updated_at_in_epoch_millis": { + "type": "long" }, - "valid_for_in_days" : { - "type" : "long" + "valid_for_in_days": { + "type": "long" } } }, - "enabled_time" : { - "type" : "long" + "enabled_time": { + "type": "long" }, - "endpoint" : { - "type" : "text" + "endpoint": { + "type": "text" }, - "name" : { - "type" : "text" + "indices": { + "type": "text" }, - "indices" : { - "type" : "text" + "last_update_time": { + "type": "long" }, - "last_update_time" : { - "type" : "long" + "name": { + "type": "text" }, - "schedule" : { - "properties" : { - "interval" : { - "properties" : { - "period" : { - "type" : "long" + "schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "long" }, - "start_time" : { - "type" : "long" + "start_time": { + "type": "long" }, - "unit" : { - "type" : "text" + "unit": { + "type": "text" } } } } }, - "state" : { - "type" : "text" + "state": { + "type": "text" }, - "update_enabled" : { - "type" : "boolean" + "system_schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "unit": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "task": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, - "update_stats" : { - "properties" : { - "last_failed_at_in_epoch_millis" : { - "type" : "long" + "update_enabled": { + "type": "boolean" + }, + "update_stats": { + "properties": { + "last_failed_at_in_epoch_millis": { + "type": "long" }, - "last_processing_time_in_millis" : { - "type" : "long" + "last_processing_time_in_millis": { + "type": "long" }, - "last_skipped_at_in_epoch_millis" : { - "type" : "long" + "last_skipped_at_in_epoch_millis": { + "type": "long" }, - "last_succeeded_at_in_epoch_millis" : { - "type" : "long" + "last_succeeded_at_in_epoch_millis": { + "type": "long" + } + } + }, + "user_schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "long" + }, + "start_time": { + "type": "long" + }, + "unit": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } } } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java index 09e2dd46..5bcdbbd0 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java @@ -39,6 +39,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; +import org.opensearch.cluster.routing.Preference; import org.opensearch.common.Randomness; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; @@ -205,6 +206,7 @@ private Datasource setupClientForGetRequest(final boolean isExist, final Runtime GetRequest request = (GetRequest) actionRequest; assertEquals(datasource.getName(), request.id()); assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.index()); + assertEquals(Preference.PRIMARY.type(), request.preference()); GetResponse response = getMockedGetResponse(isExist ? datasource : null); if (exception != null) { throw exception; @@ -262,6 +264,7 @@ public void testGetDatasources_whenValidInput_thenSucceed() { assertTrue(actionRequest instanceof MultiGetRequest); MultiGetRequest request = (MultiGetRequest) actionRequest; assertEquals(2, request.getItems().size()); + assertEquals(Preference.PRIMARY.type(), request.preference()); for (MultiGetRequest.Item item : request.getItems()) { assertEquals(DatasourceExtension.JOB_INDEX_NAME, item.index()); assertTrue(datasources.stream().filter(datasource -> datasource.getName().equals(item.id())).findAny().isPresent()); @@ -295,6 +298,7 @@ public void testGetAllDatasources_whenAsynchronous_thenSucceed() { assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.indices()[0]); assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); assertEquals(1000, request.source().size()); + assertEquals(Preference.PRIMARY.type(), request.preference()); SearchResponse response = mock(SearchResponse.class); when(response.getHits()).thenReturn(searchHits); @@ -322,6 +326,7 @@ public void testGetAllDatasources_whenSynchronous_thenSucceed() { assertEquals(DatasourceExtension.JOB_INDEX_NAME, request.indices()[0]); assertEquals(QueryBuilders.matchAllQuery(), request.source().query()); assertEquals(1000, request.source().size()); + assertEquals(Preference.PRIMARY.type(), request.preference()); SearchResponse response = mock(SearchResponse.class); when(response.getHits()).thenReturn(searchHits); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java index 45380aa3..0caa18d0 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java @@ -41,6 +41,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.routing.Preference; import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.bytes.BytesReference; @@ -236,7 +237,7 @@ public void testGetGeoIpData_whenDataExist_thenReturnTheData() { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assert actionRequest instanceof SearchRequest; SearchRequest request = (SearchRequest) actionRequest; - assertEquals("_local", request.preference()); + assertEquals(Preference.LOCAL.type(), request.preference()); assertEquals(1, request.source().size()); assertEquals(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip), request.source().query()); @@ -269,7 +270,7 @@ public void testGetGeoIpData_whenNoData_thenReturnEmpty() { verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { assert actionRequest instanceof SearchRequest; SearchRequest request = (SearchRequest) actionRequest; - assertEquals("_local", request.preference()); + assertEquals(Preference.LOCAL.type(), request.preference()); assertEquals(1, request.source().size()); assertEquals(QueryBuilders.termQuery(IP_RANGE_FIELD_NAME, ip), request.source().query()); From 90f3b7c12c9c568bafc773a0c48540411350e9b4 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 5 Jul 2023 11:10:10 -0700 Subject: [PATCH 60/61] Wait until GeoIP data to be replicated to all data nodes (#348) Signed-off-by: Heemin Kim --- .../jobscheduler/DatasourceUpdateService.java | 29 ++++++++++++++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 4 ++- .../DatasourceUpdateServiceTests.java | 33 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index f14dd1f1..2abf8a79 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -22,6 +22,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; @@ -30,6 +31,8 @@ @Log4j2 public class DatasourceUpdateService { + private static final int SLEEP_TIME_IN_MILLIS = 5000; // 5 seconds + private static final int MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS = 10 * 60 * 60 * 1000; // 10 hours private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final DatasourceDao datasourceDao; @@ -86,10 +89,36 @@ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable geoIpDataDao.putGeoIpData(indexName, header, reader.iterator(), renewLock); } + waitUntilAllShardsStarted(indexName, MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS); Instant endTime = Instant.now(); updateDatasourceAsSucceeded(indexName, datasource, manifest, fieldsToStore, startTime, endTime); } + /** + * We wait until all shards are ready to serve search requests before updating datasource metadata to + * point to a new index so that there won't be latency degradation during GeoIP data update + * + * @param indexName the indexName + */ + @VisibleForTesting + protected void waitUntilAllShardsStarted(final String indexName, final int timeout) { + Instant start = Instant.now(); + try { + while (Instant.now().toEpochMilli() - start.toEpochMilli() < timeout) { + if (clusterService.state().routingTable().allShards(indexName).stream().allMatch(shard -> shard.started())) { + return; + } + Thread.sleep(SLEEP_TIME_IN_MILLIS); + } + throw new OpenSearchException( + "index[{}] replication did not complete after {} millis", + MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS + ); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + /** * Return header fields of geo data with given url of a manifest file * diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index e59e2c35..7bf2961f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -96,6 +96,8 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected Ip2GeoLockService ip2GeoLockService; @Mock protected Ip2GeoProcessorDao ip2GeoProcessorDao; + @Mock + protected RoutingTable routingTable; protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; @@ -119,7 +121,7 @@ public void prepareIp2GeoTestCase() { when(clusterService.state()).thenReturn(clusterState); when(clusterState.metadata()).thenReturn(metadata); when(clusterState.getMetadata()).thenReturn(metadata); - when(clusterState.routingTable()).thenReturn(RoutingTable.EMPTY_ROUTING_TABLE); + when(clusterState.routingTable()).thenReturn(routingTable); when(ip2GeoExecutor.forDatasourceUpdate()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); when(ingestService.getClusterService()).thenReturn(clusterService); when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 80a3beac..5e08400b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -6,6 +6,7 @@ package org.opensearch.geospatial.ip2geo.jobscheduler; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; @@ -28,6 +29,7 @@ import org.apache.commons.csv.CSVParser; import org.junit.Before; import org.opensearch.OpenSearchException; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.SuppressForbidden; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; @@ -134,6 +136,9 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { File sampleFile = new File(this.getClass().getClassLoader().getResource("ip2geo/sample_valid.csv").getFile()); when(geoIpDataDao.getDatabaseReader(any())).thenReturn(CSVParser.parse(sampleFile, StandardCharsets.UTF_8, CSVFormat.RFC4180)); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(true); + when(routingTable.allShards(anyString())).thenReturn(Arrays.asList(shardRouting)); Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); @@ -158,6 +163,34 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { verify(geoIpDataDao).putGeoIpData(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class)); } + public void testWaitUntilAllShardsStarted_whenTimedOut_thenThrowException() { + String indexName = GeospatialTestHelper.randomLowerCaseString(); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(false); + when(routingTable.allShards(indexName)).thenReturn(Arrays.asList(shardRouting)); + + // Run + Exception e = expectThrows(OpenSearchException.class, () -> datasourceUpdateService.waitUntilAllShardsStarted(indexName, 10)); + + // Verify + assertTrue(e.getMessage().contains("did not complete")); + } + + @SneakyThrows + public void testWaitUntilAllShardsStarted_whenInterrupted_thenThrowException() { + String indexName = GeospatialTestHelper.randomLowerCaseString(); + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(false); + when(routingTable.allShards(indexName)).thenReturn(Arrays.asList(shardRouting)); + + // Run + Thread.currentThread().interrupt(); + Exception e = expectThrows(RuntimeException.class, () -> datasourceUpdateService.waitUntilAllShardsStarted(indexName, 10)); + + // Verify + assertEquals(InterruptedException.class, e.getCause().getClass()); + } + @SneakyThrows public void testGetHeaderFields_whenValidInput_thenReturnCorrectValue() { File manifestFile = new File(this.getClass().getClassLoader().getResource("ip2geo/manifest.json").getFile()); From c0987cb802d56a08c9e476460b5d296e7727310e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 14 Jul 2023 10:54:58 -0700 Subject: [PATCH 61/61] Update packages according to a change in OpenSearch core (#354) * Update packages according to a change in OpenSearch core Signed-off-by: Heemin Kim * Update packages according to a change in OpenSearch core (#353) Signed-off-by: Heemin Kim --------- Signed-off-by: Heemin Kim --- .../exceptions/ConcurrentModificationException.java | 4 ++-- .../exceptions/IncompatibleDatasourceException.java | 4 ++-- .../geospatial/exceptions/ResourceInUseException.java | 4 ++-- .../geospatial/ip2geo/action/DeleteDatasourceRequest.java | 4 ++-- .../geospatial/ip2geo/action/GetDatasourceRequest.java | 4 ++-- .../geospatial/ip2geo/action/GetDatasourceResponse.java | 4 ++-- .../geospatial/ip2geo/action/PutDatasourceRequest.java | 4 ++-- .../geospatial/ip2geo/action/UpdateDatasourceRequest.java | 4 ++-- .../org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java | 4 ++-- .../opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java | 2 +- .../geospatial/ip2geo/jobscheduler/Datasource.java | 6 +++--- .../exceptions/ConcurrentModificationExceptionTests.java | 4 ++-- .../exceptions/IncompatibleDatasourceExceptionTests.java | 4 ++-- .../geospatial/exceptions/ResourceInUseExceptionTests.java | 4 ++-- .../ip2geo/action/DeleteDatasourceRequestTests.java | 2 +- .../geospatial/ip2geo/action/GetDatasourceRequestTests.java | 2 +- .../ip2geo/action/GetDatasourceResponseTests.java | 2 +- .../geospatial/ip2geo/action/PutDatasourceRequestTests.java | 2 +- .../ip2geo/action/RestPutDatasourceHandlerTests.java | 2 +- .../ip2geo/action/RestUpdateDatasourceHandlerTests.java | 2 +- .../geospatial/ip2geo/action/UpdateDatasourceIT.java | 2 +- .../ip2geo/action/UpdateDatasourceRequestTests.java | 2 +- .../geospatial/ip2geo/common/Ip2GeoLockServiceTests.java | 2 +- .../geospatial/ip2geo/dao/DatasourceDaoTests.java | 4 ++-- .../opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java | 2 +- .../geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java | 4 ++-- .../geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java | 2 +- .../geospatial/ip2geo/processor/Ip2GeoProcessorIT.java | 2 +- .../opensearch/geospatial/plugin/GeospatialPluginTests.java | 2 +- 29 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java b/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java index 579f22f4..f3a2ae11 100644 --- a/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java +++ b/src/main/java/org/opensearch/geospatial/exceptions/ConcurrentModificationException.java @@ -8,8 +8,8 @@ import java.io.IOException; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; /** * General ConcurrentModificationException corresponding to the {@link RestStatus#BAD_REQUEST} status code diff --git a/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java b/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java index d4f00c02..ce30d5c0 100644 --- a/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java +++ b/src/main/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceException.java @@ -8,8 +8,8 @@ import java.io.IOException; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; /** * IncompatibleDatasourceException corresponding to the {@link RestStatus#BAD_REQUEST} status code diff --git a/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java b/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java index c606abea..d102bb9d 100644 --- a/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java +++ b/src/main/java/org/opensearch/geospatial/exceptions/ResourceInUseException.java @@ -8,8 +8,8 @@ import java.io.IOException; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; /** * Generic ResourceInUseException corresponding to the {@link RestStatus#BAD_REQUEST} status code diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java index f8acbbfb..60bc3f0c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequest.java @@ -13,8 +13,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; /** * GeoIP datasource delete request diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java index 889d7899..3cf94e55 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequest.java @@ -12,8 +12,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; /** * Ip2Geo datasource get request diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java index c2e3cb0c..c6afac0b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponse.java @@ -14,9 +14,9 @@ import lombok.Setter; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java index 9f2335f9..e764f6c4 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequest.java @@ -21,10 +21,10 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java index 45f0132e..2114d6c1 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequest.java @@ -18,10 +18,10 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java index fd6ad995..144add66 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDao.java @@ -40,11 +40,12 @@ import org.opensearch.client.Client; import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; @@ -54,7 +55,6 @@ import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java index dfe44ceb..8c2e686d 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDao.java @@ -23,6 +23,7 @@ import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.annotation.VisibleForTesting; @@ -32,7 +33,6 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexingOperationListener; -import org.opensearch.index.shard.ShardId; /** * Data access object for Datasource and GeoIP data with added caching layer diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index b3d6b328..a256aa27 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -22,10 +22,10 @@ import lombok.Setter; import lombok.ToString; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java index 98ddb3d7..ef57b94f 100644 --- a/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java +++ b/src/test/java/org/opensearch/geospatial/exceptions/ConcurrentModificationExceptionTests.java @@ -7,9 +7,9 @@ import lombok.SneakyThrows; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; public class ConcurrentModificationExceptionTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java index 9d03b10c..69d026a7 100644 --- a/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java +++ b/src/test/java/org/opensearch/geospatial/exceptions/IncompatibleDatasourceExceptionTests.java @@ -7,9 +7,9 @@ import lombok.SneakyThrows; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; public class IncompatibleDatasourceExceptionTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java b/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java index 00465840..f63cdbc0 100644 --- a/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java +++ b/src/test/java/org/opensearch/geospatial/exceptions/ResourceInUseExceptionTests.java @@ -7,9 +7,9 @@ import lombok.SneakyThrows; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; public class ResourceInUseExceptionTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java index f5925135..a3bc17fa 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/DeleteDatasourceRequestTests.java @@ -8,8 +8,8 @@ import lombok.SneakyThrows; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java index 6b61e6e2..7ee19c63 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceRequestTests.java @@ -6,8 +6,8 @@ package org.opensearch.geospatial.ip2geo.action; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java index 7fa21681..f1efad93 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/GetDatasourceResponseTests.java @@ -10,9 +10,9 @@ import java.util.Locale; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java index 2c8bf9f6..b182b3c1 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/PutDatasourceRequestTests.java @@ -12,9 +12,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.Randomness; import org.opensearch.common.Strings; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index f3689a69..ecbba865 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -14,11 +14,11 @@ import org.junit.Before; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; import org.opensearch.rest.RestRequest; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java index 4227e7c6..ef15d030 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java @@ -12,9 +12,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java index 706ea6c5..6c7baa2f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java @@ -16,10 +16,10 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.opensearch.client.ResponseException; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; -import org.opensearch.rest.RestStatus; public class UpdateDatasourceIT extends GeospatialRestTestCase { // Use this value in resource name to avoid name conflict among tests diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java index 36ff271f..c37b6830 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceRequestTests.java @@ -11,9 +11,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.Randomness; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java index adcfae7f..971bec0d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoLockServiceTests.java @@ -17,9 +17,9 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.update.UpdateRequest; import org.opensearch.action.update.UpdateResponse; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; -import org.opensearch.index.shard.ShardId; import org.opensearch.jobscheduler.spi.LockModel; public class Ip2GeoLockServiceTests extends Ip2GeoTestCase { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java index 5bcdbbd0..88329bb3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/DatasourceDaoTests.java @@ -41,8 +41,9 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.routing.Preference; import org.opensearch.common.Randomness; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; @@ -50,7 +51,6 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java index 0caa18d0..8007d4bf 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.Preference; import org.opensearch.common.Strings; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java index c02a9cd1..9ce2f792 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoCachedDaoTests.java @@ -18,16 +18,16 @@ import lombok.SneakyThrows; import org.junit.Before; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.common.DatasourceState; import org.opensearch.geospatial.ip2geo.jobscheduler.Datasource; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.Engine; -import org.opensearch.index.shard.ShardId; public class Ip2GeoCachedDaoTests extends Ip2GeoTestCase { private Ip2GeoCachedDao ip2GeoCachedDao; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java index b298d8ef..9088b0de 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/Ip2GeoProcessorDaoTests.java @@ -15,8 +15,8 @@ import java.util.Map; import org.junit.Before; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoTestCase; import org.opensearch.geospatial.ip2geo.processor.Ip2GeoProcessor; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index a99570ce..d73893f3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -20,11 +20,11 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Randomness; +import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; -import org.opensearch.rest.RestStatus; public class Ip2GeoProcessorIT extends GeospatialRestTestCase { // Use this value in resource name to avoid name conflict among tests diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index d4d5db55..bbe51267 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -26,9 +26,9 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.component.LifecycleComponent; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment;