[Backport 2.x] Add denylist ip config for datasource endpoint

CHANGELOG.md

@@ -15,6 +15,7 @@ See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on ho
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.11...2.x)
 ### Features
+* Add denylist ip config for datasource endpoint ([#573](https://github.com/opensearch-project/geospatial/pull/573))
 ### Enhancements
 ### Bug Fixes
 ### Infrastructure

build.gradle

@@ -164,6 +164,7 @@ dependencies {
     implementation "org.apache.commons:commons-csv:1.10.0"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
+    implementation "com.github.seancfoley:ipaddress:5.4.0"
 }
 
 licenseHeaders.enabled = true

src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java

@@ -17,6 +17,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestToXContentListener;
@@ -38,9 +39,11 @@
 public class RestPutDatasourceHandler extends BaseRestHandler {
     private static final String ACTION_NAME = "ip2geo_datasource_put";
     private final ClusterSettings clusterSettings;
+    private final URLDenyListChecker urlDenyListChecker;
 
-    public RestPutDatasourceHandler(final ClusterSettings clusterSettings) {
+    public RestPutDatasourceHandler(final ClusterSettings clusterSettings, final URLDenyListChecker urlDenyListChecker) {
         this.clusterSettings = clusterSettings;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     @Override
@@ -62,6 +65,9 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No
         if (putDatasourceRequest.getUpdateInterval() == null) {
             putDatasourceRequest.setUpdateInterval(TimeValue.timeValueDays(clusterSettings.get(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL)));
         }
+
+        // Call to validate if URL is in a deny-list or not.
+        urlDenyListChecker.toUrlIfNotInDenyList(putDatasourceRequest.getEndpoint());
         return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel));
     }
 

src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java

@@ -14,6 +14,7 @@
 
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestToXContentListener;
@@ -24,6 +25,12 @@
 public class RestUpdateDatasourceHandler extends BaseRestHandler {
     private static final String ACTION_NAME = "ip2geo_datasource_update";
 
+    private final URLDenyListChecker urlDenyListChecker;
+
+    public RestUpdateDatasourceHandler(final URLDenyListChecker urlDenyListChecker) {
+        this.urlDenyListChecker = urlDenyListChecker;
+    }
+
     @Override
     public String getName() {
         return ACTION_NAME;
@@ -37,6 +44,10 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No
                 UpdateDatasourceRequest.PARSER.parse(parser, updateDatasourceRequest, null);
             }
         }
+        if (updateDatasourceRequest.getEndpoint() != null) {
+            // Call to validate if URL is in a deny-list or not.
+            urlDenyListChecker.toUrlIfNotInDenyList(updateDatasourceRequest.getEndpoint());
+        }
         return channel -> client.executeLocally(
             UpdateDatasourceAction.INSTANCE,
             updateDatasourceRequest,

src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java

@@ -8,7 +8,9 @@
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.unit.TimeValue;
@@ -73,12 +75,23 @@ public class Ip2GeoSettings {
         Setting.Property.Dynamic
     );
 
+    /**
+     * A list of CIDR which will be blocked to be used as datasource endpoint
+     */
+    public static final Setting<List<String>> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting(
+        "plugins.geospatial.ip2geo.datasource.endpoint.denylist",
+        Collections.emptyList(),
+        Function.identity(),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     /**
      * Return all settings of Ip2Geo feature
      * @return a list of all settings for Ip2Geo feature
      */
     public static final List<Setting<?>> settings() {
-        return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE);
+        return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE, DATASOURCE_ENDPOINT_DENYLIST);
     }
 
     /**

src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.geospatial.ip2geo.common;
+
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.List;
+
+import lombok.extern.log4j.Log4j2;
+
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.settings.ClusterSettings;
+
+import inet.ipaddr.IPAddressString;
+
+/**
+ * A class to check url against a deny-list
+ */
+@Log4j2
+public class URLDenyListChecker {
+    private final ClusterSettings clusterSettings;
+
+    public URLDenyListChecker(final ClusterSettings clusterSettings) {
+        this.clusterSettings = clusterSettings;
+    }
+
+    /**
+     * Convert String to URL after verifying the url is not on a deny-list
+     *
+     * @param url value to validate and convert to URL
+     * @return value in URL type
+     */
+    public URL toUrlIfNotInDenyList(final String url) {
+        try {
+            return toUrlIfNotInDenyList(url, clusterSettings.get(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST));
+        } catch (UnknownHostException e) {
+            log.error("Unknown host", e);
+            throw new IllegalArgumentException("host provided in the datasource endpoint is unknown");
+        } catch (MalformedURLException e) {
+            log.error("Malformed URL", e);
+            throw new IllegalArgumentException("URL provided in the datasource endpoint is malformed");
+        }
+    }
+
+    @SuppressForbidden(reason = "Need to connect to http endpoint to read GeoIP database file")
+    private URL toUrlIfNotInDenyList(final String url, final List<String> denyList) throws UnknownHostException, MalformedURLException {
+        URL urlToReturn = new URL(url);
+        if (isInDenyList(new IPAddressString(InetAddress.getByName(urlToReturn.getHost()).getHostAddress()), denyList)) {
+            throw new IllegalArgumentException(
+                "given endpoint is blocked by deny list in cluster setting " + Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey()
+            );
+        }
+        return urlToReturn;
+    }
+
+    private boolean isInDenyList(final IPAddressString url, final List<String> denyList) {
+        return denyList.stream().map(cidr -> new IPAddressString(cidr)).anyMatch(cidr -> cidr.contains(url));
+    }
+}

src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java

@@ -61,6 +61,7 @@
 import org.opensearch.geospatial.constants.IndexSetting;
 import org.opensearch.geospatial.ip2geo.common.DatasourceManifest;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.shared.Constants;
 import org.opensearch.geospatial.shared.StashedThreadContext;
 import org.opensearch.index.query.QueryBuilders;
@@ -91,11 +92,13 @@ public class GeoIpDataDao {
     private final ClusterService clusterService;
     private final ClusterSettings clusterSettings;
     private final Client client;
+    private final URLDenyListChecker urlDenyListChecker;
 
-    public GeoIpDataDao(final ClusterService clusterService, final Client client) {
+    public GeoIpDataDao(final ClusterService clusterService, final Client client, final URLDenyListChecker urlDenyListChecker) {
         this.clusterService = clusterService;
         this.clusterSettings = clusterService.getClusterSettings();
         this.client = client;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     /**
@@ -172,7 +175,7 @@ public CSVParser getDatabaseReader(final DatasourceManifest manifest) {
         SpecialPermission.check();
         return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
             try {
-                URL zipUrl = new URL(manifest.getUrl());
+                URL zipUrl = urlDenyListChecker.toUrlIfNotInDenyList(manifest.getUrl());
                 return internalGetDatabaseReader(manifest, zipUrl.openConnection());
             } catch (IOException e) {
                 throw new OpenSearchException("failed to read geoip data from {}", manifest.getUrl(), e);

src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java

@@ -25,6 +25,7 @@
 import org.opensearch.geospatial.annotation.VisibleForTesting;
 import org.opensearch.geospatial.ip2geo.common.DatasourceManifest;
 import org.opensearch.geospatial.ip2geo.common.DatasourceState;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
@@ -37,16 +38,19 @@ public class DatasourceUpdateService {
     private final ClusterSettings clusterSettings;
     private final DatasourceDao datasourceDao;
     private final GeoIpDataDao geoIpDataDao;
+    private final URLDenyListChecker urlDenyListChecker;
 
     public DatasourceUpdateService(
         final ClusterService clusterService,
         final DatasourceDao datasourceDao,
-        final GeoIpDataDao geoIpDataDao
+        final GeoIpDataDao geoIpDataDao,
+        final URLDenyListChecker urlDenyListChecker
     ) {
         this.clusterService = clusterService;
         this.clusterSettings = clusterService.getClusterSettings();
         this.datasourceDao = datasourceDao;
         this.geoIpDataDao = geoIpDataDao;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     /**
@@ -61,7 +65,7 @@ public DatasourceUpdateService(
      * @throws IOException
      */
     public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws IOException {
-        URL url = new URL(datasource.getEndpoint());
+        URL url = urlDenyListChecker.toUrlIfNotInDenyList(datasource.getEndpoint());
         DatasourceManifest manifest = DatasourceManifest.Builder.build(url);
 
         if (shouldUpdate(datasource, manifest) == false) {

src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java

@@ -54,6 +54,7 @@
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao;
@@ -97,6 +98,7 @@ public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlug
     private Ip2GeoCachedDao ip2GeoCachedDao;
     private DatasourceDao datasourceDao;
     private GeoIpDataDao geoIpDataDao;
+    private URLDenyListChecker urlDenyListChecker;
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
@@ -105,8 +107,9 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+        this.urlDenyListChecker = new URLDenyListChecker(parameters.ingestService.getClusterService().getClusterSettings());
         this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService());
-        this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client);
+        this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client, urlDenyListChecker);
         this.ip2GeoCachedDao = new Ip2GeoCachedDao(parameters.ingestService.getClusterService(), datasourceDao, geoIpDataDao);
         return MapBuilder.<String, Processor.Factory>newMapBuilder()
             .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory())
@@ -153,7 +156,12 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier
     ) {
-        DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao);
+        DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(
+            clusterService,
+            datasourceDao,
+            geoIpDataDao,
+            urlDenyListChecker
+        );
         Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool);
         Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client);
         /**
@@ -187,9 +195,9 @@ public List<RestHandler> getRestHandlers(
         List<RestHandler> geoJsonHandlers = List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction());
 
         List<RestHandler> ip2geoHandlers = List.of(
-            new RestPutDatasourceHandler(clusterSettings),
+            new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker),
             new RestGetDatasourceHandler(),
-            new RestUpdateDatasourceHandler(),
+            new RestUpdateDatasourceHandler(urlDenyListChecker),
             new RestDeleteDatasourceHandler()
         );
 

src/test/java/org/opensearch/geospatial/ClusterSettingHelper.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.geospatial;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir;
+import static org.opensearch.test.NodeRoles.dataNode;
+import static org.opensearch.test.OpenSearchTestCase.getTestTransportPlugin;
+import static org.opensearch.test.OpenSearchTestCase.getTestTransportType;
+import static org.opensearch.test.OpenSearchTestCase.randomLong;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.network.NetworkModule;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.env.Environment;
+import org.opensearch.geospatial.plugin.GeospatialPlugin;
+import org.opensearch.node.MockNode;
+import org.opensearch.node.Node;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.MockHttpTransport;
+
+@SuppressForbidden(reason = "used only for testing")
+public class ClusterSettingHelper {
+    public Node createMockNode(Map<String, Object> configSettings) throws IOException {
+        Path configDir = createTempDir();
+        File configFile = configDir.resolve("opensearch.yml").toFile();
+        FileWriter configFileWriter = new FileWriter(configFile);
+
+        for (Map.Entry<String, Object> setting : configSettings.entrySet()) {
+            configFileWriter.write("\"" + setting.getKey() + "\": " + setting.getValue());
+        }
+        configFileWriter.close();
+        return new MockNode(baseSettings().build(), basePlugins(), configDir, true);
+    }
+
+    private List<Class<? extends Plugin>> basePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>();
+        plugins.add(getTestTransportPlugin());
+        plugins.add(MockHttpTransport.TestPlugin.class);
+        plugins.add(GeospatialPlugin.class);
+        return plugins;
+    }
+
+    private static Settings.Builder baseSettings() {
+        final Path tempDir = createTempDir();
+        return Settings.builder()
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType())
+            .put(dataNode());
+    }
+}

src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java

@@ -47,6 +47,7 @@
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao;
@@ -98,6 +99,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase {
     protected Ip2GeoProcessorDao ip2GeoProcessorDao;
     @Mock
     protected RoutingTable routingTable;
+    protected URLDenyListChecker urlDenyListChecker;
     protected IngestMetadata ingestMetadata;
     protected NoOpNodeClient client;
     protected VerifyingClient verifyingClient;
@@ -115,6 +117,7 @@ public void prepareIp2GeoTestCase() {
         clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings()));
         lockService = new LockService(client, clusterService);
         ingestMetadata = new IngestMetadata(Collections.emptyMap());
+        urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings));
         when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);