diff --git a/build.gradle b/build.gradle index c16c2e563e58a..2b7e241bd2d80 100644 --- a/build.gradle +++ b/build.gradle @@ -516,7 +516,7 @@ subprojects { subprojects { Project subproj -> plugins.withType(TestClustersPlugin).whenPluginAdded { testClusters.all { - systemProperty "geoip.downloader.enabled.default", "false" + systemProperty "ingest.geoip.downloader.enabled.default", "false" } } } diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml index ae4f5ad3e8811..d408cbb36ab21 100644 --- a/distribution/docker/docker-compose.yml +++ b/distribution/docker/docker-compose.yml @@ -16,7 +16,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false - - geoip.downloader.enabled=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true @@ -60,7 +60,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false - - geoip.downloader.enabled=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true diff --git a/docs/build.gradle b/docs/build.gradle index f031e0b6c3121..b7992815acb7e 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -60,6 +60,8 @@ testClusters.integTest { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' + setting 'ingest.geoip.downloader.enabled', 'false' + systemProperty 'es.geoip_v2_feature_flag_enabled', 'true' systemProperty 'es.shutdown_feature_flag_enabled', 'true' keystorePassword 'keystore-password' } diff --git a/docs/reference/ingest/apis/geoip-stats-api.asciidoc b/docs/reference/ingest/apis/geoip-stats-api.asciidoc new file mode 100644 index 0000000000000..6ef0db546342b --- /dev/null +++ b/docs/reference/ingest/apis/geoip-stats-api.asciidoc @@ -0,0 +1,93 @@ +[[geoip-stats-api]] +=== GeoIP stats API +++++ +GeoIP stats +++++ + +Gets download statistics for GeoIP2 databases used with the +<>. + +[source,console] +---- +GET _ingest/geoip/stats +---- + +[[geoip-stats-api-request]] +==== {api-request-title} + +`GET _ingest/geoip/stats` + +[[geoip-stats-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `monitor` or +`manage` <> to use this API. + +* If <> is +disabled, this API returns zero values and an empty `nodes` object. + +[role="child_attributes"] +[[geoip-stats-api-response-body]] +==== {api-response-body-title} + +`stats`:: +(object) +Download statistics for all GeoIP2 databases. ++ +.Properties of `stats` +[%collapsible%open] +==== +`successful_downloads`:: +(integer) +Total number of successful database downloads. + +`failed_downloads`:: +(integer) +Total number of failed database downloads. + +`total_download_time`:: +(integer) +Total milliseconds spent downloading databases. + +`database_count`:: +(integer) +Current number of databases available for use. + +`skipped_updates`:: +(integer) +Total number of database updates skipped. +==== + +`nodes`:: +(object) +Downloaded GeoIP2 databases for each node. ++ +.Properties of `nodes` +[%collapsible%open] +==== +``:: +(object) +Downloaded databases for the node. The field key is the node ID. ++ +.Properties of `` +[%collapsible%open] +===== +`databases`:: +(array of objects) +Downloaded databases for the node. ++ +.Properties of `databases` objects +[%collapsible%open] +====== +`name`:: +(string) +Name of the database. +====== + +`files_in_temp`:: +(array of strings) +Downloaded database files, including related license files. {es} stores these +files in the node's <>: +`$ES_TMPDIR/geoip-databases/`. +===== +==== \ No newline at end of file diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc index c1ad765fcc6bd..090a39e3834a5 100644 --- a/docs/reference/ingest/apis/index.asciidoc +++ b/docs/reference/ingest/apis/index.asciidoc @@ -1,15 +1,29 @@ [[ingest-apis]] == Ingest APIs -The following ingest APIs are available for managing pipelines: +Use ingest APIs to manage tasks and resources related to <> and processors. -* <> to add or update a pipeline -* <> to return a specific pipeline +[[ingest-pipeline-apis]] +=== Ingest pipeline APIs + +Use the following APIs to create, manage, and test ingest pipelines: + +* <> to create or update a pipeline +* <> to retrieve a pipeline configuration * <> to delete a pipeline -* <> to simulate a call to a pipeline +* <> to test a pipeline + +[[ingest-stat-apis]] +=== Stat APIs +Use the following APIs to get statistics about ingest processing: + +* <> to get download statistics for GeoIP2 databases used with +the <>. include::put-pipeline.asciidoc[] -include::get-pipeline.asciidoc[] include::delete-pipeline.asciidoc[] +include::get-pipeline.asciidoc[] +include::geoip-stats-api.asciidoc[] include::simulate-pipeline.asciidoc[] diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 92ed0a09dc79d..bde43310fc046 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -4,21 +4,20 @@ GeoIP ++++ -The `geoip` processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases. -This processor adds this information by default under the `geoip` field. The `geoip` processor can resolve both IPv4 and -IPv6 addresses. - -The `ingest-geoip` module ships by default with the GeoLite2 City, GeoLite2 Country and GeoLite2 ASN GeoIP2 databases from Maxmind made available -under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/ - -The `geoip` processor can run with other city, country and ASN GeoIP2 databases -from Maxmind. The database files must be copied into the `ingest-geoip` config -directory located at `$ES_CONFIG/ingest-geoip`. Custom database files must be -stored uncompressed and the extension must be `-City.mmdb`, `-Country.mmdb`, or -`-ASN.mmdb` to indicate the type of the database. These database files can not -have the same filename as any of the built-in database names. The -`database_file` processor option is used to specify the filename of the custom -database to use for the processor. +The `geoip` processor adds information about the geographical location of an +IPv4 or IPv6 address. + +[[geoip-automatic-updates]] +By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2 +ASN GeoIP2 databases from +http://dev.maxmind.com/geoip/geoip2/geolite2/[MaxMind], shared under the +CCA-ShareAlike 4.0 license. {es} automatically downloads updates for +these databases from the Elastic GeoIP endpoint: +https://geoip.elastic.co/v1/database. To get download statistics for these +updates, use the <>. + +If your cluster can't connect to the Elastic GeoIP endpoint or you want to +manage your own updates, see <>. [[using-ingest-geoip]] ==== Using the `geoip` Processor in a Pipeline @@ -29,7 +28,7 @@ database to use for the processor. |====== | Name | Required | Default | Description | `field` | yes | - | The field to get the ip address from for the geographical lookup. -| `target_field` | no | geoip | The field that will hold the geographical information looked up from the Maxmind database. +| `target_field` | no | geoip | The field that will hold the geographical information looked up from the MaxMind database. | `database_file` | no | GeoLite2-City.mmdb | The database filename referring to a database the module ships with (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb) or a custom database in the `ingest-geoip` config directory. | `properties` | no | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document @@ -304,6 +303,79 @@ GET /my_ip_locations/_search // TESTRESPONSE[s/"took" : 3/"took" : $body.took/] //// +[[manage-geoip-database-updates]] +==== Manage your own GeoIP2 database updates + +If you can't <> your GeoIP2 +databases from the Elastic endpoint, you have a few other options: + +* <> +* <> +* <> + +[[use-proxy-geoip-endpoint]] +**Use a proxy endpoint** + +If you can't connect directly to the Elastic GeoIP endpoint, consider setting up +a secure proxy. You can then specify the proxy endpoint URL in the +<> setting +of each node’s `elasticsearch.yml` file. + +[[use-custom-geoip-endpoint]] +**Use a custom endpoint** + +You can create a service that mimics the Elastic GeoIP endpoint. You can then +get automatic updates from this service. + +. Download your `.mmdb` database files from the +http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site]. + +. Copy your database files to a single directory. + +. From your {es} directory, run: ++ +[source,sh] +---- +./bin/elasticsearch-geoip -s my/source/dir [-t target/directory] +---- + +. Serve the static database files from your directory. For example, you can use +Docker to serve the files from an nginx server: ++ +[source,sh] +---- +docker run -v my/source/dir:/usr/share/nginx/html:ro nginx +---- + +. Specify the service's endpoint URL in the +<> setting +of each node’s `elasticsearch.yml` file. ++ +By default, {es} checks the endpoint for updates every three days. To use +another polling interval, use the <> to set +<>. + +[[manually-update-geoip-databases]] +**Manually update your GeoIP2 databases** + +. Use the <> to set +`ingest.geoip.downloader.enabled` to `false`. This disables automatic updates +that may overwrite your database changes. This also deletes all downloaded +databases. + +. Download your `.mmdb` database files from the +http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site]. ++ +You can also use custom city, country, and ASN `.mmdb` files. These files must +be uncompressed and use the respective `-City.mmdb`, `-Country.mmdb`, or +`-ASN.mmdb` extensions. + +. Copy the database files to `$ES_CONFIG/ingest-geoip`. + +. In your `geoip` processors, configure the `database_file` parameter to use a +custom database file. + [[ingest-geoip-settings]] ===== Node Settings @@ -314,3 +386,28 @@ The `geoip` processor supports the following setting: The maximum number of results that should be cached. Defaults to `1000`. Note that these settings are node settings and apply to all `geoip` processors, i.e. there is one cache for all defined `geoip` processors. + +[[geoip-cluster-settings]] +===== Cluster settings + +[[ingest-geoip-downloader-enabled]] +`ingest.geoip.downloader.enabled`:: +(<>, Boolean) +If `true`, {es} automatically downloads and manages updates for GeoIP2 databases +from the `ingest.geoip.downloader.endpoint`. If `false`, {es} does not download +updates and deletes all downloaded databases. Defaults to `true`. + +[[ingest-geoip-downloader-endpoint]] +`ingest.geoip.downloader.endpoint`:: +(<>, string) +Endpoint URL used to download updates for GeoIP2 databases. Defaults to +`https://geoip.elastic.co/v1/database`. {es} stores downloaded database files in +each node's <> at +`$ES_TMPDIR/geoip-databases/`. + +[[ingest-geoip-downloader-poll-interval]] +`ingest.geoip.downloader.poll.interval`:: +(<>, <>) +How often {es} checks for GeoIP2 database updates at the +`ingest.geoip.downloader.endpoint`. Must be greater than `1d` (one day). Defaults +to `3d` (three days). diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 3a8dd480a854a..41352a8e601f9 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1494,8 +1494,3 @@ See <>. === Rollup API See <>. - -[role="exclude",id="geoip-stats-api"] -=== GeoIP stats API - -coming::[7.x] diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 73a8e44a3280c..cd0d1d7255437 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -58,9 +58,9 @@ public class GeoIpDownloader extends AllocatedPersistentTask { private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class); - public static final Setting POLL_INTERVAL_SETTING = Setting.timeSetting("geoip.downloader.poll.interval", + public static final Setting POLL_INTERVAL_SETTING = Setting.timeSetting("ingest.geoip.downloader.poll.interval", TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope); - public static final Setting ENDPOINT_SETTING = Setting.simpleString("geoip.downloader.endpoint", + public static final Setting ENDPOINT_SETTING = Setting.simpleString("ingest.geoip.downloader.endpoint", "https://geoip.elastic.co/v1/database", Property.NodeScope); public static final String GEOIP_DOWNLOADER = "geoip-downloader"; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index ffafa3d7ca3e0..a2dc5804f566c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -33,12 +33,12 @@ /** * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node. - * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'geoip.downloader.enabled' setting + * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'ingest.geoip.downloader.enabled' setting */ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor implements ClusterStateListener { - private static final boolean ENABLED_DEFAULT = "false".equals(System.getProperty("geoip.downloader.enabled.default")) == false; - public static final Setting ENABLED_SETTING = Setting.boolSetting("geoip.downloader.enabled", ENABLED_DEFAULT, + private static final boolean ENABLED_DEFAULT = "false".equals(System.getProperty("ingest.geoip.downloader.enabled.default")) == false; + public static final Setting ENABLED_SETTING = Setting.boolSetting("ingest.geoip.downloader.enabled", ENABLED_DEFAULT, Setting.Property.Dynamic, Setting.Property.NodeScope); private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index cb961abed7805..d7f4eb5baaa06 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -92,7 +92,8 @@ public static void filterDistros() { @Before public void setupTest() throws IOException { - installation = runContainer(distribution(), builder().envVars(Collections.singletonMap("geoip.downloader.enabled", "false"))); + installation = runContainer(distribution(), builder().envVars( + Collections.singletonMap("ingest.geoip.downloader.enabled", "false"))); tempDir = createTempDir(DockerTests.class.getSimpleName()); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index c46f7eee1cb45..695a6d8901969 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -105,7 +105,7 @@ public void test12InstallDockerDistribution() throws Exception { installation = Docker.runContainer( distribution(), - builder().envVars(Collections.singletonMap("geoip.downloader.enabled", "false")) + builder().envVars(Collections.singletonMap("ingest.geoip.downloader.enabled", "false")) ); try { @@ -310,7 +310,7 @@ public void test60DockerEnvironmentVariablePassword() throws Exception { Map volumes = singletonMap(localKeystoreFile, dockerKeystore); Map envVars = new HashMap<>(); envVars.put("KEYSTORE_PASSWORD", password); - envVars.put("geoip.downloader.enabled", "false"); + envVars.put("ingest.geoip.downloader.enabled", "false"); runContainer(distribution(), builder().volumes(volumes).envVars(envVars)); waitForElasticsearch(installation); ServerUtils.runElasticsearchTests(); @@ -343,7 +343,7 @@ public void test61DockerEnvironmentVariablePasswordFromFile() throws Exception { Map envVars = new HashMap<>(); envVars.put("KEYSTORE_PASSWORD_FILE", "/run/secrets/" + passwordFilename); - envVars.put("geoip.downloader.enabled", "false"); + envVars.put("ingest.geoip.downloader.enabled", "false"); runContainer(distribution(), builder().volumes(volumes).envVars(envVars)); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 669c5c8c69d92..c29a77ceb7180 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -254,10 +254,10 @@ public static String makeRequest(Request request, String username, String passwo } public static void disableGeoIpDownloader(Installation installation) throws IOException { - List yaml = Collections.singletonList("geoip.downloader.enabled: false"); + List yaml = Collections.singletonList("ingest.geoip.downloader.enabled: false"); Path yml = installation.config("elasticsearch.yml"); try (Stream lines = Files.readAllLines(yml).stream()) { - if (lines.noneMatch(s -> s.startsWith("geoip.downloader.enabled"))) { + if (lines.noneMatch(s -> s.startsWith("ingest.geoip.downloader.enabled"))) { Files.write(installation.config("elasticsearch.yml"), yaml, CREATE, APPEND); } } diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml index ed5dd34417426..d2ca0688217f7 100644 --- a/qa/remote-clusters/docker-compose.yml +++ b/qa/remote-clusters/docker-compose.yml @@ -16,7 +16,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false - - geoip.downloader.enabled=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true @@ -70,7 +70,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false - - geoip.downloader.enabled=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true diff --git a/x-pack/test/idp-fixture/docker-compose.yml b/x-pack/test/idp-fixture/docker-compose.yml index 193112e2c21bb..ee9ba763272e3 100644 --- a/x-pack/test/idp-fixture/docker-compose.yml +++ b/x-pack/test/idp-fixture/docker-compose.yml @@ -15,7 +15,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false - - geoip.downloader.enabled=false + - ingest.geoip.downloader.enabled=false - xpack.license.self_generated.type=trial - xpack.security.enabled=true - xpack.security.http.ssl.enabled=true