Update geoip fixture files and utility methods (elastic#111913)

cbuescher · Sep 4, 2024 · dd7286c · dd7286c
1 parent 8e1c468
commit dd7286c
Show file tree

Hide file tree

Showing 15 changed files with 117 additions and 115 deletions.
diff --git a/...st-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java b/...st-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java
@@ -16,17 +16,14 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.StreamsUtils;
 import org.junit.ClassRule;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.List;
 
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
+
 public abstract class AbstractGeoIpIT extends ESIntegTestCase {
     private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;
 
@@ -45,23 +42,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Override
     protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) {
         final Path databasePath = createTempDir();
-        try {
-            Files.createDirectories(databasePath);
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
-                databasePath.resolve("GeoLite2-City.mmdb")
-            );
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
-                databasePath.resolve("GeoLite2-Country.mmdb")
-            );
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
-                databasePath.resolve("GeoLite2-ASN.mmdb")
-            );
-        } catch (final IOException e) {
-            throw new UncheckedIOException(e);
-        }
+        copyDefaultDatabases(databasePath);
         return Settings.builder()
             .put("ingest.geoip.database_path", databasePath)
             .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false)

diff --git a/...-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/...-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
@@ -66,6 +66,7 @@
 import java.util.zip.GZIPInputStream;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.Matchers.anEmptyMap;
@@ -688,12 +689,7 @@ private void setupDatabasesInConfigDirectory() throws Exception {
             .forEach(path -> {
                 try {
                     Files.createDirectories(path);
-                    Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), path.resolve("GeoLite2-City.mmdb"));
-                    Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), path.resolve("GeoLite2-ASN.mmdb"));
-                    Files.copy(
-                        GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"),
-                        path.resolve("GeoLite2-Country.mmdb")
-                    );
+                    copyDefaultDatabases(path);
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }

diff --git a/...st/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/...st/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
@@ -22,10 +22,8 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -34,7 +32,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -68,8 +67,8 @@ public void test() throws Exception {
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
         DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService);
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         lazyLoadReaders(databaseNodeService);
@@ -138,18 +137,14 @@ public void test() throws Exception {
                             assertThat(previous1.current(), equalTo(-1));
                         });
                     } else {
-                        Files.copy(
-                            ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
-                            geoIpTmpDir.resolve("GeoLite2-City.mmdb"),
-                            StandardCopyOption.REPLACE_EXISTING
-                        );
+                        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
                         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
                     }
                     DatabaseReaderLazyLoader previous2 = databaseNodeService.get("GeoLite2-City-Test.mmdb");
-                    InputStream source = ConfigDatabases.class.getResourceAsStream(
-                        i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : "/GeoLite2-City-Test.mmdb"
+                    copyDatabase(
+                        i % 2 == 0 ? "GeoIP2-City-Test.mmdb" : "GeoLite2-City-Test.mmdb",
+                        geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")
                     );
-                    Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING);
                     databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
 
                     DatabaseReaderLazyLoader current1 = databaseNodeService.get("GeoLite2-City.mmdb");
@@ -194,7 +189,7 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI
         throws IOException {
         GeoIpCache cache = new GeoIpCache(0);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
         DatabaseNodeService databaseNodeService = new DatabaseNodeService(
             geoIpTmpDir,
             mock(Client.class),

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java
@@ -20,12 +20,11 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.nio.file.CopyOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -62,8 +61,8 @@ public void testLocalDatabasesEmptyConfig() throws Exception {
 
     public void testDatabasesConfigDir() throws Exception {
         Path configDir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
 
         ConfigDatabases configDatabases = new ConfigDatabases(configDir, new GeoIpCache(0));
         configDatabases.initialize(resourceWatcherService);
@@ -92,9 +91,9 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
             assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country"));
         }
 
-        CopyOption option = StandardCopyOption.REPLACE_EXISTING;
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option);
+        copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
+
         assertBusy(() -> {
             assertThat(configDatabases.getConfigDatabases().size(), equalTo(4));
             DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-ASN.mmdb");
@@ -116,7 +115,8 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
 
     public void testDatabasesUpdateExistingConfDatabase() throws Exception {
         Path configDir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoLite2-City.mmdb", configDir);
+
         GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
         ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
         configDatabases.initialize(resourceWatcherService);
@@ -131,11 +131,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
             assertThat(cache.count(), equalTo(1));
         }
 
-        Files.copy(
-            ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
-            configDir.resolve("GeoLite2-City.mmdb"),
-            StandardCopyOption.REPLACE_EXISTING
-        );
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
         assertBusy(() -> {
             assertThat(configDatabases.getConfigDatabases().size(), equalTo(1));
             assertThat(cache.count(), equalTo(0));
@@ -154,11 +150,9 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
         });
     }
 
-    private static Path prepareConfigDir() throws IOException {
+    private static Path prepareConfigDir() {
         Path dir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb"));
+        copyDefaultDatabases(dir);
         return dir;
     }
 

diff --git a/...s/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/...s/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
@@ -83,7 +83,7 @@
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
-import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
 import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE;
 import static org.hamcrest.Matchers.empty;
@@ -117,10 +117,9 @@ public class DatabaseNodeServiceTests extends ESTestCase {
     @Before
     public void setup() throws IOException {
         final Path geoIpConfigDir = createTempDir();
-        Files.createDirectories(geoIpConfigDir);
         GeoIpCache cache = new GeoIpCache(1000);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
 
         threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName());
         Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build();