From e14e0b023e1c5ba69a34deda4d1e1d9006062361 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 12:57:06 +0100 Subject: [PATCH 01/17] Add support for .tgz files in GeoIPDownloader --- .../org/elasticsearch/common/io/TarEntry.java | 133 ++++++++++++++++ .../common/io/TarInputStream.java | 145 ++++++++++++++++++ .../common/io/TarInputStreamTests.java | 82 ++++++++++ .../org/elasticsearch/common/io/tar1.tar | Bin 0 -> 2048 bytes .../org/elasticsearch/common/io/tar2.tar | Bin 0 -> 3072 bytes .../org/elasticsearch/common/io/tar3.tar | Bin 0 -> 3072 bytes .../ingest/geoip/GeoIpDownloaderIT.java | 35 +++-- .../ingest/geoip/DatabaseRegistry.java | 31 +++- .../ingest/geoip/GeoIpDownloader.java | 13 +- .../ingest/geoip/GeoIpTaskState.java | 45 ++++-- .../ingest/geoip/DatabaseRegistryTests.java | 10 +- .../ingest/geoip/GeoIpDownloaderTests.java | 4 +- .../GeoIpTaskStateSerializationTests.java | 2 +- .../src/main/resources/GeoLite2-ASN.mmdb.gz | Bin 5797 -> 0 bytes .../src/main/resources/GeoLite2-ASN.tgz | Bin 0 -> 6438 bytes .../src/main/resources/data.json | 6 +- 16 files changed, 462 insertions(+), 44 deletions(-) create mode 100644 libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java create mode 100644 libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java create mode 100644 libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java create mode 100644 libs/core/src/test/resources/org/elasticsearch/common/io/tar1.tar create mode 100644 libs/core/src/test/resources/org/elasticsearch/common/io/tar2.tar create mode 100644 libs/core/src/test/resources/org/elasticsearch/common/io/tar3.tar delete mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.mmdb.gz create mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.tgz diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java new file mode 100644 index 0000000000000..ca60d8802d93a --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.io; + +import java.util.Objects; + +public class TarEntry { + + private final String name; + private final int mode; + private final int uid; + private final int gid; + private final long size; + private final long modificationTime; + private final Type type; + private final String linkName; + private final String uname; + private final String gname; + private final int devMajor; + private final int devMinor; + private final boolean ustarFormat; + + public TarEntry(String name, int mode, int uid, int gid, long size, long modificationTime, Type type, String linkName, String uname, + String gname, int devMajor, int devMinor, boolean ustarFormat) { + this.name = name; + this.mode = mode; + this.uid = uid; + this.gid = gid; + this.size = size; + this.modificationTime = modificationTime; + this.type = type; + this.linkName = linkName; + this.uname = uname; + this.gname = gname; + this.devMajor = devMajor; + this.devMinor = devMinor; + this.ustarFormat = ustarFormat; + } + + public String getName() { + return name; + } + + public int getMode() { + return mode; + } + + public int getUid() { + return uid; + } + + public int getGid() { + return gid; + } + + public long getSize() { + return size; + } + + public long getModificationTime() { + return modificationTime; + } + + public Type getType() { + return type; + } + + public String getLinkName() { + return linkName; + } + + public String getUname() { + return uname; + } + + public String getGname() { + return gname; + } + + public int getDevMajor() { + return devMajor; + } + + public int getDevMinor() { + return devMinor; + } + + public boolean isUstarFormat() { + return ustarFormat; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TarEntry tarEntry = (TarEntry) o; + return mode == tarEntry.mode + && uid == tarEntry.uid + && gid == tarEntry.gid + && size == tarEntry.size + && modificationTime == tarEntry.modificationTime + && devMajor == tarEntry.devMajor + && devMinor == tarEntry.devMinor + && ustarFormat == tarEntry.ustarFormat + && Objects.equals(name, tarEntry.name) + && type == tarEntry.type + && Objects.equals(linkName, tarEntry.linkName) + && Objects.equals(uname, tarEntry.uname) + && Objects.equals(gname, tarEntry.gname); + } + + @Override + public int hashCode() { + return Objects.hash(name, mode, uid, gid, size, modificationTime, type, linkName, uname, gname, devMajor, devMinor, ustarFormat); + } + + public enum Type { + FILE, LINK, SYMLINK, CHARACTER_DEVICE, BLOCK_DEVICE, DIRECTORY, FIFO, CONTIGUOUS_FILE; + } + + static Type getType(String type) { + if (type == null || type.isBlank()) { + return Type.FILE; + } + return Type.values()[Integer.parseInt(type)]; + } +} diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java new file mode 100644 index 0000000000000..ee480f17c4880 --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.io; + +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class TarInputStream extends FilterInputStream { + + private TarEntry currentEntry; + private long remaining; + private final byte[] buf = new byte[512]; + + public TarInputStream(InputStream is) { + super(is); + } + + public TarEntry getNextEntry() throws IOException { + if (currentEntry != null) { + //go to the end of the current entry + skipN(remaining); + long reminder = currentEntry.getSize() % 512; + if (reminder != 0) { + skipN(512 - reminder); + } + } + int read = in.readNBytes(buf, 0, 512); + if (read == 0) { + return null; + } + if (read != 512) { + throw new EOFException(); + } + if (bufAllZeroes()) { + return null; + } + String name = getString(0, 100); + int mode = getInt(100); + int uid = getInt(108); + int gid = getInt(116); + remaining = getLong(124); + long mTime = getLong(136); + int checksum = getInt(148); + TarEntry.Type type = TarEntry.getType(getString(156, 1)); + String linkName = getString(157, 100); + boolean ustar = getString(257, 6).equals("ustar"); + String uname = getString(265, 32); + String gname = getString(297, 32); + int devMajor = getInt(329); + int devMinor = getInt(337); + name = getString(345, 155) + name; + if (name.endsWith("/")) { + type = TarEntry.Type.DIRECTORY; + name = name.substring(0, name.length() - 1); + } + ensureConsistency(checksum); + currentEntry = new TarEntry(name, mode, uid, gid, remaining, mTime, type, linkName, uname, gname, devMajor, devMinor, ustar); + if (type == TarEntry.Type.DIRECTORY) { + remaining = 0; + } + return currentEntry; + } + + private void ensureConsistency(int checksum) throws IOException { + for (int i = 0; i < 148; i++) { + checksum -= (buf[i] & 0xff); + } + for (int i = 156; i < 512; i++) { + checksum -= (buf[i] & 0xff); + } + if (checksum != 256) { + throw new IOException("entry header corrupted - checksum mismatch"); + } + } + + @Override + public int available() throws IOException { + return remaining > 0 ? 1 : 0; + } + + @Override + public int read() throws IOException { + if (remaining == 0) { + return -1; + } + remaining--; + return in.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (remaining > 0) { + int read = in.read(b, off, remaining > Integer.MAX_VALUE ? len : (int) Math.min(len, remaining)); + remaining -= read; + return read; + } + return -1; + } + + private String getString(int offset, int maxLen) { + return new String(buf, offset, maxLen).trim(); + } + + private int getInt(int offset) { + String s = getString(offset, 8); + return s.isEmpty() ? 0 : Integer.parseInt(s, 8); + } + + private long getLong(int offset) { + String s = getString(offset, 12); + return s.isEmpty() ? 0 : Long.parseLong(s, 8); + } + + private void skipN(long n) throws IOException { + while (n > 0) { + long skip = in.skip(n); + if (skip < n) { + int read = in.read(); + if (read == -1) { + throw new EOFException(); + } + n--; + } + n -= skip; + } + } + + private boolean bufAllZeroes() { + for (byte b : buf) { + if (b != 0) { + return false; + } + } + return true; + } +} + diff --git a/libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java b/libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java new file mode 100644 index 0000000000000..17f956c6ec168 --- /dev/null +++ b/libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.io; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TarInputStreamTests extends ESTestCase { + + private final String path; + private final List entries; + + public TarInputStreamTests(@Name("path") String path, @Name("entries") List entries) { + this.path = path; + this.entries = entries; + } + + public void test() throws IOException { + try (InputStream is = TarInputStreamTests.class.getResourceAsStream(path); + TarInputStream tis = new TarInputStream(is)) { + assertNotNull(is); + for (Entry entry : entries) { + TarEntry tarEntry = tis.getNextEntry(); + assertEquals(entry.name, tarEntry.getName()); + assertEquals(entry.mtime, tarEntry.getModificationTime()); + assertEquals(entry.data, new String(tis.readAllBytes(), StandardCharsets.UTF_8)); + } + assertNull(tis.getNextEntry()); + } + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + Object[][] entries = new Object[][]{ + createTest("tar1.tar", + new Entry("a.txt", "aaa", 1616416474)), + createTest("tar2.tar", + new Entry("a.txt", "aaa", 1616416474), + new Entry("b.txt", "bbbbbb", 1616416484)), + createTest("tar3.tar", + new Entry("c.txt", Stream.generate(() -> "a").limit(512).collect(Collectors.joining()), 1616416501), + new Entry("b.txt", "bbbbbb", 1616416484)), + }; + return Arrays.asList(entries); + } + + private static Object[] createTest(String name, Entry... entries) { + return new Object[]{name, Arrays.asList(entries)}; + } + + private static class Entry { + String name; + String data; + long mtime; + + private Entry(String name, String data, long mtime) { + this.name = name; + this.data = data; + this.mtime = mtime; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/libs/core/src/test/resources/org/elasticsearch/common/io/tar1.tar b/libs/core/src/test/resources/org/elasticsearch/common/io/tar1.tar new file mode 100644 index 0000000000000000000000000000000000000000..bd1ce119eff80b6164e95225a31fd4d00731d4ca GIT binary patch literal 2048 zcmeH@OAdfA2t<1nPGF`V+w-u{xHSDF9$&QUy3@FT4PhWKJf7-N&AS00hCoJgiu0Y} zdM(lwo@{`N7Rh@@upTZM!n?Nm_Esuqv9>K`Uvev>n=-bpqo(Nnk^US&50k%YG4rqd Ps2L@o1eAah_=dm17|ChQ z>3wE|cGl7nOg0!_={5xoc{@Ez*_)=+ER( sEfE=tkCExCz%+#a)8&8tC=UN~GXM8@3DX%pkPO2VFa=BjQ(z8KL7v# literal 0 HcmV?d00001 diff --git a/libs/core/src/test/resources/org/elasticsearch/common/io/tar3.tar b/libs/core/src/test/resources/org/elasticsearch/common/io/tar3.tar new file mode 100644 index 0000000000000000000000000000000000000000..9cb7e93b81a12034eeaf649f5a4a202aac532771 GIT binary patch literal 3072 zcmeHHNeaU-4D?g{f-sI9`8`QV=&{5$^!c4S(43N67N`b{uq3n$j2=yXXLBq7NC?)@ zM(L_WyZq^b list = Files.list(geoipTmpDir)) { List files = list.map(Path::toString).collect(Collectors.toList()); - assertThat(files, containsInAnyOrder(endsWith("GeoLite2-City.mmdb"), endsWith("GeoLite2-Country.mmdb"), + assertThat(files, hasItems(endsWith("GeoLite2-City.mmdb"), endsWith("GeoLite2-Country.mmdb"), endsWith("GeoLite2-ASN.mmdb"))); } } @@ -266,7 +276,8 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assertThat(simulateResponse.getPipelineId(), equalTo("_id")); assertThat(simulateResponse.getResults().size(), equalTo(1)); SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) simulateResponse.getResults().get(0); - assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Linköping")); + assertThat(result.getFailure(), nullValue()); + assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Tumba")); assertThat(result.getIngestDocument().getFieldValue("ip-asn.organization_name", String.class), equalTo("Bredband2 AB")); assertThat(result.getIngestDocument().getFieldValue("ip-country.country_name", String.class), equalTo("Sweden")); }); @@ -278,7 +289,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assertBusy(() -> { for (Path geoipTmpDir : geoipTmpDirs) { try (Stream list = Files.list(geoipTmpDir)) { - List files = list.map(Path::toString).collect(Collectors.toList()); + List files = list.map(Path::toString).filter(p->p.endsWith(".mmdb")).collect(Collectors.toList()); assertThat(files, empty()); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index c8fa5e1a7af96..54561f051f39d 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -22,6 +22,8 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.TarEntry; +import org.elasticsearch.common.io.TarInputStream; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.query.TermQueryBuilder; @@ -30,6 +32,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.watcher.ResourceWatcherService; +import java.io.BufferedInputStream; import java.io.Closeable; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; @@ -52,6 +55,9 @@ import java.util.function.Consumer; import java.util.zip.GZIPInputStream; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; + /** * A component that is responsible for making the databases maintained by {@link GeoIpDownloader} * available for ingest processors. @@ -87,7 +93,7 @@ final class DatabaseRegistry implements Closeable { DatabaseRegistry(Environment environment, Client client, GeoIpCache cache, Consumer genericExecutor) { this( environment.tmpFile(), - new OriginSettingClient(client, "geoip"), + new OriginSettingClient(client, INGEST_ORIGIN), cache, new LocalDatabases(environment, cache), genericExecutor @@ -128,7 +134,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { @Override public FileVisitResult visitFileFailed(Path file, IOException e) { - if(e instanceof NoSuchFileException == false) { + if (e instanceof NoSuchFileException == false) { LOGGER.warn("can't delete stale file [" + file + "]", e); } return FileVisitResult.CONTINUE; @@ -257,8 +263,25 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta decompress(databaseTmpGzFile, databaseTmpFile); Path databaseFile = geoipTmpDirectory.resolve(databaseName); + if (metadata.isTar()) { + try (TarInputStream is = new TarInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpFile)))) { + TarEntry entry; + while ((entry = is.getNextEntry()) != null) { + if (entry.getType() != TarEntry.Type.FILE) { + continue; + } + String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); + if (name.startsWith(databaseName)) { + Files.copy(is, databaseTmpFile, REPLACE_EXISTING); + } else { + Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), REPLACE_EXISTING); + } + } + } + } + LOGGER.debug("moving database from [{}] to [{}]", databaseTmpFile, databaseFile); - Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, REPLACE_EXISTING); updateDatabase(databaseName, recordedMd5, databaseFile); Files.delete(databaseTmpGzFile); }, @@ -349,7 +372,7 @@ void retrieveDatabase(String databaseName, static void decompress(Path source, Path target) throws IOException { try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(source), 8192)) { - Files.copy(in, target, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, target, REPLACE_EXISTING); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 528c02bdf2973..207d07a6f3f41 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.Setting; @@ -45,6 +46,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; + /** * Main component responsible for downloading new GeoIP databases. * New databases are downloaded in chunks and stored in .geoip_databases index @@ -81,7 +84,7 @@ class GeoIpDownloader extends AllocatedPersistentTask { Map headers) { super(id, type, action, description, parentTask, headers); this.httpClient = httpClient; - this.client = client; + this.client = new OriginSettingClient(client, INGEST_ORIGIN); this.threadPool = threadPool; endpoint = ENDPOINT_SETTING.get(settings); pollInterval = POLL_INTERVAL_SETTING.get(settings); @@ -117,7 +120,8 @@ private List fetchDatabasesOverview() throws IOException { //visible for testing void processDatabase(Map databaseInfo) { - String name = databaseInfo.get("name").toString().replace(".gz", ""); + String originalName = databaseInfo.get("name").toString(); + String name = originalName.replace(".mmdb.gz", "").replace(".tgz", "") + ".mmdb"; String md5 = (String) databaseInfo.get("md5_hash"); if (state.contains(name) && Objects.equals(md5, state.get(name).getMd5())) { updateTimestamp(name, state.get(name)); @@ -129,7 +133,8 @@ void processDatabase(Map databaseInfo) { int firstChunk = state.contains(name) ? state.get(name).getLastChunk() + 1 : 0; int lastChunk = indexChunks(name, is, firstChunk, md5); if (lastChunk > firstChunk) { - state = state.put(name, new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5)); + Metadata metadata = new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5, originalName.endsWith(".tgz")); + state = state.put(name, metadata); updateTaskState(); logger.info("updated geoip database [" + name + "]"); deleteOldChunks(name, firstChunk); @@ -154,7 +159,7 @@ void deleteOldChunks(String name, int firstChunk) { //visible for testing protected void updateTimestamp(String name, Metadata old) { logger.info("geoip database [" + name + "] is up to date, updated timestamp"); - state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5())); + state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5(), false)); updateTaskState(); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index 08297866bf51f..b16c8ad38ba74 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.VersionedNamedWriteable; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { @@ -60,8 +62,7 @@ public static GeoIpTaskState fromXContent(XContentParser parser) throws IOExcept } GeoIpTaskState(StreamInput input) throws IOException { - databases = Collections.unmodifiableMap(input.readMap(StreamInput::readString, - in -> new Metadata(in.readLong(), in.readVInt(), in.readVInt(), in.readString()))); + databases = Collections.unmodifiableMap(input.readMap(StreamInput::readString, Metadata::new)); } public GeoIpTaskState put(String name, Metadata metadata) { @@ -121,31 +122,28 @@ public Version getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeMap(databases, StreamOutput::writeString, (o, v) -> { - o.writeLong(v.lastUpdate); - o.writeVInt(v.firstChunk); - o.writeVInt(v.lastChunk); - o.writeString(v.md5); - }); + out.writeMap(databases, StreamOutput::writeString, (o, v) -> v.writeTo(o)); } - static class Metadata implements ToXContentObject { + static class Metadata implements ToXContentObject, Writeable { static final String NAME = GEOIP_DOWNLOADER + "-metadata"; private static final ParseField LAST_UPDATE = new ParseField("last_update"); private static final ParseField FIRST_CHUNK = new ParseField("first_chunk"); private static final ParseField LAST_CHUNK = new ParseField("last_chunk"); private static final ParseField MD5 = new ParseField("md5"); + private static final ParseField TAR = new ParseField("tar"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, - args -> new Metadata((long) args[0], (int) args[1], (int) args[2], (String) args[3])); + args -> new Metadata((long) args[0], (int) args[1], (int) args[2], (String) args[3], args[4] != null && (boolean) args[4])); static { PARSER.declareLong(constructorArg(), LAST_UPDATE); PARSER.declareInt(constructorArg(), FIRST_CHUNK); PARSER.declareInt(constructorArg(), LAST_CHUNK); PARSER.declareString(constructorArg(), MD5); + PARSER.declareBoolean(optionalConstructorArg(), TAR); } public static Metadata fromXContent(XContentParser parser) { @@ -160,12 +158,18 @@ public static Metadata fromXContent(XContentParser parser) { private final int firstChunk; private final int lastChunk; private final String md5; + private final boolean tar; - Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5) { + Metadata(StreamInput in) throws IOException { + this(in.readLong(), in.readVInt(), in.readVInt(), in.readString(), in.readBoolean()); + } + + Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, boolean tar) { this.lastUpdate = lastUpdate; this.firstChunk = firstChunk; this.lastChunk = lastChunk; this.md5 = Objects.requireNonNull(md5); + this.tar = tar; } public long getLastUpdate() { @@ -184,6 +188,10 @@ public String getMd5() { return md5; } + public boolean isTar() { + return tar; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -192,12 +200,13 @@ public boolean equals(Object o) { return lastUpdate == metadata.lastUpdate && firstChunk == metadata.firstChunk && lastChunk == metadata.lastChunk - && md5.equals(metadata.md5); + && md5.equals(metadata.md5) + && tar == metadata.tar; } @Override public int hashCode() { - return Objects.hash(lastUpdate, firstChunk, lastChunk, md5); + return Objects.hash(lastUpdate, firstChunk, lastChunk, md5, tar); } @Override @@ -208,9 +217,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(FIRST_CHUNK.getPreferredName(), firstChunk); builder.field(LAST_CHUNK.getPreferredName(), lastChunk); builder.field(MD5.getPreferredName(), md5); + builder.field(TAR.getPreferredName(), tar); } builder.endObject(); return builder; } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(lastUpdate); + out.writeVInt(firstChunk); + out.writeVInt(lastChunk); + out.writeString(md5); + out.writeBoolean(tar); + } } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index 8d0aaec86ef99..82ed02fc6031e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -127,7 +127,7 @@ public void testCheckDatabases() throws Exception { String md5 = mockSearches("GeoIP2-City.mmdb", 5, 14); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5, false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -151,7 +151,7 @@ public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Excepti String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -175,7 +175,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Ex String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -216,7 +216,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenGeoIpDownloadTask() throws E public void testRetrieveDatabase() throws Exception { String md5 = mockSearches("_name", 0, 29); - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 29, md5); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 29, md5, false); @SuppressWarnings("unchecked") CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); @@ -234,7 +234,7 @@ public void testRetrieveDatabase() throws Exception { public void testRetrieveDatabaseCorruption() throws Exception { String md5 = mockSearches("_name", 0, 9); String incorrectMd5 = "different"; - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 9, incorrectMd5); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 9, incorrectMd5, false); @SuppressWarnings("unchecked") CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index 375b66cd41d12..c8acdcb3b907f 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -244,13 +244,13 @@ void deleteOldChunks(String name, int firstChunk) { } }; - geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test", new GeoIpTaskState.Metadata(0, 5, 8, "0"))); + geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test", new GeoIpTaskState.Metadata(0, 5, 8, "0", false))); geoIpDownloader.processDatabase(Map.of("name", "test.gz", "url", "a.b/t1", "md5_hash", "1")); } public void testProcessDatabaseSame() throws IOException { - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1"); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1", false); GeoIpTaskState taskState = GeoIpTaskState.EMPTY.put("test", metadata); ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); when(httpClient.get("a.b/t1")).thenReturn(bais); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java index dd5faa9d8fa33..83b87d00fdef4 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java @@ -30,7 +30,7 @@ protected GeoIpTaskState createTestInstance() { GeoIpTaskState state = GeoIpTaskState.EMPTY; int databaseCount = randomInt(20); for (int i = 0; i < databaseCount; i++) { - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32)); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32), false); state = state.put(randomAlphaOfLengthBetween(5, 10), metadata); } return state; diff --git a/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.mmdb.gz b/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.mmdb.gz deleted file mode 100644 index f36a1263264b433827f3e6367db7be80ce19776a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5797 zcmZ8jcQl)A*pH+riWhBFsTrG6yOd~ANsXAbM{AYZwA4!QQq&%`1+_;htvy1uYK9uM zsoJYz)QaSL^!?-e&UemreoyY_xyJAMU3cztbG*5H8SVzN20_&A+%z4%?Ic9+>uZY| z*m-&1c5$(N3VM-F8Nl}CuzUV$kJDStJ2x^z$(U@E87KacM~sT}l^qX~hcXDynwoky z8NP4e<%$92h0ncNS5$rg=Y)PGdmuo@7{w)bY2?B0JPM-QijR+QCgCvQpmn`g(rwOd zo-Kh>sGKJ;T2M6#1>ue{V9}w|1PL|tisU~kC__8tQlwoT1;4;Tt!b<^tc9(gTT4{a zR*Q<0Ptc`NYeMQMpJM5-v1qcF;FlCdkY><4$R5-TmJFUn&ju@^{c_2kUHWtB57?fp zHn=r75bXv&1D}zdK`23#U`jGd$ZOC%xQ#Y|@)7_7s&ir;JT8a12 zM-tk6dn`PveL7y>I)d__phZuf*6J&eF&DPc6F|Q_I_U2V#i2PMFtQ+OHX6beIXeACu65shruY-D zg)TjlC*yX-Oxc~%I3D?wj?$4E?d~Uay`sbW*<@J=@9V$V@@wB&5+iH8w)v0n0ooQJ z{WNHr;5X30Ff!&mAfJ z8xF6$#NpicjUsZ?LQx=`T3k(VN%aU0LW=6O1j&HKz+z-#HjlPKtd@g~*@y$_zZqJu zOQW*aVgwHPcCYWs3@hkPHm_A!3;c#~{1X@<7}!->6@S99V6(uqpuZrw;FT}dAo+5M zMBe7@&a-9z0vB`@j3!c56a%n^aS`ebrlPBhK13Sml1t@+wY95>k(F>w5ygZOUR~xw zJ~A_qncf%jXWgg#MM-IFV>97hb-GdI&dh~H;1TpTLYdS>r<-yMQB5>74T*#Qz#Z}TMEY7rGIHkDZMW36as**ej4>?3`sfQd^EjlsQvYAghGy3y*aV%u& zPPV1dIPAn-v*qWcsWeRTdS7p&4)~_0o$?&O>(2v^`kU zWuE)a*5N68_Bf$~DUca?Ag*U}HnS6HY}bW{0_}c)KIZh|t)@1n4WK_(+J!9K za6RH-tU#q0TZE&Dcz#A07k)&qR|cTfowwqub{XhR2I$Qr-;CL8AG)ygjar*bo7clM z`GkhC*hU{0Aw$q)bcmVX?W}#{)#yx1bzRSc3J4(k!o-}cJ*bw|H1UeidHEiP(Xt+6 z>QCKWA->zI{bmUB;L#MU;c!27_u@^)9}m-iBs8!~yG2O26+CLDmnD;hv|n=d5EyS6 z2o8G;jJqB&Gw2(8i6*}x^ODB<1}15g2Kdw{Cn95BZSlyLqQ&#_xDp-|2`6=#F0eBW z!SMVGb0yTaIy1;c+}hS@$rT9!b|^3qdUd-v5Q?0g7k{Jd9vzUl7_lq-WH}p}=LdOHh-0R_0Xr2U3MGP|IjNpnycAkA=P{h>n@XY{-)E0JV&ULAB#Wfi7!aQ4$74!r;}8KMmP+JG5GBes z_mZ2%%2tWTP6TZYU7;jK9ktqo3Y5gdM60x=ap+xQQWCtTt1l92Lb8>bjRuJGV5 z22Tq-fA@YfO8p@>F)4zy^tez+=F2I6Jx}B@@&RcnxrEyC5$^#~8(3FSd-9o(mfl~a z8bSQ$KA1TgeSi#tr0EL%(z!pL^lyJ3oCH0HD{nMV!7ppn5POz^dP{j1Mfnl2q@|#e zMp;A`iV=@dB{*17y|MUB&!)VG71-xouugofnhqT6T8yXsnfW#Lpi_P`geQT2)q1=o zGlF!?)YGU;DUR!S8Geiesw?XLXjSGgH+rXQu)Q~G@zu|etu9a37Ri? za&=d#=WtjV2f~3#oGf2Zo{(K+Vgw1=jWT3~w}G6!-HYfAt_~jb4JPy$MG-Y?h^3+k z0OS*gU_HRhjrr!GCy%AJrh$R<_`K^s0(GJGAc+aYo;W~4!&dm--xHit_P%x?yYz7g z)RJ*I75GDN5%q^)U#KgD;DXGoLKab14Ow)wojQ+qhr@%*60h#SuI<1q>}karN<=B5 z-=`u9a#7^lIu$4rZk0O0w5Z3wnZ zHUtA0BCwQyi#qud!i8JX1jk{sT2Y6{^Qch)i4oLb4`FiN zoO5*((-eOJXaMB1FCZ51161t>{muHn5O#)p$33l>GfJhqv5Ji>(2kNoQv5?I)koYX zjq;rq%MtZg^Lo<(b+WbosxkI!aN@*(mbK!xG1&p~1F^Imp>|q)oA(PI(?ne=T>u0a zalarpw6Q2-Kp67e(ER~oBN?LL8}gLI!i-z+`=kqErM{Wl^3CZ^jD?1J_3=&Dr1MZ= zsR2L^`>Np7n?T<_AoppYo-t{(uVCltJr$s5#fKZNU(bO__IAVZ6=U*x&smNdnjZ~* zRO)+sPSwENfa_?}xU;ofc`8{-sq=?<;r;^G!jlDt1>lZCCtaRz$xcapxbeGWPu1WL zboXj$v2I?B#67r&yZr9Wqu_zjHAplv7>C9UTU!BKy?Lhs z5gWp(J&dcIAS!FtPZYgqPiwGF@Ovis2jaRY!idKZqIt5Zz_q9A5ZG zs$Hb)#O3?^eG{gkXraw`C-N!uU1bGv<~7g?Sp}-;4PV=h>=aU>{rLz-JOF)g6_V6F zOgPbT4HKR@Of$r?iBSqU;4YvpH<;Mv9hPL%W`e!Z(?D=U3Y}wh(aeS{oQe_s&;^Cj z>sOdgi-C^t7tvi8Pat2_MbgzI0ANts^N}Igz-h5j0^l$X%PuxG*VtGF?Z6Hny*6!W ze>Q`10>XmY-WpkwKyqL6_AY&|K%JU@{~Ltk0wQsI03f-~YZEa)x)YR?9%X}+Ne7Vz zG_*h1NbC9t9-^>z>LC(y|K@2iUfNoeTP@}oi3!uZtRf0dnOaj|4P+ntc@`8SCUOv# z^@NFqHXoyRPXDHf=N^917sCf;9Vzf;z$FjLO9C|Aaq(>bxSPjZ(CzOvyt;% zf!tXmvJhY4;rjT$Iz1#9y=ImKy7TRC>dd%LlG?-sMO*;-z~Yn9}PYFsdwIwsWcKpd?Lm)s4;#ZenIvj@x&(??91S;(AO0{v7jFIcayX)Pq!N z1-6_kcn)a$*EW9e@c4xuWPXcYgq_R$H>MzZUsTe22GV*QXa=AiJ=qR7cQG;Nv8$z;Ps-)g6~+|8BR^GUx(l;y4F{}lN%1qo&L`4)-Q_j)fUirx6&IyKBXi(=Yzebf(6YrvSWUpX}y zY%QMcx;}7dR6sC)pNC^*!%HAW7-M5!tsB-Er%!y3auv%wq{r+ggNrSB0D*_ zf9)SdtknKD$djphLwgqX+A`T+g|7ofRvl`dOleHsjaeD#$o2oWb*ye|v>zGy#q2!b zwu0Q&euE$i$|kL~{gUXGK0$DWF)eAvVayopA3lk{NOI8*5>#QJv~1^`vdvMi(aw2GWR>@_ujH#wW-SKQBK-vrobO$x)jOsTl)%R^ z{sHk1jXRFtg9h!lc9rVIetn>96+w=(!cQ`_?oOqQ{n%~$E5_rp09GevW-$9(+aQz4GH^oax7m%EMKjf1atPX=au z?)9&Rv%b>TFr;t8#j?^7uPLi!ss3DY zF8v%7?vUhNZWSbA^lW2Wf+|N6BQUbD+>(EHn!0vVtHNzRU0#8%0)GT$w5sn+^r~i> zw4w2F$9#OuiYuP36mv1%As39}z_R~btji?#sK1kf%ru^EY0IH=_^iK*&v9>#p6wra z=sD*+d!XRUlX^)-CC|~-d@VZ9Sy@x)r7=|;Bz-Basz>%c?AI`qpAT!yXwz)C2R~(L_{~~bP8%M=<}=V1Icw4*B5eHV zbgQiJbM*QXtvxOMneG1KanVl5Btw*XrwFxV=@RwMOmrc|tY3?DpeM4SvpS@}i_m>e zUm@gM#X@>l=rU&N@>IQ_)^Cep-K@>caV3o3vBHO*q`uSC(rxuK%TTGd_pW$zuXbsL z;+R>hcii`JYvxZexA?0H90&i+ksR`1Z~c(7IaP*x>^at>*fC*zIS!VL{~E6KL1yr$ zoWh+zFU?fh1rg7{_DIpKHN?k(-;Nq-JLDFTv3FfuDvJt#3_Ly8<*VB?bG68^o?Lf! z*%sfi()dkin>*2pjGbdQI&HC-_nokgv&ftcZ1#LNRD$35=q*dC}NV}?Mi#9R#pQu<>WQXQpHJkRV`|z3XAJ0UdiHyF~tdp5r zo@?Bh+5V#Bypt7;W533yc%sOp5YY0Ho_)a0{E3jlV)~-wcy*?w&Yn+zHP}pMe68_& zPj{xV!%yDg(M7``;fSisH73)xe5l^aaLDw))qWQt|H<1v-+Y(-sD))VDISl$n%fxT zg^2!Z>L)E z8lAhz?=*~{v)S;K!yB0B!M2=`6jNa;{gXu6ofalsZj!91Vj^`cJ3gk;)I6rt5&QHrCcP!|y7&%as3l%Jh5(9oRNlIQwQQgVlcf6E5g%X7`)I z>|E~-XSVYa<*V&ovY^`Dij8gsB&o#5J8j*^&HjmB%uI_ET)i;$1>I7pLCrh|4dv@k zTPT}?e1)D~$pyP};!N3{_VQW!=82?1*9El=&C|&aP4N|I;UrRQ_MNd)YW$Hb&1_BO zK))S3Ou=YvEz`vra~SSmO-PwryK&a{;T{K3q+IJ8W7~9#b%oT=J-WHmycXFdhp-jh pmBP;j^ROE$BaJWY^yMX;Rnl!yGRE6glSC3|&iP8tLvax3e*hPLjobhL diff --git a/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.tgz b/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-ASN.tgz new file mode 100644 index 0000000000000000000000000000000000000000..4e92ad982227d57395dce3323a91c8e028738400 GIT binary patch literal 6438 zcmZ8l2Q*w=*B-qE(V`P2glLIgVuEN5CkDnMsK48@#-ad8G;0f5p@V5 zAwfj%W#<0Z`+fgf|N7V3>p64Ioc)~r?7QczeTO@d9FkgRVGkkN$_oy$9-JP+Fb+4u zN6WJv`B)_Gk~&F`%19^Vdd+!+IKyAvecU-aF<8|-a%0q=euDhE$GT{sY8-@uCn|-7 zjJ%jwYdz|E==%LBbg;|!#qLJ!Gr`r1{pTIYHB0*KlWSc88uNeH&KFSg^YiUZuIwM> z?Y+W@vJtvAYF@Z}i=zP7097xW@bGZXoH-O?GTuL@I_r1&;YE%Lv|#kI^m)G5uMvci zeh;@gK1*p@`isrtg{XI4Z&8Ipr@%$~RNu|r*;!rQqJd)KTCal{j^o~Y zl#ZuNC?17DBxW6jb|?9Cne;u>R*nlvX{E`*OJwP`t=lfG-izpWQe)u9IuhTBd+kcv9VipOmS#Ke)abs1 zVG~c)y2weQn0#qBc8YVjch^G+FMfYo`T$pd|I_)AB>R(sX(FP8?}#u?;5mT#r>3f4 z}dNc|0r80RWQ8H-&Sw_*t#!gz(JQ-D4g_8rfen7_#^iKg4T7lSDK` z#1Kho&nrQJCB`yhuVFc|ufYl66mSwawJt$9;WtPIx&<8+Nzg(T1Suy_B(#H+6Dbl% zMixZbMA}3N+Y1;_PeI;4;54B9Ovn~Fhvu*st7orQtS74%tY@y5-Bn-KL1#s26j8X5 zT1J}Lb4hTULz)SwqSPNKjuQkD+B}f0r-eaje>=uFKA^29#xk2ziVz(W9h1>OXoxe3 z$O!I4MiV5@3eYx7>mY8vk`&{8B=Lr4pS3F9jMo(vjeCfL)UB#r&p}P`YnG?TL^6#5 zYBZdBQt{)DON=JEwpinM2i8hCD)xYBbV{_2m6{Ys>y#8J`Qw`cvJiHlV?Khn?DzHzSJ3x4<60>tt*sp zOGZ38WEeVncbt&A#UV+$gG*Xp6Z6-N=v^8}ZDqi{*z+vOV{N^;CY}(%WZL;=Xm$4J z{GG0?f2;qz7kL4tFLm#aPhs)f_LnzO5gJ0l^oy}Z+Zuef>g&PiZghD5LBvUAQqAFx z&0Cz)6Ru^wewhmunYD$LIF_ynZkPJlu$aEzXND2DXC)tZ`?;e&mewk%q@y+jqoA7Z zFaj7627)2P6l=p@*e$;@=fL-;`p|c?bmQs2jvB5rEt@V&m&}$n%Nm$R6nBwy=^fJK z9*V^&tFPKcokX5Q(MHlnCC4lO?#HG8)TaY|)Adc$_b&{|A?Jb;g@osVMTlclL&_w- zdiQEXN&ZL?fwP4Fiz66Z!%?+Cv*g9Vz-_6kx=kM$8wjgLII<-S8;*%;!C7PFMrRD& zC?%M%FNqIfaCYapH^iHEbI-`VV2*K)@jO8s?Up2)C@i~tFrf=X=~3rH=WR-p)NEFD zGo(F^AwatxpJAh0GS*i%jm(byL~0+DE<@(^Hve(FYFHY!ilq_v1jlfZ z0=g9N_i#v;q&_re^th^X`f=#pr5ce2h})f7$%d4HlG{`IC8aEl@)p2DS=GlztUg~G zds9Kb`yo6po1OWK5TM#jUO{o{ub$9UEG-sAyt#y}(dJlR=FLLNk*D4%`4mlniQ+rs zPLN9=es#=gz51vq`5|w=2-E;Y_7p=Y1vFt`m`_k?DcFy#q5$?H z)+ApIJn6ySK+a&n0TE5XmPkYosdw_IzgWV-I1|U<013x4(F7{G4)Rlnsm9BewwQr3 z!&PT~G0ibh;vM(ug1D;4swi%PSvf|3+cmW69dls7lfTwgN&f|j;ezgqTaOpw^PzkM zxI0@3w!H}h;^Sl-^2DB~w&;g%5ssX`R21u|Q)yq%bdr0wy#@E!8H1jsld>`fr$you zI!l|wB9{nG(%NV(w#h)^5B#TEa>p9v)=`mozecKtv)!+Pc01~OyI50}VZvm{zbs5-V3H-ah$vbCjSkN)uZ>V?Xzl2f%FmJQZ zj%w8if6ugCfd3HY&E(uyY@!ORnt<~W9@n%Z*!IIT98~`Gf3|PetBh#@tIUxbgHl0E zPE6XLx{?lIs(@S`)fW+t9{x>~wceGF8VF*%!+2DFVt$iWKY#?;FQH-lB8=VO%|3j; zEr`bPr+Y| z*f(F61Mj0=sQ)_UlW?P{FUTlkt-@E$laZ5Ak|p`$&}TP>FUbq3^BJ7wTg^Y58LpD> zizlfvr^eui6J@<~y3$vjwW2Mx)={eg@m-`AwY=a#_;txTJDq)nw>Y_1mlVGDhfx}- zvJfaA1r8~z;VzqcIY=6PG@mREE?8fh`GaPrsX=y{ZDAhp9e< zyTIxC(Q|W2k~+>41Iwnqp6MJIKLhG<4Kwv{|F2$g%?0Y5#kk-c}A+yn5;?1 zy3jxLMd?%7soSMbiKfks+5nR_ATV_$isFHKpF?1oE-wjg)HBWzXKaF)aZsj~xK7W<^H*HcEx^B7}zm1c3Dc#%DZOl1O~z|8^rZ zff7Os2w~8`Q1C+Qs4Es~G_C@ifQzRH%JT|epc1sAtB=dT)hb5^(>9dp5=w$I{x?1_ z#48)WS{R4MUcf+mPaFB~NOE?y4#0lkjYWxyI4Sq9xc;vdAx4P)<))9MeE=|SLrsBC z6p!Vyg=V0ODxelz_7sHI_yIZi-65RJj3w$-4`-MC8I0@s<(xQS9oyaMDEL|s8X zxDON>NBG}({tXI*vkOdg>H9F^O1{$ZlgLsnPEK_$A~az9G4%z$lKK$V095E}W+K67 zq7VyGr~pH+u}@sb<*NJq&P*XV<7&hoN264HWq>xgGa`Qc>@5~BX66axdHta+DOymP zW-Pvg*dFj%P<$&Eu#Aw3ureBl4xU%k3NN^H_Scs0-5)eGnoa4rAzzF*Q|oaAU`^BL zVht5wiW>5{1G`jba zdXsnu)Vm73?Cvt{<#|dr5ka*;u|Tpwy+CwH(>CjH>1#5AL+@zg2rRhbC%A~Yc#k9| zxP;36_@Ql+s_eW4jh_yolsQ5Z1{EZ zqapl}?o({H2_8k=eZcx^TP6Mo=JP0&L-neXNhIj9iQzx(lP6@FS?lE`Sh0jydMqV2 z)G1l&5cb5!V*znxmv|Wa>H?QN0Fy-)z8rspW|s4p+1Mc88YuRVeH2Q%Q9##DRe#5T z48 zC6pRBxqCH!Ij=wof`GXRLaz$jc||cI_XtLd1nFm>1*MeMI)A1i+t4GnILdrGscon< z=&NYg$5+=&!|_MrSZkshaV|A6enkYkbp<>iBn#jcOA2P=kjlK2oD>g$Lqih+>e;`* z#){Sl;Eq(*5e%gT9<}F*rVM$-hDRy0re1$ZDJRVIbq9X9Aw}1_GkY2vtDMN^a}x}- z8o|~JzH6o^XZf+Z+tz@N7{PD?|9)Vy4am=4Y6XQH!t4}`7_VZ(<;6>)dR~pjC|1;w z`U-!YP5Dov09iURNsw&)_rPVg3Lv$K>PftU=tI4DC){M~ReDcxDEuw=cP+MYM*9pj z0PF(*CP|=Qa(WjC#N!!Uji~|HQbK-+%F0jOCj5q=76_W!AS@w=a1}$iwTIR^9y3tf zb@x-mxyswVzOS@I$?)q5iCP8RHm3+Ac3&@7ldTs36`i|JfTwAwBYk(L+JGsr5BD;- z%I?2*r1OlEFk`X4LCR`DDLs>((T@L-vlTo zP-y)Jz@s7rvd64h{1&}IB(>|i9dJK;S}Z-9?@$X&bDIg<^53-Sr)0RV@|jzrpko}btv09A1g8C zJeKl{E|dD1QBsG0sy@P88r?C0u5o%{GJENCn-yI9%@^*%ifm!!C)wF3Th0O8wU6KL zEt!bfEYse?!Gl~{neBd+Z7osKUJI>s$g9@TNvBSN;P3^PX0p*nob$%JL%Or-6mR)zV24y~$V{%4td5U>!;@g%JUbz?1>5pYy zgC_^1PhJWwxl~G8yFPn7uXA)({tf2ZpPtq1#rCP}Mcby6ixcxqGGfVO$nw%!hGFNV z{rlTqVGfDc2VNI;(iw@0mw_qL1rb5Fb?by%oIY;0%0ymZuix^F#$J6XPqe&QcqAB7_E8hUSBoVRW%`H0#|q#RC`H?;NWey$PEQ=6ZO z&#ZSIkn`6ewbm}r`S>BYM$FDb*virC{%h|>%nzp4+qxIF4kB>)W{7b+OvE~8>&U1| z*BHtE)1yAdfSN{r<{dG@e1mgsMPqyHS;F|m?^LeG^v-8gWcQ=|@xV4kRRLlH?*pv%2 zCn>J{n~tlok}L|D_~YKr=KRttDdM-DEqvvGXFTH=6T4tDhak>0&Hued|%d$OO~QXQ!_uGN{;} zh+blxCbZe5y-6`4#>t{V!@Nz{9jXN}9S8qK84<~hgKr(KL1k&)KFQER%@3x(nI9kP zMtgZo(ptF+cRIEoj`Y2?dEtV8utUfWeMV+HH_7<><%NGl`KgOl^)~?*1wH35<9XPw zrPfs5kN~@Ucix3;P{bP@eXPwK-7zV#22FnO`pxP5_k@pfbk5~n-Mx%O_SUJI6|@+BtgsMM+h5{W zPd)dPq1BJOxk@He=Syo#v)_AW{`=dJi~X_gvO135jzh|DVJqgK;Ffl2C1)NCMQ>IM zvfWT`EVl*~LG_lx8(u!SHX4g^a7yR_wUt8Vwlj_C8i490tzyM57RL{F#m!d)DZA?$yML*}0miQS-UBb`#SP!5DZ7&rOOGv$*$+u9#B#kD@|u1& ziaZh8XerI4oGeF556iy3D=W!u-b7C239ktD4&C&BCFOixPQe|ow(Q&#P-5swE=Btxm|EfkRM)&nyztJncMSYy7 z>izcv-R3|bMsmxx&MGyzS|a13<>_&=*fxCYE~n0X*!=JJEiKu<#b#e=(#T{ecK;dr zMUYvbE-LCZgmc@rTv&YH)=*JXHCd+Y{!#YESo62q>9HQpb^^C4ts~3c;}|WJthGSw zujXCeR#FBEXvQ_H62fe?yz!H$RmzGf(^|N<<+F#4m+fxD)nB*&{GD(0^MhHzU35O; zC9~R0B}ylS)t;7CxhUG6-nE6c49bk{vbl}PI$WNZOCM7+e1D`ENFrHcw^}nXIse^q zCuzhiNK>YvmMjO~EBNh<#5rPQs6tVWzwq5FB|`;n>hoX{AUwZpxp5y@EPH?ET1Z{2o6 zJ`l@z)z1f1B9kRD68Gcsf3l13R^F6hbh>w(!Fnmwd4m-0inQ}K#*Vaf>J9PD=5RN3 z+%Y#~4)vV9!R%H%AXJe(!rE_f`TkCOD^p^ZEmz#7!*C`%_SG6@M_QhSu|?lussfi^ z$%dmH-ReVY)`x^_1ovi ztisT)7BX=MNC)wXr_|>gm(5I>#ayce4evT7dmn8HQy|-avMo4-dH3-aHc$QIPvyC52atGBcyEGJ&cNT_4bPL2~^{I^v|2#_!4N$q) z(9eBs1!q|s^32pilNb2X#WeRrr8j)FRqDpAOI;<-YYN)#`^KZeqHpJUR?ogXXk9ot zcd0&X+sQAk>HLnpS9Prwed^g2rP7iWE&u(J)hsXN*7LR4KKq8gGB2g0aQQk6hWyKW z5#3k{gu2q_vc^rF7R&VC(&_rupNKti2d#5hMb>b<75JC-am(+SG3?qI2^w*P>y>{R zAcM#Ic&TlzpCIrIn6#vTuT40W@?9o7$!r>i_K+E06T7zgT(e;Gh~Q58UG;#;k@of& zsm>Zd=%BjvaRX%DSUrRF5AErvfpec;(KdzMxHiTht&66)zRz6J(rRjI(?s8Mqq_F0 z_l%{(BeI(ZXHNevnPp|2U^SRN7f~4PU8)QQi>@D%o_-+U8(kqZl4;4M%^anUTaZsQ zRm^F9?#7U%uskO&{}cPgEXDb?VqamTyNjmO=ytQ>RGZq<%#6C(;aBeOYDEvzyZ6VB zMnVL*XlR(p?j{@x>o?=oil6O0Z}(&Lv@%Cxaw|7DYk!n@YV@+opC>7oV%8=9-|*ZP P13*+^-qa8S2;_eNB$DzO literal 0 HcmV?d00001 diff --git a/test/fixtures/geoip-fixture/src/main/resources/data.json b/test/fixtures/geoip-fixture/src/main/resources/data.json index 2424be465b03a..0b23251ce9e3b 100644 --- a/test/fixtures/geoip-fixture/src/main/resources/data.json +++ b/test/fixtures/geoip-fixture/src/main/resources/data.json @@ -6,9 +6,9 @@ "provider": "maxmind" }, { - "md5_hash": "f452b1aece8dc6137485ddbb0401d6fe", - "name": "GeoLite2-ASN.mmdb.gz", - "url": "endpoint/db/GeoLite2-ASN.mmdb.gz", + "md5_hash": "61c38f0fcec4a7b0b359201f124004df", + "name": "GeoLite2-ASN.tgz", + "url": "endpoint/db/GeoLite2-ASN.tgz", "provider": "maxmind" }, { From dc7e08f15009b11e00142f34f9bb98c7c0561ec2 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 13:34:01 +0100 Subject: [PATCH 02/17] fix new String --- .../main/java/org/elasticsearch/common/io/TarInputStream.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java index ee480f17c4880..3fe20eea79ca9 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java +++ b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java @@ -12,6 +12,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; public class TarInputStream extends FilterInputStream { @@ -106,7 +107,7 @@ public int read(byte[] b, int off, int len) throws IOException { } private String getString(int offset, int maxLen) { - return new String(buf, offset, maxLen).trim(); + return new String(buf, offset, maxLen, StandardCharsets.UTF_8).trim(); } private int getInt(int offset) { From 0a5fe4821f22fceeb172222e079ce2ef9de60404 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 13:52:43 +0100 Subject: [PATCH 03/17] unused import --- .../main/java/org/elasticsearch/common/io/TarInputStream.java | 2 +- .../java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java index 3fe20eea79ca9..d441bc8de9885 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java +++ b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java @@ -112,7 +112,7 @@ private String getString(int offset, int maxLen) { private int getInt(int offset) { String s = getString(offset, 8); - return s.isEmpty() ? 0 : Integer.parseInt(s, 8); + return s.isEmpty() ? 0 : Integer.parseInt(s, 8);GeoIpDownloaderIT } private long getLong(int offset) { diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index 635e7c6f59dbe..b0e9e694bd20e 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -58,7 +58,6 @@ import java.util.zip.GZIPInputStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; From 54d90e7269849b98949c082085bfd44319611716 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 13:54:13 +0100 Subject: [PATCH 04/17] fix compilation --- .../main/java/org/elasticsearch/common/io/TarInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java index d441bc8de9885..3fe20eea79ca9 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java +++ b/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java @@ -112,7 +112,7 @@ private String getString(int offset, int maxLen) { private int getInt(int offset) { String s = getString(offset, 8); - return s.isEmpty() ? 0 : Integer.parseInt(s, 8);GeoIpDownloaderIT + return s.isEmpty() ? 0 : Integer.parseInt(s, 8); } private long getLong(int offset) { From 0b20d227e786db6d4ab1831308ee7c012c1b7b92 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 16:56:07 +0100 Subject: [PATCH 05/17] formatting --- .../ingest/geoip/DatabaseRegistryTests.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index 82ed02fc6031e..b7271c1aa2ac3 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -127,7 +127,8 @@ public void testCheckDatabases() throws Exception { String md5 = mockSearches("GeoIP2-City.mmdb", 5, 14); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5, false)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5, + false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -151,7 +152,8 @@ public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Excepti String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", + new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -175,7 +177,8 @@ public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Ex String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", + new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -270,7 +273,7 @@ private String mockSearches(String databaseName, int firstChunk, int lastChunk) throw new UncheckedIOException(ex); } - SearchHits hits = new SearchHits(new SearchHit[] {hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); + SearchHits hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); SearchResponse searchResponse = new SearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 0), null, 1, 1, 0, 1L, null, null); @SuppressWarnings("unchecked") @@ -312,7 +315,7 @@ private static List gzip(String content, int chunks) throws IOException int chunkSize = all.length / chunks; List data = new ArrayList<>(); - for (int from = 0; from < all.length;) { + for (int from = 0; from < all.length; ) { int to = from + chunkSize; if (to > all.length) { to = all.length; @@ -323,7 +326,7 @@ private static List gzip(String content, int chunks) throws IOException if (data.size() > chunks) { byte[] last = data.remove(data.size() - 1); - byte[] secondLast = data.remove(data.size() -1); + byte[] secondLast = data.remove(data.size() - 1); byte[] merged = new byte[secondLast.length + last.length]; System.arraycopy(secondLast, 0, merged, 0, secondLast.length); System.arraycopy(last, 0, merged, secondLast.length, last.length); From 152458cfa8d04cdb5ea1464b4d6f47c6870333b5 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 16:56:23 +0100 Subject: [PATCH 06/17] formatting --- .../ingest/geoip/GeoIpTaskStateSerializationTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java index 83b87d00fdef4..4b4827a56427f 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java @@ -30,7 +30,8 @@ protected GeoIpTaskState createTestInstance() { GeoIpTaskState state = GeoIpTaskState.EMPTY; int databaseCount = randomInt(20); for (int i = 0; i < databaseCount; i++) { - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32), false); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), + randomAlphaOfLength(32), false); state = state.put(randomAlphaOfLengthBetween(5, 10), metadata); } return state; From f326eb14640db4279d6b1a0411fa857f500b37dc Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 17:35:36 +0100 Subject: [PATCH 07/17] formatting --- .../java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index b0e9e694bd20e..b87cc3f790218 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -288,7 +288,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assertBusy(() -> { for (Path geoipTmpDir : geoipTmpDirs) { try (Stream list = Files.list(geoipTmpDir)) { - List files = list.map(Path::toString).filter(p->p.endsWith(".mmdb")).collect(Collectors.toList()); + List files = list.map(Path::toString).filter(p -> p.endsWith(".mmdb")).collect(Collectors.toList()); assertThat(files, empty()); } } From 45697ded265f6d587b924989b3cce6550aa1ad86 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 23 Mar 2021 20:36:54 +0100 Subject: [PATCH 08/17] fix tests --- .../elasticsearch/ingest/geoip/GeoIpDownloader.java | 5 ++++- .../ingest/geoip/GeoIpDownloaderTests.java | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 06fc39ecf3c3d..e0ab99ec500db 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -122,7 +122,10 @@ private List fetchDatabasesOverview() throws IOException { //visible for testing void processDatabase(Map databaseInfo) { String originalName = databaseInfo.get("name").toString(); - String name = originalName.replace(".mmdb.gz", "").replace(".tgz", "") + ".mmdb"; + String name = originalName + .replace(".mmdb.gz", "") + .replace(".gz", "") + .replace(".tgz", "") + ".mmdb"; String md5 = (String) databaseInfo.get("md5_hash"); if (state.contains(name) && Objects.equals(md5, state.get(name).getMd5())) { updateTimestamp(name, state.get(name)); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index c8acdcb3b907f..376604dca9b23 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -221,8 +221,8 @@ public void testProcessDatabaseUpdate() throws IOException { 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { @Override void updateTaskState() { - assertEquals(9, state.get("test").getFirstChunk()); - assertEquals(10, state.get("test").getLastChunk()); + assertEquals(9, state.get("test.mmdb").getFirstChunk()); + assertEquals(10, state.get("test.mmdb").getLastChunk()); } @Override @@ -239,19 +239,19 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { @Override void deleteOldChunks(String name, int firstChunk) { - assertEquals("test", name); + assertEquals("test.mmdb", name); assertEquals(9, firstChunk); } }; - geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test", new GeoIpTaskState.Metadata(0, 5, 8, "0", false))); + geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0", false))); geoIpDownloader.processDatabase(Map.of("name", "test.gz", "url", "a.b/t1", "md5_hash", "1")); } public void testProcessDatabaseSame() throws IOException { GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1", false); - GeoIpTaskState taskState = GeoIpTaskState.EMPTY.put("test", metadata); + GeoIpTaskState taskState = GeoIpTaskState.EMPTY.put("test.mmdb", metadata); ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); when(httpClient.get("a.b/t1")).thenReturn(bais); @@ -271,7 +271,7 @@ int indexChunks(String name, InputStream is, int chunk, String expectedMd5) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) { assertEquals(metadata, newMetadata); - assertEquals("test", name); + assertEquals("test.mmdb", name); } @Override From ffed31d155ffaca38b5b9377ffcd3870420514f8 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:26:59 +0100 Subject: [PATCH 09/17] handle only tgz files --- .../org/elasticsearch/common/io/TarEntry.java | 133 ------------------ .../ingest/geoip/GeoIpDownloaderIT.java | 31 ++-- .../ingest/geoip/DatabaseRegistry.java | 26 ++-- .../ingest/geoip/GeoIpDownloader.java | 17 +-- .../ingest/geoip/GeoIpTaskState.java | 19 +-- .../ingest/geoip}/TarInputStream.java | 96 +++++-------- .../ingest/geoip/DatabaseRegistryTests.java | 32 +++-- .../ingest/geoip/GeoIpDownloaderTests.java | 16 +-- .../GeoIpTaskStateSerializationTests.java | 3 +- .../ingest/geoip}/TarInputStreamTests.java | 30 ++-- .../org/elasticsearch/ingest/geoip}/tar1.tar | Bin 2048 -> 2048 bytes .../org/elasticsearch/ingest/geoip}/tar2.tar | Bin 3072 -> 3072 bytes .../org/elasticsearch/ingest/geoip}/tar3.tar | Bin 3072 -> 3072 bytes .../org/elasticsearch/ingest/geoip/tar4.tar | Bin 0 -> 3584 bytes .../src/main/resources/GeoLite2-City.mmdb.gz | Bin 10043 -> 0 bytes .../src/main/resources/GeoLite2-City.tgz | Bin 0 -> 10582 bytes .../main/resources/GeoLite2-Country.mmdb.gz | Bin 8470 -> 0 bytes .../src/main/resources/GeoLite2-Country.tgz | Bin 0 -> 9041 bytes .../src/main/resources/data.json | 12 +- 19 files changed, 125 insertions(+), 290 deletions(-) delete mode 100644 libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java rename {libs/core/src/main/java/org/elasticsearch/common/io => modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip}/TarInputStream.java (51%) rename {libs/core/src/test/java/org/elasticsearch/common/io => modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip}/TarInputStreamTests.java (68%) rename {libs/core/src/test/resources/org/elasticsearch/common/io => modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip}/tar1.tar (93%) rename {libs/core/src/test/resources/org/elasticsearch/common/io => modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip}/tar2.tar (81%) rename {libs/core/src/test/resources/org/elasticsearch/common/io => modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip}/tar3.tar (93%) create mode 100644 modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar4.tar delete mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.mmdb.gz create mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.tgz delete mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-Country.mmdb.gz create mode 100644 test/fixtures/geoip-fixture/src/main/resources/GeoLite2-Country.tgz diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java b/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java deleted file mode 100644 index ca60d8802d93a..0000000000000 --- a/libs/core/src/main/java/org/elasticsearch/common/io/TarEntry.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.io; - -import java.util.Objects; - -public class TarEntry { - - private final String name; - private final int mode; - private final int uid; - private final int gid; - private final long size; - private final long modificationTime; - private final Type type; - private final String linkName; - private final String uname; - private final String gname; - private final int devMajor; - private final int devMinor; - private final boolean ustarFormat; - - public TarEntry(String name, int mode, int uid, int gid, long size, long modificationTime, Type type, String linkName, String uname, - String gname, int devMajor, int devMinor, boolean ustarFormat) { - this.name = name; - this.mode = mode; - this.uid = uid; - this.gid = gid; - this.size = size; - this.modificationTime = modificationTime; - this.type = type; - this.linkName = linkName; - this.uname = uname; - this.gname = gname; - this.devMajor = devMajor; - this.devMinor = devMinor; - this.ustarFormat = ustarFormat; - } - - public String getName() { - return name; - } - - public int getMode() { - return mode; - } - - public int getUid() { - return uid; - } - - public int getGid() { - return gid; - } - - public long getSize() { - return size; - } - - public long getModificationTime() { - return modificationTime; - } - - public Type getType() { - return type; - } - - public String getLinkName() { - return linkName; - } - - public String getUname() { - return uname; - } - - public String getGname() { - return gname; - } - - public int getDevMajor() { - return devMajor; - } - - public int getDevMinor() { - return devMinor; - } - - public boolean isUstarFormat() { - return ustarFormat; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TarEntry tarEntry = (TarEntry) o; - return mode == tarEntry.mode - && uid == tarEntry.uid - && gid == tarEntry.gid - && size == tarEntry.size - && modificationTime == tarEntry.modificationTime - && devMajor == tarEntry.devMajor - && devMinor == tarEntry.devMinor - && ustarFormat == tarEntry.ustarFormat - && Objects.equals(name, tarEntry.name) - && type == tarEntry.type - && Objects.equals(linkName, tarEntry.linkName) - && Objects.equals(uname, tarEntry.uname) - && Objects.equals(gname, tarEntry.gname); - } - - @Override - public int hashCode() { - return Objects.hash(name, mode, uid, gid, size, modificationTime, type, linkName, uname, gname, devMajor, devMinor, ustarFormat); - } - - public enum Type { - FILE, LINK, SYMLINK, CHARACTER_DEVICE, BLOCK_DEVICE, DIRECTORY, FIFO, CONTIGUOUS_FILE; - } - - static Type getType(String type) { - if (type == null || type.isBlank()) { - return Type.FILE; - } - return Type.values()[Integer.parseInt(type)]; - } -} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index b87cc3f790218..ee60fd2b48ad2 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -17,8 +17,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.TarEntry; -import org.elasticsearch.common.io.TarInputStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -58,10 +56,9 @@ import java.util.zip.GZIPInputStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -135,16 +132,11 @@ public void testGeoIpDatabasesDownload() throws Exception { data.add((byte[]) hit.getSourceAsMap().get("data")); } - InputStream stream = new GZIPInputStream(new MultiByteArrayInputStream(data)); - - if (metadata.isTar()) { - TarInputStream tarIs = new TarInputStream(stream); - stream = tarIs; - TarEntry entry; - while ((entry = tarIs.getNextEntry()) != null) { - if (entry.getName().endsWith(".mmdb")) { - break; - } + TarInputStream stream = new TarInputStream(new GZIPInputStream(new MultiByteArrayInputStream(data))); + TarInputStream.TarEntry entry; + while ((entry = stream.getNextEntry()) != null) { + if (entry.getName().endsWith(".mmdb")) { + break; } } @@ -160,6 +152,7 @@ public void testGeoIpDatabasesDownload() throws Exception { @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/69972") public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { + assumeTrue("only test with fixture to have stable results", ENDPOINT != null); // setup: BytesReference bytes; try (XContentBuilder builder = JsonXContent.contentBuilder()) { @@ -262,9 +255,11 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assertBusy(() -> { for (Path geoipTmpDir : geoipTmpDirs) { try (Stream list = Files.list(geoipTmpDir)) { - List files = list.map(Path::toString).collect(Collectors.toList()); - assertThat(files, hasItems(endsWith("GeoLite2-City.mmdb"), endsWith("GeoLite2-Country.mmdb"), - endsWith("GeoLite2-ASN.mmdb"))); + List files = list.map(Path::getFileName).map(Path::toString).collect(Collectors.toList()); + assertThat(files, containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", + "GeoLite2-City.mmdb_COPYRIGHT.txt","GeoLite2-Country.mmdb_COPYRIGHT.txt","GeoLite2-ASN.mmdb_COPYRIGHT.txt", + "GeoLite2-City.mmdb_LICENSE.txt","GeoLite2-Country.mmdb_LICENSE.txt","GeoLite2-ASN.mmdb_LICENSE.txt", + "GeoLite2-ASN.mmdb_README.txt")); } } }); @@ -276,7 +271,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assertThat(simulateResponse.getResults().size(), equalTo(1)); SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) simulateResponse.getResults().get(0); assertThat(result.getFailure(), nullValue()); - assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Tumba")); + assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Linköping")); assertThat(result.getIngestDocument().getFieldValue("ip-asn.organization_name", String.class), equalTo("Bredband2 AB")); assertThat(result.getIngestDocument().getFieldValue("ip-country.country_name", String.class), equalTo("Sweden")); }); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index 7c80517fd2c7c..8d1781f658706 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -22,8 +22,6 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.common.io.TarEntry; -import org.elasticsearch.common.io.TarInputStream; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.query.TermQueryBuilder; @@ -266,19 +264,17 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta decompress(databaseTmpGzFile, databaseTmpFile); Path databaseFile = geoipTmpDirectory.resolve(databaseName); - if (metadata.isTar()) { - try (TarInputStream is = new TarInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpFile)))) { - TarEntry entry; - while ((entry = is.getNextEntry()) != null) { - if (entry.getType() != TarEntry.Type.FILE) { - continue; - } - String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); - if (name.startsWith(databaseName)) { - Files.copy(is, databaseTmpFile, REPLACE_EXISTING); - } else { - Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), REPLACE_EXISTING); - } + try (TarInputStream is = new TarInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpFile)))) { + TarInputStream.TarEntry entry; + while ((entry = is.getNextEntry()) != null) { + if (entry.isNotFile()) { + continue; + } + String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); + if (name.startsWith(databaseName)) { + Files.copy(is, databaseTmpFile, REPLACE_EXISTING); + } else { + Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), REPLACE_EXISTING); } } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index e0ab99ec500db..8b8bfd36919f5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -104,7 +104,9 @@ void updateDatabases() throws IOException { logger.info("updating geoip databases"); List> response = fetchDatabasesOverview(); for (Map res : response) { - processDatabase(res); + if (res.get("name").toString().endsWith(".tgz")) { + processDatabase(res); + } } } @@ -121,11 +123,7 @@ private List fetchDatabasesOverview() throws IOException { //visible for testing void processDatabase(Map databaseInfo) { - String originalName = databaseInfo.get("name").toString(); - String name = originalName - .replace(".mmdb.gz", "") - .replace(".gz", "") - .replace(".tgz", "") + ".mmdb"; + String name = databaseInfo.get("name").toString().replace(".tgz", "") + ".mmdb"; String md5 = (String) databaseInfo.get("md5_hash"); if (state.contains(name) && Objects.equals(md5, state.get(name).getMd5())) { updateTimestamp(name, state.get(name)); @@ -138,7 +136,7 @@ void processDatabase(Map databaseInfo) { int firstChunk = state.contains(name) ? state.get(name).getLastChunk() + 1 : 0; int lastChunk = indexChunks(name, is, firstChunk, md5); if (lastChunk > firstChunk) { - Metadata metadata = new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5, originalName.endsWith(".tgz")); + Metadata metadata = new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5); state = state.put(name, metadata); updateTaskState(); stats = stats.successfulDownload(System.currentTimeMillis() - start).count(state.getDatabases().size()); @@ -166,8 +164,7 @@ void deleteOldChunks(String name, int firstChunk) { //visible for testing protected void updateTimestamp(String name, Metadata old) { logger.info("geoip database [" + name + "] is up to date, updated timestamp"); - state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5(), - old.isTar())); + state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5())); updateTaskState(); } @@ -239,7 +236,7 @@ protected void onCancelled() { @Override public GeoIpDownloaderStats getStatus() { - return isCancelled() || isCompleted() ? null: stats; + return isCancelled() || isCompleted() ? null : stats; } private void scheduleNextRun(TimeValue time) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index b16c8ad38ba74..77ccbbfa5cfa8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -136,7 +136,7 @@ static class Metadata implements ToXContentObject, Writeable { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, - args -> new Metadata((long) args[0], (int) args[1], (int) args[2], (String) args[3], args[4] != null && (boolean) args[4])); + args -> new Metadata((long) args[0], (int) args[1], (int) args[2], (String) args[3])); static { PARSER.declareLong(constructorArg(), LAST_UPDATE); @@ -158,18 +158,16 @@ public static Metadata fromXContent(XContentParser parser) { private final int firstChunk; private final int lastChunk; private final String md5; - private final boolean tar; Metadata(StreamInput in) throws IOException { - this(in.readLong(), in.readVInt(), in.readVInt(), in.readString(), in.readBoolean()); + this(in.readLong(), in.readVInt(), in.readVInt(), in.readString()); } - Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, boolean tar) { + Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5) { this.lastUpdate = lastUpdate; this.firstChunk = firstChunk; this.lastChunk = lastChunk; this.md5 = Objects.requireNonNull(md5); - this.tar = tar; } public long getLastUpdate() { @@ -188,10 +186,6 @@ public String getMd5() { return md5; } - public boolean isTar() { - return tar; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -200,13 +194,12 @@ public boolean equals(Object o) { return lastUpdate == metadata.lastUpdate && firstChunk == metadata.firstChunk && lastChunk == metadata.lastChunk - && md5.equals(metadata.md5) - && tar == metadata.tar; + && md5.equals(metadata.md5); } @Override public int hashCode() { - return Objects.hash(lastUpdate, firstChunk, lastChunk, md5, tar); + return Objects.hash(lastUpdate, firstChunk, lastChunk, md5); } @Override @@ -217,7 +210,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(FIRST_CHUNK.getPreferredName(), firstChunk); builder.field(LAST_CHUNK.getPreferredName(), lastChunk); builder.field(MD5.getPreferredName(), md5); - builder.field(TAR.getPreferredName(), tar); } builder.endObject(); return builder; @@ -229,7 +221,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(firstChunk); out.writeVInt(lastChunk); out.writeString(md5); - out.writeBoolean(tar); } } } diff --git a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java similarity index 51% rename from libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java rename to modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java index 3fe20eea79ca9..51ba768b08cc7 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/io/TarInputStream.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -6,29 +6,34 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.io; +package org.elasticsearch.ingest.geoip; import java.io.EOFException; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; -public class TarInputStream extends FilterInputStream { +/** + * {@link InputStream} with very basic support for tar format, just enough to parse archives provided by GeoIP database service from Infra. + * This class is not suitable for general purpose tar processing! + */ +class TarInputStream extends FilterInputStream { private TarEntry currentEntry; private long remaining; private final byte[] buf = new byte[512]; - public TarInputStream(InputStream is) { - super(is); + TarInputStream(InputStream in) { + super(in); } public TarEntry getNextEntry() throws IOException { if (currentEntry != null) { //go to the end of the current entry skipN(remaining); - long reminder = currentEntry.getSize() % 512; + long reminder = currentEntry.size % 512; if (reminder != 0) { skipN(512 - reminder); } @@ -40,53 +45,21 @@ public TarEntry getNextEntry() throws IOException { if (read != 512) { throw new EOFException(); } - if (bufAllZeroes()) { + if (Arrays.compare(buf, new byte[512]) == 0) { return null; } - String name = getString(0, 100); - int mode = getInt(100); - int uid = getInt(108); - int gid = getInt(116); - remaining = getLong(124); - long mTime = getLong(136); - int checksum = getInt(148); - TarEntry.Type type = TarEntry.getType(getString(156, 1)); - String linkName = getString(157, 100); - boolean ustar = getString(257, 6).equals("ustar"); - String uname = getString(265, 32); - String gname = getString(297, 32); - int devMajor = getInt(329); - int devMinor = getInt(337); - name = getString(345, 155) + name; - if (name.endsWith("/")) { - type = TarEntry.Type.DIRECTORY; - name = name.substring(0, name.length() - 1); - } - ensureConsistency(checksum); - currentEntry = new TarEntry(name, mode, uid, gid, remaining, mTime, type, linkName, uname, gname, devMajor, devMinor, ustar); - if (type == TarEntry.Type.DIRECTORY) { + String name = getString(345, 155) + getString(0, 100); + + String sizeString = getString(124, 12); + remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + boolean notFile = (buf[156] != 0 && buf[156] != '0') || name.endsWith("/"); + currentEntry = new TarEntry(name, remaining, notFile); + if (notFile) { remaining = 0; } return currentEntry; } - private void ensureConsistency(int checksum) throws IOException { - for (int i = 0; i < 148; i++) { - checksum -= (buf[i] & 0xff); - } - for (int i = 156; i < 512; i++) { - checksum -= (buf[i] & 0xff); - } - if (checksum != 256) { - throw new IOException("entry header corrupted - checksum mismatch"); - } - } - - @Override - public int available() throws IOException { - return remaining > 0 ? 1 : 0; - } - @Override public int read() throws IOException { if (remaining == 0) { @@ -110,16 +83,6 @@ private String getString(int offset, int maxLen) { return new String(buf, offset, maxLen, StandardCharsets.UTF_8).trim(); } - private int getInt(int offset) { - String s = getString(offset, 8); - return s.isEmpty() ? 0 : Integer.parseInt(s, 8); - } - - private long getLong(int offset) { - String s = getString(offset, 12); - return s.isEmpty() ? 0 : Long.parseLong(s, 8); - } - private void skipN(long n) throws IOException { while (n > 0) { long skip = in.skip(n); @@ -134,13 +97,24 @@ private void skipN(long n) throws IOException { } } - private boolean bufAllZeroes() { - for (byte b : buf) { - if (b != 0) { - return false; - } + static class TarEntry { + private final String name; + private final long size; + private final boolean notFile; + + TarEntry(String name, long size, boolean notFile) { + this.name = name; + this.size = size; + this.notFile = notFile; + } + + public String getName() { + return name; + } + + public boolean isNotFile() { + return notFile; } - return true; } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index b7271c1aa2ac3..3bd96f3d8e11b 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -127,8 +127,7 @@ public void testCheckDatabases() throws Exception { String md5 = mockSearches("GeoIP2-City.mmdb", 5, 14); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5, - false)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 5, 14, md5)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -153,7 +152,7 @@ public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Excepti String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", - new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); + new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -178,7 +177,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Ex String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", - new GeoIpTaskState.Metadata(0L, 0, 9, md5, false)))); + new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -219,7 +218,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenGeoIpDownloadTask() throws E public void testRetrieveDatabase() throws Exception { String md5 = mockSearches("_name", 0, 29); - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 29, md5, false); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 29, md5); @SuppressWarnings("unchecked") CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); @@ -237,7 +236,7 @@ public void testRetrieveDatabase() throws Exception { public void testRetrieveDatabaseCorruption() throws Exception { String md5 = mockSearches("_name", 0, 9); String incorrectMd5 = "different"; - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 9, incorrectMd5, false); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 9, incorrectMd5); @SuppressWarnings("unchecked") CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); @@ -258,7 +257,7 @@ public void testRetrieveDatabaseCorruption() throws Exception { private String mockSearches(String databaseName, int firstChunk, int lastChunk) throws IOException { String dummyContent = "test: " + databaseName; - List data = gzip(dummyContent, lastChunk - firstChunk + 1); + List data = gzip(databaseName, dummyContent, lastChunk - firstChunk + 1); assertThat(gunzip(data), equalTo(dummyContent)); for (int i = firstChunk; i <= lastChunk; i++) { @@ -305,10 +304,20 @@ private static RoutingTable createIndexRoutingTable() { return RoutingTable.builder().add(IndexRoutingTable.builder(index).addIndexShard(table).build()).build(); } - private static List gzip(String content, int chunks) throws IOException { + private static List gzip(String name, String content, int chunks) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); GZIPOutputStream gzipOutputStream = new GZIPOutputStream(bytes); - gzipOutputStream.write(content.getBytes(StandardCharsets.UTF_8)); + byte[] header = new byte[512]; + byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + byte[] sizeBytes = String.format(Locale.ROOT, "%1$012o", contentBytes.length).getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + System.arraycopy(sizeBytes, 0, header, 124, 12); + gzipOutputStream.write(header); + gzipOutputStream.write(contentBytes); + gzipOutputStream.write(512 - contentBytes.length); + gzipOutputStream.write(new byte[512]); + gzipOutputStream.write(new byte[512]); gzipOutputStream.close(); byte[] all = bytes.toByteArray(); @@ -324,7 +333,7 @@ private static List gzip(String content, int chunks) throws IOException from = to; } - if (data.size() > chunks) { + while (data.size() > chunks) { byte[] last = data.remove(data.size() - 1); byte[] secondLast = data.remove(data.size() - 1); byte[] merged = new byte[secondLast.length + last.length]; @@ -344,7 +353,8 @@ private static String gunzip(List chunks) throws IOException { System.arraycopy(chunk, 0, gzippedContent, written, chunk.length); written += chunk.length; } - GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(gzippedContent)); + TarInputStream gzipInputStream = new TarInputStream(new GZIPInputStream(new ByteArrayInputStream(gzippedContent))); + gzipInputStream.getNextEntry(); return Streams.readFully(gzipInputStream).utf8ToString(); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index 376604dca9b23..feb3e3ff34ec3 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -210,7 +210,7 @@ void deleteOldChunks(String name, int firstChunk) { }; geoIpDownloader.setState(GeoIpTaskState.EMPTY); - geoIpDownloader.processDatabase(Map.of("name", "test.gz", "url", "a.b/t1", "md5_hash", "1")); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "a.b/t1", "md5_hash", "1")); } public void testProcessDatabaseUpdate() throws IOException { @@ -244,13 +244,13 @@ void deleteOldChunks(String name, int firstChunk) { } }; - geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0", false))); - geoIpDownloader.processDatabase(Map.of("name", "test.gz", "url", "a.b/t1", "md5_hash", "1")); + geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0"))); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "a.b/t1", "md5_hash", "1")); } public void testProcessDatabaseSame() throws IOException { - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1", false); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1"); GeoIpTaskState taskState = GeoIpTaskState.EMPTY.put("test.mmdb", metadata); ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); when(httpClient.get("a.b/t1")).thenReturn(bais); @@ -280,7 +280,7 @@ void deleteOldChunks(String name, int firstChunk) { } }; geoIpDownloader.setState(taskState); - geoIpDownloader.processDatabase(Map.of("name", "test.gz", "url", "a.b/t1", "md5_hash", "1")); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "a.b/t1", "md5_hash", "1")); } @SuppressWarnings("unchecked") @@ -317,12 +317,12 @@ public void updatePersistentTaskState(PersistentTaskState state, ActionListener< } public void testUpdateDatabases() throws IOException { - List> maps = List.of(Map.of("a", 1), Map.of("a", 2)); + List> maps = List.of(Map.of("a", 1, "name", "a.tgz"), Map.of("a", 2, "name", "a.tgz")); ByteArrayOutputStream baos = new ByteArrayOutputStream(); XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); builder.startArray(); - builder.map(Map.of("a", 1)); - builder.map(Map.of("a", 2)); + builder.map(Map.of("a", 1, "name", "a.tgz")); + builder.map(Map.of("a", 2, "name", "a.tgz")); builder.endArray(); builder.close(); when(httpClient.getBytes("a.b?elastic_geoip_service_tos=agree")) diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java index 4b4827a56427f..dd5faa9d8fa33 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java @@ -30,8 +30,7 @@ protected GeoIpTaskState createTestInstance() { GeoIpTaskState state = GeoIpTaskState.EMPTY; int databaseCount = randomInt(20); for (int i = 0; i < databaseCount; i++) { - GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), - randomAlphaOfLength(32), false); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32)); state = state.put(randomAlphaOfLengthBetween(5, 10), metadata); } return state; diff --git a/libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java similarity index 68% rename from libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java rename to modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java index 17f956c6ec168..7173df65c5efd 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/io/TarInputStreamTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.io; +package org.elasticsearch.ingest.geoip; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -35,10 +35,12 @@ public void test() throws IOException { TarInputStream tis = new TarInputStream(is)) { assertNotNull(is); for (Entry entry : entries) { - TarEntry tarEntry = tis.getNextEntry(); + TarInputStream.TarEntry tarEntry = tis.getNextEntry(); assertEquals(entry.name, tarEntry.getName()); - assertEquals(entry.mtime, tarEntry.getModificationTime()); - assertEquals(entry.data, new String(tis.readAllBytes(), StandardCharsets.UTF_8)); + if (entry.notFile == false) { + assertEquals(entry.data, new String(tis.readAllBytes(), StandardCharsets.UTF_8)); + } + assertEquals(entry.notFile, tarEntry.isNotFile()); } assertNull(tis.getNextEntry()); } @@ -48,13 +50,17 @@ public void test() throws IOException { public static Iterable parameters() throws Exception { Object[][] entries = new Object[][]{ createTest("tar1.tar", - new Entry("a.txt", "aaa", 1616416474)), + new Entry("a.txt", "aaa\n", false)), createTest("tar2.tar", - new Entry("a.txt", "aaa", 1616416474), - new Entry("b.txt", "bbbbbb", 1616416484)), + new Entry("a.txt", "aaa\n", false), + new Entry("b.txt", "bbbbbb\n", false)), createTest("tar3.tar", - new Entry("c.txt", Stream.generate(() -> "a").limit(512).collect(Collectors.joining()), 1616416501), - new Entry("b.txt", "bbbbbb", 1616416484)), + new Entry("c.txt", Stream.generate(() -> "-").limit(512).collect(Collectors.joining()), false), + new Entry("b.txt", "bbbbbb\n", false)), + createTest("tar4.tar", + new Entry("./", null, true), + new Entry("./b.txt", "bbb\n", false), + new Entry("./a.txt", "aaa\n", false)) }; return Arrays.asList(entries); } @@ -66,12 +72,12 @@ private static Object[] createTest(String name, Entry... entries) { private static class Entry { String name; String data; - long mtime; + boolean notFile; - private Entry(String name, String data, long mtime) { + private Entry(String name, String data, boolean notFile) { this.name = name; this.data = data; - this.mtime = mtime; + this.notFile = notFile; } @Override diff --git a/libs/core/src/test/resources/org/elasticsearch/common/io/tar1.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar1.tar similarity index 93% rename from libs/core/src/test/resources/org/elasticsearch/common/io/tar1.tar rename to modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar1.tar index bd1ce119eff80b6164e95225a31fd4d00731d4ca..86e0aa3627085ec918868460c083d225dd5751fd 100644 GIT binary patch delta 49 zcmZn=Xb_mtCT^l&XkuVwZeV6+YGS5fU}$V$V#1(cFgcMiWiulq6C)$nWS@ E0UdJ-3jhEB delta 47 zcmZn=Xb_mtCT^@?XkuVwW@un;Y;2@pU}$V$WX_;qFgcMiWiulq6XRw(%}v2dO^x76Sy>@U-?L6s-~s^1pA=>Q delta 88 zcmZpWXporDCT^@?XkuVwW@un;Y;2@pU}$V$WX_;qFgcMiWiulq6XRwiqPE_Cq0F06pJOBUy diff --git a/libs/core/src/test/resources/org/elasticsearch/common/io/tar3.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar3.tar similarity index 93% rename from libs/core/src/test/resources/org/elasticsearch/common/io/tar3.tar rename to modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar3.tar index 9cb7e93b81a12034eeaf649f5a4a202aac532771..85f638baca76c77a35e3ebfea6160dfa62d940f8 100644 GIT binary patch literal 3072 zcmeHH%L;@r4D8vj=nt$+yJ_=1x+osSR`ByTTJYxHe8479h9;qt3`{?DUa~ni07N2d z1UZHc;mci8aB{38ct(s&!NR>kWGv1dtGqs^+)Ogs(511P=5)jEAh?6KpI t)+d!-8h@=od%yq1HoW^kaPfce>i;YLiFS+X3M%DL1yli5Koxk90$-aei97%R literal 3072 zcmeHHNeaU-4D?g{f-sI9`8`QV=&{5$^!c4S(43N67N`b{uq3n$j2=yXXLBq7NC?)@ zM(L_WyZq^bq&v_!W%(7GqbNtD)WflX@P7x+<9wxyT-!m#LnlAu|-b?Yu za3NTC?kRLGX&l%HArMM1)*2_IZ{8UZ9J%05tv!~!E&GvcoK&{1wIs#wdwlRm=}0N1 zlPb^Bv?Y=zxXbT~|L)8G*+dW$ns#*`{^$Oul%^gsGsl<;D2-#y|7_|1=lSpTS`d-v Rzn9p|!wRqhtiV7Dya0jEM7#h1 literal 0 HcmV?d00001 diff --git a/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.mmdb.gz b/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.mmdb.gz deleted file mode 100644 index f71d1181119630cfc82e7007005b0f71df699729..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10043 zcmZ9x2Q-{t)HRMM5z!O9Mj0fC-VH%SC%WiFj}k=GNurIO1c^RDL?@#6KB5zY=!xFP zh+!~hzQ^zVzw7_4Z!PP%&%OKXv-df7o;3^3gqt@FeZckvH#DF7=(-0y7Z+A@4+s`E zdF~$|^5TWFBSBymc^F%N4L-Cb^o?)yls)~?t;7%P*{nU87C!VW>u}J{ciLn(a5MS-u9#DtE5?Z+eEqg3aY?$ z7}Z3v>J3Fc3Zr(sqt2nO+~)=rqamS*rqQ73r%{E{L7zg!p)aAgpfxn$q6YgovD7`DNxpu`tQUbrv0kIxSNVV{XGf;O9!_O07H=%YR+&sWGWrh8)+JxZC*529U_iNx3)N!1fVw{)RM ziPIZr>(u#kkKqyfc{$V7-mLY@&K0@D^&I_!1B(iru^FSk*{nl*+ND3&1bgpk!8cng zNoXjz8N?cxmS8V^EQxCFI5Im8qw6K|`f?4X29*+(B4@N4SA^hs{exRvW@?JmHKYZ- z2j4%{1j;rqD>c8ZVf1~;)0c2qub24Z)w5?@4dM++eIjt%f+IBt2Zd-Dh3^ZpK7HY_ zg8I5=LEtIkly;iSMc2bv*{sw#@a=eBT;)2Ji-7Pn2I}(GeF1Z9J zUXjn@Zlf5_<_8k5?t<{7s7@MKQ)AdIR1h3TJ&PkmWrhxexs_*E*=}{?zh!8YqbTT{ z*G07w%@&nqK&oE87? z90*XzOl8vszY9PoY`24Bzg`?14g%NJdB)~&u$8-G_`%UQ%ZBZ+lUc{Oh0~%Z$;dsN zrK#Ip)jJaZk(4724B-H#%-0|}x+JP8Ha8*P;{TUT&(#I~54RU_Hkr!B@#gjW|1p(c z{ktL=SZ=83`c+FkET4Q4oe3D8;<`tpM`HxdSjf+`q;9Re)_i{_Pghm3r(&l`y0?9X zh!7X?qhLe)DKtyyHX?zye-{fF~k6Apidh74tf)vHyH_qJ! z-V?sL%Gt>&k$XMSt6Qrtpmu6)YC&e=dp32zP{>9ugn?|d0u&zw6g%U_5uog>GO$w@ zG^;!zCmJKWUtJ@HDn#hQ4NF-IV*hJt?irO;A9!zktiN{8oywzReroz z-Qjud%jOMH*S%A}Y)b&=56t?y_SD%aWHL7gxP^8Sq%Hrxo0VzVOywo&b$cEQRY{Z=_S$7F%kNDu{PKog_? z^lHGzo(;P;dubbWk+QO`pf&L)IMMiO0O^8N;61!PYP{q~0**G6O0kKM&p?foW2G$2 zK;$)wf|(32yW2?tDvZ8JVZ5|e$F6N8Mnc*~f+(?TPMrmPN{Qypl^#$Q_&_sgKuz~e z#!p$y0P-K4ycUkR;oV6F ztpq3o=5Wf{1?Q>nIl$Lm-@pX@U70MnNEx?hLvxbhMIJraD#2rPg+-(rzFipHghQTO zB|p?J5`rZAmEkc&aMuqRqq2AFu}1KPiG21Y*L*-RxT>GP*$g06PVqxP!bGZ=v)@A4 zu&u;$!XO-KBat4Dp@VmWsc<$#2*9}Lza%qDM?PQxKs~8sBAnk5@@XLmmEz+BOv26{ z2(w~Tz^)HhgK|?1Ge+h}2e@5f68~k8W;PLDXNVs`0bX0cUl`+m2OZJAL*DLq`5k!^df0WXBm@Nx*h#8=P^tu|X zazYY-bbOEN0|Z2IZBGTeh7O z3>P)o04k`CWzqx;@2VIdPkrN6uxN_z_K&sDBn)?i6H7xBazGbLQ^|Qw znp7`n6>h+d8=TWs)P6U^)ysd5F94`VmP$Dx%+ zZ?dd%0fK50uB(gzSzgzUf3H5X2BxyxSw8xU6yK-6^70-oF8sP`_4P=*z`eczQ}sjD ztT-p#FF5)PwqEzQ7Y1K&!|(^Iu-Cb9l2+9~=PwMfu|CPbZQ#A``q}lZu@eIlJDmYh zKIaTX7?3^&u7agUiUZ9L3d_5Vi(9$y(!uWLCG^iG^2T+b#w4 zl_D#@lWNZ{mX|fzas7bgupuB|lK=-83Sg+1T4X&C%aMXW>KT=`0FH&$fKTfX40VE@8(3$X}oSz!azUE`Y6sOkUie zgYMHS5Sf@3HuXjZ@JNwKj9(||V0;TXcC4PE`}Fw_jBUI1F*em)9Dqb-0ym%rFL(PS z{SQq2%Js>;ILJ$Z)Dk>z2R^ezpaw7K-);+vHI7}lre?hkdl`La;NVIb>nBGB%YST; zKwY2b*HRc@HC^D~Y$PwrVDq~C%5yq2tZhN&)saE+aXSqE0~=G}$&aw9wb!(q;^*Nx*tf{cKQL^~wG-tb-d*tAA*Iq z|C-dFALtV+uxic%UJUDW*N?QRe_(*z(yzQ2P&e3EMtc=zeeFtl^s8V0UFG|4A-@K| zgt24a{BCRzYc9F)oI^n0B zJAvP@!(5(cD5RH5B;?d`*}^_j<+Qnjstm_E?A717msyGV-0OvoQ5`YEwnokUfhlzr zb75SmPpGEM&?|RG9$&fI9R|uQgyAVL%T)!d4_(SHd<1Go&pON7_qwH5@Lk~AREP4F z&O~SFUGRcA9-s)%MSd0U766dQSJ%M|C=Cd?S4W8hPRWBC%J7FS0Qyp7rOOaMxAKot zcN)?^VurROtnCJeCRTG|Tp7<#Owe}BwWzCccrL#~eK)wZ=j>j8CsXZRJcr{o!*d|O zC0Mbw1Lti7*xK$+T1<-*P>s+f8wp8mOZWvypS`$d;i)yf+Ik2e!?D-GNmrNk(VNGZMq7*?n=k}k$^9R`dDNJBNQ z)^uJn8-RtT0d|%S_gslFGQale?&C(C8qTkVtzmrM@$QAV)|o43@3k}0(|gc<8Uxio z!0M@ssK3I4Si_aO;rNgNM0Bb!XS$?pja$iJDdlXj{{qD=gx_j!7|1G(Gh2+0p_+Q8 zzhKZ85y!7Efau@N#lG`1Ag0~w=qu;fGPUCqt z8i6$?lFU9kU(eQ;kHh?AY0Ia5M1}?wK#4)C`m!wEPkyy(PhD<9=L3_zs`od!$?hP? zw0kFGE-dH;jjc2q;Y0x)%8xzL&HAk#mGZmt=5mn?%P-6XHMAq7&#+)F(@qIDyd5v561FPR` zI=pHhsBg$5aVVf-%9^HV^m*O8(DiT;8|TlS<6-&yIVF^up6p99cPjNXw3zo!y1rq1 zgnpNPeaI27Kb-Z=uznYTy|Yv33my6J&s@BxY3%fQ?m@QvLQpDg3+KYz6(Uu#OGSI- z^D@)DY!m01RQ8{wAPyO`?YZ_uZy}wp{Iju~(a*^q@=?;vG{nYlm_=2uCA{&|{ zlQB;OtPGICEzRB_)2WaUKI9M*BgDQO!^#k@%PBSVm_8$ zzgWcUH?C&y8P>%zmA($QjdZKH__>IdzS{0%d+KFoA2BybInbKyg^cjBwF}h@;%b!e zHIA^Ye>LaR`+y`k?mK^bC`IX#eLqV)fHUmrreX(A;#&iQ=E}V1Q;?p5AC_e%aXRhKFz+ zJ4o$JB%@fb0%^%uXrAWG0CS6lm40T+SPA@NKUPwe^RYfmW$Y{5C}`}LZ7Tf>HmL|o zs%OIpf~s=&+cHuz-(E@!4ai=$_{mkTx9M1!CPkZdmzfxcgx^sUDT|kiE(k&N2jqPV z%FaqyEz9t5yD^cqR4}{0J9Zj8e>h`wj=DW^RGqw3N^D>u3 zt@rY340}bT1@ObMQ_!{#m&TUKgoTFAv4EcP73Dgq_w9N#4Xs_aL@JE`$qo)H5|cJ40dOzr<}E8vqJ(8J`+@Pe&5@W~;0 zp_AU=$v2fNDV)Mc6ctC4^20BhNzMU0B*`%?@YzOzDKbr&7?b-&j)PLqOy2K4Yv|nC zp9|DnrviQL6hEak{$~qtQ)9}TPkW_mN1}XU83h=tJTR}CmlboC}Z`!}xsx5!MWmk8iVset2y(|M4 z-MFw5P57QfbJP)ie zW-I?W#m4HcYplA%T;z#5l9bBcK}nq#f(w*V~!(h1kG~Q zYX0gF+Xnlkx##EC@uU)(a%wJ(Vi@d?!*hxd^UB>lS$&znX56{(mQ+ZPR*|JHU%pEc z*4pT+BahTn>$JQn!BP&2RX;8*u+Y;C40M2EdVvkIS*_?qwSTUrKp%3-?I?C#vJ zrWj?i2%fqdv|jg#;H1V~SnOERFTtGsgXV5pR$6tzkh{KKa#S&iwv(S`vxa-%0}XG7 zthlVnU3>L6hdP`Eto1?fTQ&le+qEv$1GSsgEzd5J!pB8-=4vdeb5R!|n6x}0+2rB2 zw7JjdnDk7ib&AlysQu6g`apEzgB3jo5Aef1a-Q;6j-^nRInTTgl)4=GAEtF54w~u- zD6)V0q+rtWW&TO^_QQ4?8y=@1(ZxysBc{s07Ol#HdZE`J1`bu+M$(ZXcUnekyEM-k zWR=?H$WtHxQb6Nm%X(Z-I6ZE({!K$lnrZ(s?U&(_8ub-^xpWtNS!tJCpUh z=i0^Qt?xAja;q-AD0fU8evLcc;+V=MU8mWV{R*{K*Ay&P-O3BVJh|v6B}@NFlpf(C z5$8qqA#^@PG6ptvsDT32Dm;r(Y;L~nH%sYpT9RY$`Qh*J6j~DpwE`C>omm*-ww-=S zTe!46$biP?K#zY+kqPup+P!=jDBaFia04pDL|yi>G)a3w*f8br>8CLHT03(sd0*oP z`|@pn*XdhZwknRc&y<4n|8a<4bbsN~S4_WYG@~k%Cz7P2ZiL3wLOnCuaYLKhEpK-=rlj0r;y+@4tyoMwtQTmj3Gii^T zUX+lQzaMx`^_0I+ZFXSJ%X^8Tx}|m>nVb{W`c%d@B--C=G(Ts?t1{Fs{!w@#%MRJ& zzxi7cR}XBn*>@)NkGB+3-*y7EE5#GJ zei(b&xZ7#+tmJYUEBtLz@!Kk6^lj7VK;(*AbDUfb$10-BOzt`SntBviK~X~_|46>lApd63yk^mXN%&pUB5FQ5eu%9a#TT^t{RbTT z@;CugJEh^qP{+s7<*V%vSS32dnukaKjdxW2avn>E?@S`;sj!#5F)10}vFtKU z3)u^7))$<#LUufPzk2~?-kFjID|@+o-I|Q<8$FcEZYw=Tj8dE;NYdfG(mow)eb{_& z^@5@K{QSJR10&1`#bcIeib%N6t=>tuhKhVA2~Ac1l4%iX$|B6w9NX)&*-?|Y7KwJV zv*B;On(hq@yFK&!expmfuo~?k_>Y7yffdZ^HS}7BBxSzAGcQyAL7dP6^~%V8t|5@@ z-;;lSK_rG9>WaFhiXD>ReRW4vljY6l=YLQ5JzPKEXkL;C;SF&rc+u(laHD1GNl=nW z86|fXb-WH}-n~+>v_T+GZr4(%im@dhQ)S%i9xR)c@%V;(>x_iNY{bjOk&*4ohb4Ye zCcg54Ne+BJ1iK{bgQPY@9--dIgoVhxDG^J)IAe8kVCj z@4J*NL!IVjsNKiY&{awyyhhoIKl&8Um3@ttZg5qebFqa4jawvuF&-+TBH2T)OSp>` zkJx-X9Kuob^)9-O!FIq`chZ-UlEHeoZYpC1_g$eRzo!rN1ueN|B}{fd9J&=0I!G>) z$#NK{FMm}i7vinpEZw7ZWy<_ZSbF~@=D>lm@xW8jB{N`%bw)b#Y4jb7VwwFsc*Jvi zTwfKBe`uafK2BssU;ip_hIu0-Xh!fy+!r-Jn`pGZ=-KPjOn+7rjnA&DgH7&2rwZ|k zD-TS&{jF^ly<|%0Lqem(vkv4vztoJP;d(8f84BmDpM92d7KawLWbMjhzUED^=iRj1 zZ+T&cOln?fYv1!c?bTLe6b?hKtQcv0Za+$3A58Mw=OLL1SsUH*VW}38&Jki`d~vfM z9pT`o&AzL5Cy;3%!r_B9ZgsA{*$^r8Vw&%a)+Yy{H1ZRX*HTuc+n^G-nzIbbLun4( z3v#})_(__HH3=`7)8cin*vdWq@N0Q*iJmjU&2pV>8Z}fP3R*S_pRgHRQ$v?#=KZB& zZ=Jf`_(eXONh)>(=SIk0cITdv$$K}dqM+UGGPR>>-OE*NrMNVi?fbv)6|5UnY6e9L zynfk>T8>_DOUx@}okF^b!e+sNTj9HthZUZ#%m>E_Be)k4LXV6IuHyC7HYl67kW-mS z&56wxRa0A)b`GB+v|D(VIeFs7)6~ygeV*G4!1nT5V>S!-CbUJ{76qHH5KNZr*>qHs z1Y>rNq`uP&cCd=mvx>}w5Bg<)T7`Bf(<*+|7aQ|E zIluoxvY$F?!nKaj_Lza1kxgoD$u>RJStr(h{Ip1NG4JkW+g$YU?!S)JxAB@$@+>Hl zgm(As9-nggY2u-RtdkEINl?X)(yTk|JN0Us)Wsw7lLK>5ux9yQ5@u8WM?!tycgwk} zU-oCU4kv=u6)d*#gdB3mk!+5>IiSGO8azEkeiAyAvZw76MyLCM+z zTk;z!TUqq^_je8}knjy9i8k7BV#M{*M~q(tRQ+0VuI>;1FgS;P7Ud*>tNPK<4CQJw zE2c>}a_H@xQ<$UdNpN%D4=_Le-BFwpgtXA^Cs2O*+Jy6;6yg2y5WNhhFC`(9gw?gT z5BR2sM?Q}f{5Wz(=c>EgZf5kV8(D^A%bT@6xga%xKdaxQ(4U)Xd!4>qVx#f%H_0E8 zODa*rA#W5*FamPMDmTu?(zBuP_qBHPbOJpGBl9Vwnw7CsVAxwGCO0H4F+QN^uKo{N zz4UHcT139>`}F)zeLt~=L>tGO4~AWzGdgQL7ibs$%S&{gf0StP?xZ?@2uHvN^WZ^! zZ@_ZY-+%V{>4ro{kAq2Px2T*{$-0yvV-!1pSDz&7 zP$%*+EklQ-RzzQfCTQ^vm8+Fr9Q|y2?Iyn1tP8ujO+Qq8-}lZ$aq7@-CqI8K+W1mq zm+-^s3(g5D1GwEIGRrmN*nrqG<}{ljU9t~fY#?R}8=Dyk0YX29E8gYVj89m&3!jXH z7VJS~QpRF}SOW6Fp=GxkqjpWxAzIn4=Q9zu-{MyzF68|r79WL|Fd~a|(66F8h>tjC zJcGzRR(DUnVR-l-zjAAwbC;c2s>x#TME+I6rCa^Rl#{JA#_ptNg=~ao>K1~&)2poJ zht!*i<{oCUp}x(t2n|MP3L7mP@Q*Fpv1v=n>p36o8&Dz|I1dd`%Zufo)fzZoK0`;| zWEpsny8QI`YeOLn=><_#P`l8Wb2G^dwn7FUpu$3Sl};Xw$1}EtY9A7>yPA_ooBhkm z3OJSb_5FN)$F>B_eOtzp+b8}!xzZFQ)S?S!8M;@}5IVy3% zJ~}_?73=+w$ROWzRxi73v<6xhL#B7Ej`+|x zI)12j$61DTZSJWoQ;@C^c&t%^vr9Nc{2j_G+^z^x_}y0FV<~dFDyrF3A&K<~WpL=$ zz6s4g%_*E4`sE*!up}dN=F8mSEJdBwL<64qwr>~0<2n;Wl+k6%3^za2XbjOcg{6T54t58vZ-v~Mp@x>Cvve`A~Z zE?{6cFL?Y^1hS~(#>k{0n$1l*n!HqZhZ6ZrCPa(&s*xox6Q*^DOeCsWV34H!oqFr7 z)aJmAU8~bN$G^mE19b2uMKFJf7C66}F92k%b+W}jWOHe2vD~N|ZZ&q;y6TR$3uhrY z92PH^P%s$b<2{(|FRW^VB91*Gi z(7OBNtjctmGDZbL+{{%TVhTai`gu4Rdkok2lX0TAmDjhQ#n6*)@xWS`pYNe=9?@Ks-})kzHPsSQr78}^i%!f?Y0ITp+sNvxX5 zRG??td%5cDe;-{X8(wW6zonfUCJdA_EQ}zKfuB&X>yzvG69s5~KZd^v*@Y79(D=GU zWo5ZP&NfDEW@wi?O+Fd^{?XUJbEudk?lz_1aX}QHX+|=9mmT})uW6`;`(kNfVxcr` U-cCT+9-a_&^e9)$h=Ab#07F)lYXATM diff --git a/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.tgz b/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-City.tgz new file mode 100644 index 0000000000000000000000000000000000000000..a86f548d6e33e8e4930e6a3320c5fe7b0a1d0a0d GIT binary patch literal 10582 zcma*MWmuHo_dP5sh$1kAbgPso-QCjN-3Zbh!$>zsDLn$xB?8i*GIU9INS6#l4Kd98 zZ$7{4`{H@`yxH96oPE|_Ywdl*%msZs?!8#BspY)~|8fI@X?n7dm6c8(GN0$-$o_oI zrNG?Dc?3(XwyEltNhSQr(5r1iC6-O(WjWOHKDK-A^7kAL_0vwS&f{2wizNfKj7%@% z$B+0%Qqk2T%#8CN@fkyr<=4|?ZTM|)dN<$Iu@H&<;KI_%*2sOaMM^nguRW?0u?dVgg4WzMS^3`bG8DT$rknI|e_`v~onUWTcIwTS+tL9!Gvr74<5*WOTGHAaXtn&RUh1E#FI#$u$PIGgwJ}+m z(2sFRaAB#-soNR2{b({LeY6v7y7;(GC7$=)>wn`TL6kd)`!ES)7N^Xi!{G;$fRA;u z&8J*2qL?C3AA@7I%?2@>!N0H?Q_B{Z!AG?$d-QRBZPAdB-EdN z{Xq;N;{wA%o5hq;@hX?^{o*u`83=QY?So0ITU}p(Glam+Z(`~sznmn4Fa!t|KSUu6=gN!N<{2`RKgQ^jt3xQ{)4B!3o(NN= zl8KErT7v??LFxm9>=HQo{{$c9$7B0DE1o_`>Kcz%URnGgEJOuilj@?zMQ zX^?7=St3N-dwA&}{*aq+-74nFSo5bPmJz`Hb(rDPx`+4g+;OIa*?@)^e@YI`O>F!h zuS#I`Kj;2?nvwp`d6$TLcLo9iMu5b|XV%m(-SPeYP=fD6|2&^vnoBzmcK0)%XgAVbJQEVPlE3;#JT{*nv{t;IqlC)?&;_SM>>tceY{)UW8f5pOPzMNaI1wB|~p41ZCWtfk;$$ z(`}?<5GE8|NwCB_3OK7pz=fQSjKR@ML5fG1ON!N^Scq7^&=#cqBT(`=rp=N{xgX)?% zjsyc{qhvB^(N$#Q-1YH020~tS6v^uZ63k_jW-JP*u>llF@EFAS6Bh?jCU4-ctzttI zVoxL`;ypGYG^jREXD*Ob-`L!rz`q5N;X>0o`DJ=2>s{$2=%D+^xh0}uvI#ocLvV+7 zY<%PKU;=>gWw`N~f^82QW&@7`aB62Go09i9IUGX%ykrX|*%{`sPH%B1=T}#Js0aZX zvDLX!kz#nq`B#@pLcxc-y0Bd()W+gnzZ~wuIp7!#>L9$D(6#p|>flsZ832j|TndML zrs{f#_Be*~9LE5%@b0v^0&WnX!T3k{;gH~3H~kn0>fXj0{VF_&`L4zg(V}~6@2UgP z=48N+jsKn||EthO!@9FZ4;c~kfJvfd0T^_vn;6hswYx`~RZB+92BpP=b$ZBWKm;Tj zAYc<~Zl&Kh#u)PV;0YqRs}E2C*`g-DV`lJ^LI{zJiS_8|dIL;Idm6&m^x(MsAsUg^ z=|GIRq^AIst#ShntX}KNp!+0G#GhW#Kf+x9PCQB8V}sb3BH$! zt;}sg3(i{neq*0V{^~HsTZd8%J=}s6(v!d>J&5iC!Chb`S%XTUXURX1twDtr+t3dY z8o6T+FIh%UffooCxf00|16lWE27)&8Hw@dgzl)M3M&|?MNBLKj#wJ%NO5+2>mj&o0 zOs{jWvi#RuAk`Xlhx{5qrdXrZ$3n-3uEYX6+zF45fox%S^7|-~=fNE>(XlyM3~gkB zTpqOSwa5bDgc{9Z1s&Um-${McKK6nNzebgVIeTA2NK5Frau_heHN5Eli*;vlOw>!PgS+1YnhYTUC((sgKXOze%)AFAgyG4yxv z)LH&9uqSu6a04bFZ~^XyYbKp{w);H`Zec*j+Npj3Y8)Uz%D~alIaSyt%M6z^&TD;M zfVp0$1aNu<4J`*UeeO6pgYiFNV(}$LaH-bs0g!b$kJq7dzktrYw*eC8_DxLtF%M6= zDn}-8Ft9hPo=oUiFi>J0dzJsL0a1T}0_OH%d?bolQfL+~;iyUAniTr;xe>sRisW6o z`MEe?#3jor4G{*WbY~Bo2T&qu2V4yK4NS+%+3SbL8Z^lBP~h1LaNq5QgAzs3b)3kL z&3(vX- zgy33i42(Wl0}BI#2K!Ao_ptn*J8t&aC=%#(z6>fY70}y|`IC=$J-rzxr5To|?XPG+ zjM=1AwZ`b=ADJx>qtZIsOE_a8-!qlgp)@~VY(Y-6nSrHX8cZ?(D{T!bwu)pr)-d0k zHn!e;dC78fSwn=`%upr9*jg~BV3ZCa1oyzeehP*%_3b+JaO}m2WGo!g`%3u`GH6;m zJ>Xpa3m?edbO%s9anml*ERTYgBOhWmx!{4HVUohx0K}kH=db&i&GSy>b?C57>Lp9p zF90SWz81a)rP&A~LTHe20id3_wOE~)0j63afa`n9R3L>qjgRnqDzMD``Qk#-uB8_1 zlEv*@Jo}P`g&TM$Vo_r-D4_A)>b>`K0KKQ&N~knHZUE~<9qyUB25nz-aD_?kamAuE zX0ECp-2lnFN19RPzc>K8C+0xl83ce|UP$6M0xENYX`iD9NYnZ;wi5@kFD_XeuAfF; zvg~jHu&Y)lCS|A}@T@f`Dj)bO$O3mDzJk6#_n|`TNbEX)>BN9D0HP=&z}c2O;K{Tb zaNCpeZW*C;tYM9rwABW;VXRhl$K}D$V8vyWAs32`SSeS{uxk;sUFoZzOVV+O!?sdRU#i%7>~jW7uF z%txTJAAy2&#~owCXI88II%pOPo77e|3oD>J=goU^R9fxziaHQ^ftvJ}3iofIC-_hj z2J}F3CQu2P3H(Ou?0zu<4UWGfE^Ou;h=(3%0w*)02P}AiJAU9mfXo_H=^T*wJ^oJi zUB!?!>ifWq*aBr}JCUJ~ldr&s0}b>iU<8ymAr^|gEWaeLf>zhHQAGj(ca6JZlDzNQ zk$s(_zzMkp;0rnr078$H;+;rdwKVa3IqBjxl*+izaSx6C4fC#W_1-19(6Xo^R!>*G z;XM>y?%p{z7Kqh7sWx;F!~E^T`l=?GCThqE>pV)uND2YBCcB5Z-vNFS1__l|amCR^ z`(fiTVDLzI=ZW>6kw`4q1V1;+vMR~_$w*Pw_$z}Y=CK3Gj z0}yckE_EaL-_I_o6b+3GOvd+nofvHQ(#M7m)-1j(A(?u*DDYU(%Om&MF>Q(1x@f^z z?4Mmf<)F_#0F~VsI*%R#yjk!v;SimUcpjjJ=EkJX76cyvmy8BdKO{r2PH$^hW*?p3 ze8$DgTcW%_c76u%?Pai=m4dl+sJjKcc>3C)ra*Fz&+?i5~r(jec2 zU|FEaHLWIP?l)TL^4fHTt(nqL!3*boFAuJw9?r;RYwiJ@8>9`Gaf)k2>& zgPqah1&>{-#~K^yT%DI!W)>D^S{uHw3A1*k*7WHVXKiII_wCpw*(cfgk`233@cn52 z=CxQd+R+y8WK-_Qv$CWBAD}(=_7Ebf9Cv#&*eA-O;km9ZL{+0P(WX3cvzsv3i2qgY z3g#%ZH#PtAuOry3DkYt*{HuwL!1Qd-+v?OoLoe3p-KT|5w7)e}+sT>?ow`(;EEp5a zvE96QC%}K|xE(oMFpM6So^U1KYsUDi*gI_F4)k37{+)*B>r*A6O%ppEmHOAP0W4GY zL*NuywO$pMJXZ@q3NyQ>-a>kTz^C(MFKC{QOFZ570y*x?Gs>fu>E63oDY|l@NnP% zjA;K$-UPB(Fw#+VkeAw##_?&Aq@zNC*O+H(IbTY)K6xi+?iW9nW?dcq(yvH-r|R$J=N%tmf4) zy(vfCs~U(?>ZPLhC`wl_f;xvvTR11(Dwp5ZdUQw_{!u@JB%!!11~L8N*!$RdkywY`}}Ps)jpwgIIRnrE5!KSpnTz#Y++Sdw@}>#?cN_FCC%s3x3F&0m^50?Xi&ALcde z&lXl`Kig0bo7!vAon|Yo%l@=UlpTESytB~#PwzrRT-o_xiUHd~ z!aQU$S#gqqHVE-=PjcX916$b3i){1#gzsLkR1r_zlOi5pYdyM-mf_3ziWl>CVGXRD zUwRD4ReoBr+9^*EPWD82QatW3?3XZ}bxN*lv>9kmbEco-4LI-lYGGyXvfVOFoc)tT zkzDIMBl1(bCzDUR#b9l9f7SxLI@htwLbfZUtK<9e{xxSy?JUF-O4W~Luw;Wg(mZJA zqsdEInU#}BZuLH={>pdT!onxdR!(*v@A5k5EM1}`Ui4q&uTlP2ZTfC|Lf;<2bxP81 zrW|^`E1qz@?a^z*=gkvw5suW`=Fl?C@LUt$<+wG?E3%mO{8GI&e;7n}lIc&h+|Z&= zt-*#r>S|nLMDJ<6m!0(JqNudk2|cKw7KbW#OX4|U6zuTc+a-jg@izVT@2jw_@>P)K zn81~#zjSOI_st#uVVj$#%3S2?!(MY z+jbS)DPU#;rJ93N>?d!Z<;xQ-PP!{}U6^G#Qm)aokaPMrg>Y|#Wb##dUyTM_h28A1 zZqQ9|vW+D9Axd6*UBM9_@+E`uxdCEjINtzrdPy*c=2^bVW88WT`!a-_$rr*peBG+is1 z$=f8?R>b5f^Oiru_YTrYi<8o|X)4oq{-&8xCg@bK{T-Y+u?4)*jeD=WkgD(A1SN{-Wz9=D z8u&{pU>0k+WDfprt}oLnl6#9mNWt&*!R?=X+Lqc9ko;hd=QeQ>p7{)K$rCg&>(3o@ z-k6v_^ULlJ)hgSJZafIM28+CzU)m%UOkEhzY`=QQ95PrKLXq?PyZZ~JH(F}@Y05(z z$$yTWElP#-^Gk&V`k$%^M>>$!^mOx1z5k@eJ#V>F9%sC@_Ue>sGH1HQY)9_B&vb!S z>$65=2~FFZVyA=jF}S~N@oepUxyf=b_oV)Y-|I+)wsUKt)OjVm(4YQ>Qh+L09~85r<*XTJMzH$W=kr7j)A6q1sfd!j5vdFGnO(5))@gHwT6mck`z62Syky>dRqLItJ<0T!(pQP&h_#-z zJg&H(LzP=gTtDw=!(Qi^bgP!hqjD34^SzM;<_hh>7s{dn@Am@(jdn(h1OKT`%yySH z=#F|=Z(j{}R`FL2(!4bBmk#Q$DHYVw(HJ{f29<1s>+}EAv$K4PE6eZL-!p=4TJ0O1 z+me{-MgQjIp71PvLas~s_gP+vUzs!C9BsUxFP)IGbDpbqrDL9_xow^3Un;LHQM$+) zB*!s#C$>7Fo9FMZDQD^(tI6z0%8riTvHbG6&cr_`uZ%?nmIdepxO;dFpVgJYy)A!V+0%5`~j;E?;?q=~Khfp6Qmn~tkxnL&}kE2W}!;_d_MK^}XUHw8#{hwop- z$A0|w+LgDEcQLK`;6g790TmN_1*37vYs43HTeL22@H%DTxJ|2GrU+yRo#?ib!rMd> zl22z=lwa!|SWN2?G4ZaSW$dH+7Bh;2d53$lT=8kasBB!ny!R*1eV;?6Qe-i-((hv| zhxT6wfzq@G9bP}Fj6(y>4KIN0V-19f`q#opHFxC6oV@W^&+Q)&OS zU|D{r8|e{b^z+#I+Ix)~x6%123=MpWAKfMEb>LVC;hV@Iv0AnSrVCVGcT_wjqiCS zkM+bnT1;Nnm^avoqospt9aZ>K4_znJsyh2^pz5P^Gy7Y98-k_fIl{1&S-4`EMB*8( zXG)h5Cpv+D1fKJr_6~U)1iDS_UhxHxJZm4;;g+`A$@c+1*qtS8_!|tQ%8o72-M=DjANjGqggu0_-tQ9 z9IrV}*DYLqmhFY{&NBI9ZF72Vaj8d%pATm$)M{)_I0G;x$)`C|>=)qlUgj!+Dk>T8DDP>7neV`7OU~P!>kK3e{ta ze`3E6s0-bCFPUMHauN~huqSa25Q?T4in)@>ybBTP$QqpAlCnoe9L26=ItGkSX7Mb0 zer!`lZ}C-roN3 z-|iI5Nm(dwA?h3PJjgO?p^Zv3r=l(V{=Q8l3mJuqi#Lz4+2Y5QU^iuCs`X^clc3j~ zxm$-pdTU^+Eq_iLsNyRO!~JT9G9f{)1no{U`^r?Zg^1BeJ^{SeoSifrUAzw!obFuiI`YGq7bh{F^G`oO%c{y}60=ioj(kedQKRAzC z-k7UxAXk4UDb{s5kg*SsqqUu~5D>93Js72u7XW$p*S=~B8Qr2AI^yp#xAP9@g{tNj z1a}qnpH35RiLuP|j`)UG{8@ZXI)3;^>%qB8=d9aAM+3#k_QLhBG>J6L_`L2-sNz|~ z(V73H_3ZolK)1o$ef)nG-|6*8 z>jnwZ*_+57HlGSXt*Li3H<9w}m+=m>+twYG4Qi!J(H$Y$Ow+R~mJ4o?(05^=?+H$FlFmM9Z>{-%%gkzfl^oKF7}%wij-vIz-v`cMaY@VR!8p zcY`LqAGRCrc|t4Uc5}b}OlQJ*V_72gQdQd9?@WIoBg1P(8 zE3GHH8TWq%45>b<2|O4IpFaBZY&p$n5fpbY-@gA^16~W>lKO$fo-Z4XoyzHO24_1> z70@CIB&xuUbv4WwEb{9Jx(_=Y1Kn4Os@}-3IHm99vfsS=Y^Ih+Gj7_)BZv0Ax-3KE z6eC3T5mzc5R=ex9m15IEgN6HNe=nc1R27!*ljiL;$~quRh8M1_-(r%$${PrV|?vz^@=QyRi=(?HH!_a8DvKw|! zei{!Mlvs>;tlcpcJix(@uc5Q5l)hb@B&x$4y5WImt`>P- z=IVYsx{du#a;@U4O`M~2Wkp;LCm3r=2;l4YjyF!MWL=V3S(NDxww|0w+S$(czx<%s zWnvo68n{aScCA{ukz|FzxemQzeLwblg=WPYF;-5xY#gsQ)6ag?XbPAm@tPR0Oj~OF z9Pwo$)zPyDcYZlvt;NsBfl%hzK7vwxoehLaj~5SDy(DhfA9HG3rtpd-koV2e`_pps zc>h&(f>~5i1%!_os|IY)c$GVbo5}j=>kA`~Ho~uBc(!o&ve>$+r*kZ_7g$F1l$4S95~S zJ^O#izOM`cN#ADL{5E-#KjnKi=~7K-@9?&Z^&pX*C;33Uwj*CdgM~n1vE{@Fo}Vg; zbv3USnmFCwt3cavvA5_d5lOzUfjgVs_)%~lEOI1dAz)R3i!TtP?)O`a0^Xmhhao6YTm$H7Bunp=GSD5~FxV{v8d zt%L0I8n(RL?~}RtDkC%-pqz8fe6V%*_(gZ+0+L?DBR%X>c%JCzeO&LXnKR9q3xi$m zyh2kap&Oc@_5e4p{tdD7wUu#~HFkkHMCSEEm)E>d739*rAyaB~vma>cln{{SYIAua zsREl|-Ow2i$75!ur2E>GGX)EcTBCB$rU{M7ooL$Exj9q@cl||+ubd9zL zHt={AaTcEjbi3LH{i0qr(=Ie|_r}rPZZ>R=|`UN^h*-X;@iu?N`a9=9N z`&*t-K`(Rmc!_R1TgmTzL%sYsG&(PGXa7#*_?bqrEcF_}WZY4Z=!#UHEZ z(Pn~GJ6)J7Ne^#MV$>RYws+w}qm%p*ggaPvVzippl8{~`IMZe3N=lTXns(^`)3hMo zC%MatpY)owmi^Mr^cI6WmkgZS)tSM>97HAU8euRbeXs}u|5W_Ha=x*PY6wHK3W0Yz zdf)^@ief{svDd{3|Km35AK2c4;*74Jqnv44&Ig1aCxfrjjWZEG%2D) z3-@Coic39wZO&#xot3gPS=~A@v5s|%dcamp>~%2{e$eJx!F)c!k4l?X1K<9dmu~ao zF+`^Hj+OMtw&iO6vFaY1j40XR>R86f?C3v5(Pz!m`9(Xs!huQue}AAYk}wZKv;;rj J^S^iR{{eHtq(uM# literal 0 HcmV?d00001 diff --git a/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-Country.mmdb.gz b/test/fixtures/geoip-fixture/src/main/resources/GeoLite2-Country.mmdb.gz deleted file mode 100644 index d684ca167e8fde097e60d229106cc710d1cf7037..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8470 zcmYLtcRba9{JlsjtITX7G9o+k#+8wAMP;XqkUg(`&5+SW;ac}&WK;-AHYu*1y}QP> zXV<*$`o8u1{(irIUf28iKCkmU=R9BceIIXpiI*=w@dQ1mpwe;lG;r~Al(_@;^!M=d z3A*#x(brGX-QB^C!ae&+DEF`TUA8;}E;)EQ`1MP(+?O;N`0E|EZs;p$W_+i6)MaAD zDKQYwLEj}T@GT~paX(V3e-Yo#uHJ#oyDQN1`hM56P!1eU|EcfckJoMtnp5zwC$`BY zTne~?s;0(ITSr@Fd$v_wT2ZJFgl-MDh>6zHi5905FNOtG4j(r-p3_i4&rzm&$5V(~BcN3!zmjbs^ z;>3z?m{4+1anOp!zSHOxV1?gy;4Wq|q2^#jP&DwE&^JVb$l8URxrQ-gnw$Jk&E2Ru z`UV;wDF-HuI*JId(%2NRjr$qD&Zm`bk#SxAOaIMRKjo1_ytlHWYK7HZwdb%fZxq$j z{_BRZU+-HAT+vM{tMFQjhO<0s{5@s#u0xhD${<>m4)N@}?EMp|Gt;w%)rSfB{vw`AKfov8DW)wdk!l!tSKiMCGlD!QkB}`)ykOT zJ*z=tIQVOnBiHalIdz-BRWX#T*3i<7Z>;MObA@DXg@C2iJu4au#_9WCug@#8dQ8(z zQwBvke`*PsIq_Nx@F1uUGvw|_9EPMivvWeh$M=slk2Q|jm7BE=WjGmGIGf@b3jbci zcrDIcFD#0!wZBH%o0X_y3XAjQ7lE67o^rK7K`XLc1LI_^vvC(N=&M5n0gQYhA1kM= zNH7(YbR#V~H-c~xy9Y;VhCf{=`4*CI=MtmVGrzAGqN=89Aj>mV0-hb#M&{%mCz#;b z9et_F7Q;i^^-1~-j!?hw7zM9ZTtbjEJ0GVdjYzI{$f@x%v`#6$y%>39zf>wqHrRu4=D-E3&`iBgjt~e(_g3^bb)kb z9})rkSEnHV6cPA;48ZQmG=PoL^xy5t&_m{&t}R-1F8YSZe=RPM5^`vvHwS+@>Hb61 zvv@T4d&LZ~bGF@IN{p?QN(Bhb)L$WP3rA*M`^S3#(el>k06IPn{fxf{s4WwdONle! z_YNtj=JcTfZWu^df4cRE-Dkok))kumoOFfH=|oc)U;R4CE{rs;o4h#toVqFvU{(nZ; zA9><{cW2OP4Pan7jEUo7?Dd3>a2jHmnWm695V2TZ&>MFoR;6FI^!2 z-1@iaf9Lgo=YRy-|E0ZrKKQ@lDB$pEgj3-lC{F>;f6)NB9%`QP6XEIb3#dI!(F+9T zes$appiH=qmBc^*N&?3}6$E-LFX#ia?b)8tADv|oKi_>4`wv1oWBTber+!dSw&-a; zg5}y6%tXVOtTzT@Izsv#@nO@nh$09WHc<#ze9{11N{c>2ES+G&8KjWJMX7S@0P;*1 zz+NN^ohq{YigY0L=<7wnuxY>L_aRZRPKp;jg!Cdk88XHzZ2>@G-WLo5RF(jh(nzHU zCZ15E3EY&&)rJagf`Y=<^5N2luknEl?060Q$qjdHD#L%km27dQWc^d8jv0YaqO_B*7N!M^!96<-F{66@V$k z@eTlaool-YTmt5Je_Ak&uyMxYL}Z0JK-0X9ug@Sx_^j)rdV{xf)hKb7dOX=U`C>z9 zWzvOCMQ91M{ngX4;l8?2Fds<=-iw6v1tbtJQsU4`x27G`aHA#$Fyk)RqngI``#W6? zcMWAt8c

t9>Gb?6|yuzFTY##OVqo!$eT&3v)SF;Uti$Ur8SG7iu7-s|hLAHb3;C z?dub2I8JZU^&yHt;d8GUW`}ujmVk39OQl(*#4S+6TcJ%*!r7i%sLK_S-u|AOgD1e- ztqV-z#lJBPdJj{PdWs6_8hv%Xg&l5UA$C^Lsg;_je31$67Fy$59hiv?3%W_)U-bO3^m*oIrqy5JB zlX4vJ7rkKY8HAobySN?XAFz(-2l-zST}3v+0kGs{L<|w)@4fz_jLdQhy)M(VbOy0` z@CH}~U?MUTP=gZLf0!gPN zvuw5{!f63=b*@|(z@!ejN-(XDt&D_iCUzv0X2Cwb14O-t{~(o$F?6MA5&_AHUu=qd zzk}bgY=WpsHgr_1$Dk*UW!|dL?{5L@_&GJF$pW%b5c6~&GX7ef`v5Wz251-1zp}nu zwRmKBwdKwML_Q6`-6#*13s;l-?Vt-rRj5Oqg=3W}vNeaah^4%jiYlqR{XfQ9k3as=$78iqF?_DUHDQjL3)f>f>(HLWd89{v5i zKyK10rQu3f&z18shPf!xuf=IQk;H&VpY?~2h5aMkIj!D?2yNO$Ka5@k+kp?&%XQHBG zuuU{Lo%RU%qZ=2K14uk`bXEqodg?YP28;XA1K}g>zXLihn1m*73a;2u9wC>%0-5Y< z<8p!XvIjp?8zfx|7TyFMb^lhhBW={`0ijc$v!AyJehVBC5?l7=1!OSjkf9%RBmhKD zlx68zK$iShC4T@x8fD{3Ap-Oy7b$1NCGw%mQ%Nd5Ic8nPTggO-n$YPkCmwnBd13vG zd)JHAWfj?)D|Y~~eF$ulx=BykpapkE!Oqfto3?|(EY^)yk&-~wWq(h`Koz*VSWtSG ze0WrhfSqxUEFil`Yb&}ZoP^Oy>XuJHO?c##2tnV-odHUUG+5lsg6;<#qTKbb!JjUahXji6l;!ZP+Fr zJouw|+?EkI!G7m9Ap*%~5PSiYhV19-%)g5CIL~CBX)}Td$)lsbegKgY9ymhgKLBJn z!C#U#T#8eZj=%*B3&?!lU|>J!A!i%eWeny)Ebjo(lfsy0F9FcCQ~;_}JfBw!9=$>s z^#w0>5k?_t+Qd@iHaiEi7FUzYtH~Q>6&)a=-}xg%M0_2N8qUWqootkZ`~RBqoZHFbbka{3MLF6f3zMdg=lLsnU&x zoelghnrERV&slcJVA77K$Xn3H#YkavVfd2ATpC~o_vr#l{UVBBsr}QzlMzvX4!mK6aLHN-2^cb zd_UwnZMONMWC<$Ks{2DKYU`9k`<^#~7LbZ^_ws2-K~8bXSunECg`g?&p)vWv^JbvN zPA!t@bgKA9)s)Pd>U0XF2H?=3b))^E#28L`a-}dM;h^&Bbkx}p7so5P(6$8C&bDDp zB#ieMG`~4XMQ-MbMgtx3bV0i(Z4esY$hfm1ymeW0Il1JVPBG{;>=D+0Xr`IJh&Dq3 zc&RwLi{$2|->%tbRY|I2uxBoJ=gvPKIQyBB=QZ-n$-?lpa55KPE@M8P2)I(=$j1?@ zQdAY)%18(8puI}tqD5QTpx5Qn*#31}K|o(SD6>FKWFMg)1ndsQ8k2*P3)nlqo@8rE zz1(b50NPJJX+j|TLY2en-7_1g0 zdxS2+G{DH%0b`a381JcB%+erGb312Vunjk9wBlu)c3EAu1DH>Q&W&Fq1A$wy1k)Vg z(K6>SeX#a-G#O}6_(PU3s|o0uNCSqm)&nE}fKT>pCt|kcj?miJb_!20iVY}e9w7mQ z4F+-MNdpX_UTtiMh-cs5d-%CUt+Oo*^MX7&%Y}`>pGXB3Exd)(@Y}$n;b3?tToA)b zA|}Sc$aW-Ja+S&q+~8|Bb9iA(h#RET>Z37{ZCc}#K!hj%BYAwC$VP4 zKx~>#{4eu8tUkfd!LR4tV@yUGja@wy+WLDwsJO}33F_noT}vDJc07DbK2+^}dGDQ} z9W`lLs038CzR6GfU|pKI_bk9|W45inSpcE;wX_7PGdPrTww?VZP}#zB;I>o(oud68 z|57A&IU&$g9-o}r8(3H_J>A>&cw)bz=W@NG@bu^2Y|No~ z$vEEtz1#CA(C)6?6!s>SdVeT)lEItij*R)t5U~@JU9U=UXne(Nr$Rv9I%;Q-$I*sG zFEh{jOXS*j%- zcGc43VUr0v(xi2_&6p9tkk8VHt~kRSshqrS_R-bOl8Qabi6hDU5TAMOttN5BY70lV zaoUDo+jn>oIf>fVI9*27r>euj+sbsdmboeTD+ahSCEoMZY4X^|G=dwSV2|XMHaXO&#U*ho z6>bmwV}14feVcFnnRuDi{_UdCz~iJ0ge-4=|Bver_PepWDfBOQ{Uw(@nfANma{{-s z=d|;Q!kxTpWT-dRvOQ=4Q`pkd&34!t_}TgAdI|Spp6U^%0x+!E^dRyZP7Z0iOc? zj>=Y`WKKf;c>e|2lY13^S?%{t6nx(fe2q&UHR^W|w51LG)<@XG-d_GV@5IIKTALn% zxxw5JnlbMrT<-!A?yq0BYAD}p;886fkXI@H zWWQu8_`c_z=gndtJDJYuxx5nB?yTr7!AO*|+;pgFy;J?nV|8hJnV=_oU3aKlGmdwj z*@1rN9S0Kx)eKK3R-W~@i+57rao)?szb?{dFZ~mLs~~*pm$d!KRmrVUxG74L00d*tN|`Z?gOr+-K`{1z@FgvR~+S;j$-cJ`E92M==Tp1Qy8*)}=J+Mjw{n9=NCuG{VSk}&3 zUF+jA^ud_#F5H?5P+Zl<|A4G9Oc}2#InQzKC^c_jnp;|Y&9XI+K6^HPR>Sdyvy?_R z>b`e62xl_)x~HWKeMwck^GCfCB4A&lFYbTr~k&uACFkZ$o>p81b4ezi3Q zpQu0XW>z?%mKQ2qLPJmmYxz3opHEDqC0&TM0`h29$Wx-fM=e%&B7*ibkx z;PY+S?u^P0p%hpE4!NBd;APo&{qELJiDu~6wNv|~u~ycmKy!bK zh;*>W=YZmm0UE}Zd2SlluopsY2X*wW_3|=cH-hk$0ix4NQYevD@g~rsfkmfp1bFGE zb?x2d_n5Ok(@n`d`UlA85ulFy?PWP5^Mhq&R#WeEX6^5oyT8i@%gXn<<@nQ7+J$`^ zUNwJ)tYEBkFKWK_%KKQ={eu5gL&;wCgWONSYaWj2hh-70PPr;G7LVBcZ9hol>te19 zb`PpOf;zjGdx>;q>jl~E;K{jigL4QQPyEV;|2v+#c=GO$8TA3Q-Qw)m{m;3rXP<<% z_9CAw?_rmx<;ykO-({XAZ6*~sJL=7O>Xn#CdVGAiG+hyz|CW~&^NBmFzj6P5d*k14 zYwMc2Pki}vC|`>(OEEhkM4yefZ93nrX?_!d_c-nIUTc*;`SH8a@)-q=e!yE0$J+ha z)HWH>0sJCd`_`l!ksv+SEt&F)E4$W;um9(~Wk7~r7*B^|dzq=n-SH05hCBW625v%J z#QVBXWpQZkWLAhZbys9y2s)k6N_(k^3y;2nx$t-K#DGFs<`37((rp>BUUd)pSy-|~ zcg`obOffKNMaACcd~iYuZCuBQXLlL5nVflAi8 z-}79lJcH3X+UDrZhe8U^qwlciYhPg3L(lsye0h zGoSP+=M^^JlL=kkWXM$Js@vfB(dA2@L&=aNEtNuGxEJZ5g?rk=yG;O!id-zaGZ4(6 z{({GI>1=E~l9jM zadDsisC)G`uWPE3R^#Uy^k0rG@sjA4C|1>?(HN%}L|(OqjX-i?o?CqNlF9|D!R8kw zvi}v&J+ZrmU<;r&DDx7z#8zZi7xX(M;N2YUw+gE4% zO(~c|-pgZuE!3{~A4ijl@mjy0~eAC2xwX+1GP-e{rn)8{ZTk)cS|XPWsh}7msqo z-63}0{%ujMzdKYAzP_o$r%P?a@!{V}dfnUJoBa~W+5B_Ebfu}&FX%}? zsglBzl$HywXr76z*u|S(WALHrtA`dwav`bsLaI3S?7#I~W^^2Er^=JfCm-?3%pb9{ z=H6}B`wY_$#Y9eb2JXr8uZ-C-xPN`xT~ex(`E$hlnrGkICIaW2XUx;LYk=R-)srB3 z$qZ1EQW@DhY@Uiif$Q$8_E9oyq3K*L5$vU z+l>pdSu@J@DF=okAsbl7r|764HJ$!!`}N-Y5tA@uN4DYU zhG|Fx^mc8WLUWwjtHjG8S6pykl6OCtgCYGMhH99XXUh1F{K|{! z2-EL7V0afj}7agR@N_}H>>;EW7SnCj=tXxpqH0_b({A%E#cQaC zpR)#oMc`W6=re~_Bhdk~-{pjxs@YhZFk|+aI|&PHTYsFzY{kZ=4AIHQ(M={DkMHb0 z#e35>bgtk?4Iu{MbyqOk*8H3hylZ05fycC3>%D;v-b6)CXihx0nQEtk>T{|uBYB41 zU3#oWwN0FSU#2G{bf#+rQz%|4&5X}C2;>GI3-=7(6P%7Ghv(U z&%)gU@>?>#<$2vwyL^vXj4H>BPD0VD6h}|3TJd)ovK(`iFt<6E&wQZNwp*`nRSQ=Q zw$s@C5IQnrUu&M}GKEn2!I#X%F`9e2$4gg}UYh~D7pdG0>1f9;$A9syx_qkTfwT{m zQHN|l{shu>3vn}@2^tSLg1;K@AYQJSB!W;z)W`Gev&qYumrmxDW(GXjgPtLwzFzvo zcUA{EGamL5$_h==i}hV5CGp~XTOwXrDJjV!fl(j6eCOiwJu&wKc|?cQ+01TpTBGw$ zXjB+8sZGp)x1dO{uzwy@0>0>()!1#IxCVEYOFg!_x$JIk<|BZ~nN#NVc7G)=i|#MS z2{^q-M1tz@!k-N3_CgRviZdgsflgww>L+svBEC9RZ!sy!gFEsnY(u;o^LPm>A5zv% z71=vpjvZ0sg|I5UXHDNCJy;q3gd704Tm>#1Xrggb*9qd$zGr9l5K`F^y+R6h}}Xs zBfTK GDEBV#iF6TxK%^K-lwL);7&=HN z5drBCAoMCd5R&)dTX((RZ>_AE?>n>4KKtyMOjdYeuAIw!ZSm~fg^hIoAkN{WE!b3G z%#VLA@Rgi5Xl<9rbA^SJB$noik6n3UEu-<`fn8zwka$2#mtcF1sWaVZM~D4BL&s=` zD1M42!$pmELpM=JyDL25rnyckqFe0>lRdVfX=3#@61;Vq@bS=pYcL5$D*j$QX_${y znF(}VQ_n>gMCYEC3kV5H|M9Buv<(d0^*TOr&DM%;|cM&3G;uSJJog`LaFD__}wJyl#6lV{uJ&b_OesiFz;g4Y`7)k;hh+ohkjPf2@?L zCFl3oC^Bj4ghpLKQ-*6gu19hStqCB7^B(*J!A4u*f$qEP=M*kUU-h_cdzB#DwSti0 zJj3b&VlBd6(3%Vd;#-iv+5!2t*7oI@}L&z3T^}1@V9v9ZDBM7eNwK0`j-$35K)#OgTSdlsENu15W+y4&6diR zKZn!cd||lJJ${4^;-xmdt+XvIK~YKu+*w3r`)gSG zjW$GdoV?xCn!gayK}1|aP)D=OB}UPvDC_QBVQ0O^qMoeE9U&&?F(=Kb(s6hi>Zs5Q z(h&W5u5l{j_^E*X1Ec@ zpms#DvLOl$j3`ihQ27tPRY-{lYI_Tb^8M%8V4p89<)PoAFJIZU*JVGC6bCrI?VKPu z33mw82zWT@>P~j+22|)czjXliHu&9x>2T6}aSYW`FoQp6(WCX)0dmYVAo(P~ebXfi z-tu}1A#fUxJ^Tq$pjEiM5=NqJwIJ%vM*6-bSG(_-cVQ4S*G|o6C%2->Z(J#kgbx+1 zb%(t0$a*w58iE(|W@o2lESE)aKm?B&A-)xhPa6kb@lx^g(ax#qze&HMATpD1fvkW> z{~cs*E#HhHE|mxXgKgJyvH}>kgpjsOMyS!;8@N6whnecEeHo@- zmtRD+gW~m76?uIT$o89fG?M%lw|SQHx4d$wXJvOC#(g2Bn@B5BN4@j7i?hq3tG<=q zl5pc_X;KSN0L*XTSSk=)}-EUKoS)NkmjHG zzxo__@#sogK#LQvQ!Jgu@ii?7==@&*cg3|GDqAL78CwwH-!9`W`7WO>1_9*+k7nr7 z&A+x|UH@}`JqoCQ?=AgS*}X;vf9Ax9pbI;Cz+w=Gbr=hbOjICZf-Js#Cc^A zeHRV9U_ludfFStFRhB-eROrQB7@g30GR-#3f^#*4C@dk+LSjHc zAYvB$MrLBt{zPW7`<(v*mm9+0r zG}8aJd(Rht>-@?U?{*Rs+jTrzI|}=m%v1@n))0;!i^#_CSqc3~Qkz#nl&LKu&QUZ% z;2}~zRZvfd#N^?XXevs(b=URleNZA5&0>OSREHg+led>o z6e!m|csC%l9l3m-#1Vc&G!-CdOYmsfN7tuc9D&bWiC!{o)Um$M z+bj6(fXQ6P{Zv>U>31IN+`7><0&Jz2%f( zfm4x`nAi-z;@TOt-~4?k<3CG%&AG+|Fi-QhEWQH+-2*U?=fCI4(dei~T7W?hU}&OK zVInd4^;b6EZ6P6buLJC8fSn{t0}&2B1K?+#@TLGqLg_POCy9xo*{Zna%LkXHw4;Gq zbMbr*4b()><;#m`uQSV{0L>xCTh4vAf587qxDmwMO+rfOY{QB^SqUBqw=`8sZ42(g zo~kB#7AuDVPYnFg$FNLxS)EY$&fZU%kMJGk#6GCMJdl#fEKRe9-kJQ__K>2py~Szk z(+$`i4kRTbG){#Ze!j+|IXOAC5{bf52|ys~GV~iTNrImAlaMkxanB0*fa2(zz+jSK zOYRoc4uxOepHb+#bRCFX?nDDIr?a}Kp9lR65a=@*%gyG_impc;w5F z9AkTF<|-c`G!(wnsv5lyd&2cHy$?KjA1KYCElsiI0^~mB@en-{UQ_3ZBMQ3-(U1Yj z7Idc3CAab;rjv=nX*vMzgRKC2(~VU}#T*fWG1|@XX0(*Jxf%{o?Zyav3kYh7>C6i$ zIv%IOGJ+rB(mH@p-*DG+6kA0CH#fWj30SyAJ9UoSjR_mm1n>=lb!4QcP8JziK*6>x zA@JyRS_MGV5)z=~x$^pj5*-iSK9+=pof%*!$n*S(8o(K&ZgQ5=DOM~v0^fO+*!!xA z->Z%3>h_lMhkp{+pmlM86JLNHx*vajb&=d%&ByqX?7|B)PPACgMMpW)sYDR-YZ3`> zjFmf5;X9VVmJt=ZOU0;xCJCZ|bPra*kxe>!D~>QgS#5bQ33+m6KS_y4F8U&c5c2T% zGjg}vnJO^=*~lJ1y$#DESKdhim@EO`3D9s?;5*vHTJe4jN|e`;f7+-6G)9RAl#vmF zt59P6=az_L*xY>&$g$Se?x~~ zv`i1P1-0GGNrkVf8T3KDAtH-t#i$!c`B(;T5V>$y4R)R~bm_C`M|e><5N&P(-Ps1_ z%TbxFB*QixKv!-~Gc;l!2TIsvKFXJ13@`y=js{1kuvl6KmrLZr@|K?_ebC{1fSNDD zKlahL1%8kW(SHXaVA)BXD+In;$yZ!XE^LNAW>@Sa86vf%C`LC$hTybK6wB}kv54^E zhmlR#I^mYlnX#Ckwp<*N7{dd_~Q)&Ret6;=F%v=4A<*jYl6es7_ zyVI;jZJ^uL8bswC*Wnbk*gh!pS!EgR&t${k<*63vF#GCTV*a?!^Ham>2-oK0{QKdd zK!61~cMR*l2|z2)dK5Fz?pVd1{ba1tsi27Yn|u*7lpCv1C5jOqO+|r3+g$*lAVBue zY0i4mVMHX;QT}37Yb-IJTL?G=DmUJ7lgRna&@_hO>y6Ll0FwhkZ|uUvg#p39=fRm# zo$8VuBtslR?KD4=Vd3gFtd#Z=a0#lw-bH;5NFGv@{lW;`goYevT~vPrs#lBb{VJdg z#eC-2+k(%Z>Et<65Pvq*YQH+oKS;3FEtvYdKQMD0k!@HAyvEh251Mq=n3cepS+@u` zw_y>$#zOb|AJSTr3&CgEPC(B7)Y<(Npt8rNfE{8yGb~EIg_>N*6R8WpxD7bvS6R=F zjl&t%etE#>Z7AI5%V!{B1zh;68xP?JjRJf)&(r*AKA>C^znxXC|6v+;g-6N>h6|>0SYc5@`W-Kbw+!^0FQ1xUJ@=BEC%>TerC&2!pRdXuL+9x zlq`qf5=RaXR7^(T4v1nR_>ILjPDNpv%4FB=Y8@UO`0^{1mE;Lyoccr|FNk@RlUK6H zyU*%DCWUVE9u(7DMpO$DcX8PQfu2{Y`0I!=xBQ@miBK zfPugyj+TNYY&M3O^thL>{BtFblys_jr1rE+J!%aV`uE z|3o%STn~`4qVHI`2fykq!@h12a<})2@-Ip#*dh?mD z&oqUvLV+UY-NXAc_JCSn!)3_Hp(K9hM`|Dj1y}0+ah#74AkY9YzifOwqD<9C{Xq3V z?cm-4(){Rzl@(6OqVPyztRBOTcWgvwSIqDB`9u_C5#PxPGwgJ!OgW_1)Oo9RD>nJUmsMQ!k*BRT762 zmXWLL^iX2Jb#wJ`^|*+lrxdMQKfIgn=LqoKC~$7&`RWtMgJZyWJbo;emBkc1Sz3Hko z5LkG0F}t$Ru*Y!4(UU)dG3}R?Jmw87Z!b!ZZRS$R+7o5UDC#73OCmVA zzvx+UCQjVm?G=QDt>{3iMu@5ys*sqZ+=+_nCw$(pCK;`fMORGQQkD0)&{QKvl8t&o3@8-T0UDMY*}q?_=3np$q%yb zTHYg{v$ZYrR4|aZ+Jce?Ra*0l{}z4A2y={Ks44m$4!N_{uI__ms}RLtSWw{k%8fDB zIjQi8)i%qz(@yd&69!#I)rSz=;2OWA;fDQUfn&JCq8Y@+jvTA8(>upCa^u36(d6@ zbJ^(uDXU9Jmc5-h+b;xb4r{wj5#T2(mlOW?_3$y(Z>#j}F(K(lJD>bhK^ z?L+EB9bAja9WzNXsFlDp(S+;JO|_nPQ*#pg(zz;L9h{Zj&x6`oCB^ma)tDkvQfJ31 z3)8+yVN7Z@8^$%X^2AY9OZI~O0#!+zx6I$`p#!?we(^PsriwF1J8b>W$ON5Ffw-}prc%IN{Tn^g$a>T9^pOQuWGwbh_ZDlW) zrUxIzg7OJU)9Eo=5^Qt*wSRK07819T{%|E-bK8ET!g<7-D7HW$p~2wAnY= zX>!Hescg%yN`30&&XOeT?20#^bzqo=3Ly(MSl8^q+E&Nz{$3ImC`n|3z5 zk(v!nsKAfbhA){1U6N-)NjZnvib_U1PdnVCh#!2n`XDoNVYBh6?iEjNqtTHH^LqFF zI_@S88oD}g;~5)%?vgCzTNK2%vsN>G$HKK^_Z@&@QcxV^_L7&6?2v*x%Gce z^f`;7b4FaO!`eGb=~Mc|*gU%XK0O$Rmqnw%1!ChpM^aKkPX|`a!0xx!cx6t4eVjPM z&7IxL$4D6DY3qnnlh`bIYw&shz5MR+-JUPgN#i1OV{dpQj$Z z4fYaWQu*2=`@EvL67OI3Pmb|#yTd#K7X4xcszvV1n8P?u1usc~YG0!9rt#COctZ1| z>SUv8`71q6OfFZ{rbD(w{Uy()yn$TqL%e*!lh_6)E)koAX{ZWQ9|4}F>?r6g6N_%Z)1&*%-@6yF*CG%_ z3#z{_(J=~T3So!U>lc@qs6!JLj4w@scNn8ePJ||@)HzoDsR-cejw*r!r;PV&x##lW zL)^X5&GBeQ7xCtXGFFto1)@GKr$j>0iDlnhmFcL*t&LG?k0Ill#`e;fb${K41LlvV$FNU!}2 zyN+~l{qyr{#JF{o&ot(P)vSC-=a3y3mFB(ABoi;|jev_Zjk)yb9$%F_p{1#H^2Y1A zseFm5-l);AzVQ2_$Ly;c?oU_q1WtI?Uk5~xft_c_!-LRYZ(e zerc`UcTPUzB0DP42UWz;C;M@wa=7^oeej*TWTIJ|vr?T^%7jykd-Df-tpjCS&a(OA z_M{NDAPQYCT@&Ne%0Gs7Nb7&q>u;iDnYA8kZlP8xr}eVt3WnVE7X<SOD}vt=kMgLV(Q${iTFEUO~u2aJPMo;8aIESX;Dx$Sb( zIQfN*Npm=Ge4b0K+s^RzWJbc0({$Pv;|W^3AJ$Q}6z!$={f1G&?!P$|_H=P!#bc5d zi1J6mDNnL5d;XB|^KEYJeL97#{i+=Qu&MNi@jH9)pzGb9j|DgFHGPs^Eb_exISrTE z`GK#pZ(d=^m3)y%5SB}rF6|x7nP!%BLdIR>Yh3nFfmMaoRzNUNg~9k21;Y-0xQh&1 zQwilEHV3=Wc8x30RMT~Typ4E%lLE0q^4IQOk&Wk(nM>=NiLf~B((<@l)mMwL5+e_k zh|H*-U%IzlnubMq23>FUZ%Zw)gvI$wrl!|-j(Scm74=}N4_}svVK?wW%4JvEaFB`J zJ^A+SH@;K?qz_O?L4WzDC4P?ph>vJFqq*_R_)+9POHTBBEZIJcBcBIB^FdObeFthc ztVRxxK4-f0`h5>m%q&%~)8FQxSLpdYG`djRgBta+oN}^x8H7nID(Q9|IQiZq#rkX$ zGxpD}_O~mX^4qr!Lm)q*?WsMia@=MFzEkwZnJ-XZYQ*Mly&eqC5RoI=m88zRU&+%C z7&at%i89D(D^xFAGL=>jV29pTIMOoBvF_9b=tIo%hcqkA4T5__V>&kP=ZqEvWFofg z3)U20qyPMruuM1R9K7=Dg*(3WlTF@=$Sd#0u0m|zyN+LqjOc(^GK3@vN3XVvW%}ny z(ND9Reo4h;;vabBK)T!8IQ_U!=9VLGTAb&!{2FzlJ~li0O20-@8EdBUnlt4@71HB3Cn8L_Z_s~x*qFCg zphFxk*9rJ{h@tew;{l`M8LVq}8PA#=i^bQNK^d5Vj-|)<$>S((m;@^>XjSj{KWxC1rf7Dg#Hw5?PTiR-o!unUl zWWrEg6wQh3(sJ8%AD2+>*$n&HXx}(<(Jt)5L{Qpg#n>=?mG5MH!YPb;4SsDJHb<;Qk1W1uJETe0XjlUkd#u`=mxD@)HQl~Q{H7+F|s^q2U?}BLk%yzZV9W-l9bxCy_Q9K0F_g-C>eT|*w*@(K7 z9UUi3w>8GQEqOAS>^Rw#y4Vgkgd@1pFeMIuNR1~f2P!^St{s%eP(iMxZH_=brVIO2 zOl=9*jDbZq`pjekwQC%}+)?Xt@C#$z&EHR~E8R&3l@|wn_^Re z3u^~IQ{~!xaG-2q%(tHCU`|Y7SG#FCwdr*vCi+oe?7hPaLf=eKb^X7|#Ho)>QuFRl zc)hvC8uH!B^er8@>1>Y@WNO!D?YXW;w;KkhfyctV?vlRl8m>`K%l%%$adG*hiSe+}y<<0F ztt}L13^?BTe$}B~8degu?wp>WU86M^yD>1q)ZL+UYI{5RZP~R>bnFk1gNX?Y*FKKo zIE~XVEWRGwlokQHLC3?GYBsH~Gn)vMp z-`>!xg8hl?e~aByX}o2i^m`D;y;*)U6K+z~5N0`$pyMOBCwB9i+giPsK}q(^HWPok zx0&<+3v}UDhU8qdJx}jK+4ra-WH6Iqz7%uXMp+&H2<;y;L9v<$Uwi5HCC12jI9_)q ziR`vI-$dSmu2jzV!zn98q8%0ru6xYW%Tn_l?|BSKThLRO^XXXl5?{K*+zTj{l#WOvl-{+zjJy5)T9}C%(v5w8dO*Kj)-9|g+q%%2_{*|Ef&pp zP?+bntR8VvC`{(+J-UUzO3RIc-v)AZYk1~~Y`ijj5GqXn3i!F)>O*s*c8-OG7J}rU zd@IB4sG!Uu|6FmMnI(UxfN$Vpx!*&pa{tbAkJ$Kjke-k}IF38%#?X5B`mev=Hw2xKHM7!LDro-~ABSu^*a4xMj)e+h2 vxrL7gm9+k3YcUVLd?o7oqx2hd?VR!CJM;hN3uSNvdS3A1lZWU0&Yk-o`Z%pi literal 0 HcmV?d00001 diff --git a/test/fixtures/geoip-fixture/src/main/resources/data.json b/test/fixtures/geoip-fixture/src/main/resources/data.json index 0b23251ce9e3b..62c49507be79b 100644 --- a/test/fixtures/geoip-fixture/src/main/resources/data.json +++ b/test/fixtures/geoip-fixture/src/main/resources/data.json @@ -1,8 +1,8 @@ [ { - "md5_hash": "a7813f5053276eb22639b61c61e3e56c", - "name": "GeoLite2-City.mmdb.gz", - "url": "endpoint/db/GeoLite2-City.mmdb.gz", + "md5_hash": "da5bb1c00c74e3f5a34ca1ec0022c550", + "name": "GeoLite2-City.tgz", + "url": "endpoint/db/GeoLite2-City.tgz", "provider": "maxmind" }, { @@ -12,9 +12,9 @@ "provider": "maxmind" }, { - "md5_hash": "dc366cf57a4f101d722b4cd10d4a9825", - "name": "GeoLite2-Country.mmdb.gz", - "url": "endpoint/db/GeoLite2-Country.mmdb.gz", + "md5_hash": "8f3229d6158f85adef296f8781f7ab49", + "name": "GeoLite2-Country.tgz", + "url": "endpoint/db/GeoLite2-Country.tgz", "provider": "maxmind" } ] From 8c14d7c3868441966b879d4275831007b2b02e59 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:31:45 +0100 Subject: [PATCH 10/17] rollback unneeded changes --- .../ingest/geoip/GeoIpTaskState.java | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index 77ccbbfa5cfa8..08297866bf51f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.VersionedNamedWriteable; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -30,7 +29,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { @@ -62,7 +60,8 @@ public static GeoIpTaskState fromXContent(XContentParser parser) throws IOExcept } GeoIpTaskState(StreamInput input) throws IOException { - databases = Collections.unmodifiableMap(input.readMap(StreamInput::readString, Metadata::new)); + databases = Collections.unmodifiableMap(input.readMap(StreamInput::readString, + in -> new Metadata(in.readLong(), in.readVInt(), in.readVInt(), in.readString()))); } public GeoIpTaskState put(String name, Metadata metadata) { @@ -122,17 +121,21 @@ public Version getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeMap(databases, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + out.writeMap(databases, StreamOutput::writeString, (o, v) -> { + o.writeLong(v.lastUpdate); + o.writeVInt(v.firstChunk); + o.writeVInt(v.lastChunk); + o.writeString(v.md5); + }); } - static class Metadata implements ToXContentObject, Writeable { + static class Metadata implements ToXContentObject { static final String NAME = GEOIP_DOWNLOADER + "-metadata"; private static final ParseField LAST_UPDATE = new ParseField("last_update"); private static final ParseField FIRST_CHUNK = new ParseField("first_chunk"); private static final ParseField LAST_CHUNK = new ParseField("last_chunk"); private static final ParseField MD5 = new ParseField("md5"); - private static final ParseField TAR = new ParseField("tar"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, @@ -143,7 +146,6 @@ static class Metadata implements ToXContentObject, Writeable { PARSER.declareInt(constructorArg(), FIRST_CHUNK); PARSER.declareInt(constructorArg(), LAST_CHUNK); PARSER.declareString(constructorArg(), MD5); - PARSER.declareBoolean(optionalConstructorArg(), TAR); } public static Metadata fromXContent(XContentParser parser) { @@ -159,10 +161,6 @@ public static Metadata fromXContent(XContentParser parser) { private final int lastChunk; private final String md5; - Metadata(StreamInput in) throws IOException { - this(in.readLong(), in.readVInt(), in.readVInt(), in.readString()); - } - Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5) { this.lastUpdate = lastUpdate; this.firstChunk = firstChunk; @@ -214,13 +212,5 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeLong(lastUpdate); - out.writeVInt(firstChunk); - out.writeVInt(lastChunk); - out.writeString(md5); - } } } From 2da15f08ed38126c95c6f746a63b3e24e23c1db2 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:32:29 +0100 Subject: [PATCH 11/17] rollback changes --- .../java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 8b8bfd36919f5..d7c3d534708be 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -136,8 +136,7 @@ void processDatabase(Map databaseInfo) { int firstChunk = state.contains(name) ? state.get(name).getLastChunk() + 1 : 0; int lastChunk = indexChunks(name, is, firstChunk, md5); if (lastChunk > firstChunk) { - Metadata metadata = new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5); - state = state.put(name, metadata); + state = state.put(name, new Metadata(System.currentTimeMillis(), firstChunk, lastChunk - 1, md5)); updateTaskState(); stats = stats.successfulDownload(System.currentTimeMillis() - start).count(state.getDatabases().size()); logger.info("updated geoip database [" + name + "]"); @@ -165,6 +164,7 @@ void deleteOldChunks(String name, int firstChunk) { protected void updateTimestamp(String name, Metadata old) { logger.info("geoip database [" + name + "] is up to date, updated timestamp"); state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5())); + stats = stats.skippedDownload(); updateTaskState(); } From a2821860b58bf263ce5d38d249a5d7d9ae92c2fe Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:34:30 +0100 Subject: [PATCH 12/17] rollback --- .../elasticsearch/ingest/geoip/DatabaseRegistry.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index 8d1781f658706..875887786c2c7 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -57,8 +57,6 @@ import java.util.stream.Stream; import java.util.zip.GZIPInputStream; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - /** * A component that is responsible for making the databases maintained by {@link GeoIpDownloader} * available for ingest processors. @@ -272,15 +270,15 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta } String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); if (name.startsWith(databaseName)) { - Files.copy(is, databaseTmpFile, REPLACE_EXISTING); + Files.copy(is, databaseTmpFile, StandardCopyOption.REPLACE_EXISTING); } else { - Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), REPLACE_EXISTING); + Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), StandardCopyOption.REPLACE_EXISTING); } } } LOGGER.debug("moving database from [{}] to [{}]", databaseTmpFile, databaseFile); - Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, REPLACE_EXISTING); + Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); updateDatabase(databaseName, recordedMd5, databaseFile); Files.delete(databaseTmpGzFile); }, @@ -371,7 +369,7 @@ void retrieveDatabase(String databaseName, static void decompress(Path source, Path target) throws IOException { try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(source), 8192)) { - Files.copy(in, target, REPLACE_EXISTING); + Files.copy(in, target, StandardCopyOption.REPLACE_EXISTING); } } From 35ffd36ec53f060eb86ef6d9438a6b332f38cbca Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:35:52 +0100 Subject: [PATCH 13/17] clean up --- .../ingest/geoip/DatabaseRegistryTests.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index 3bd96f3d8e11b..4e359280cce8d 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -151,8 +151,7 @@ public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Excepti String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", - new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -176,8 +175,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Ex String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); - task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", - new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); ClusterState state = ClusterState.builder(new ClusterName("name")) @@ -272,7 +270,7 @@ private String mockSearches(String databaseName, int firstChunk, int lastChunk) throw new UncheckedIOException(ex); } - SearchHits hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); + SearchHits hits = new SearchHits(new SearchHit[] {hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); SearchResponse searchResponse = new SearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 0), null, 1, 1, 0, 1L, null, null); @SuppressWarnings("unchecked") @@ -324,7 +322,7 @@ private static List gzip(String name, String content, int chunks) throws int chunkSize = all.length / chunks; List data = new ArrayList<>(); - for (int from = 0; from < all.length; ) { + for (int from = 0; from < all.length;) { int to = from + chunkSize; if (to > all.length) { to = all.length; From 854efe7d8e4f238e11fe317aba92807e87fdfcd4 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 25 Mar 2021 20:40:04 +0100 Subject: [PATCH 14/17] imports --- .../java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index f775c536663eb..0c4885ef6d8fc 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -37,8 +37,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; From 63d9e19822dc0969ba78d42206fcf0f84cd5a65f Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Fri, 26 Mar 2021 12:53:23 +0100 Subject: [PATCH 15/17] comments and refactorings --- .../ingest/geoip/DatabaseRegistry.java | 4 +++ .../ingest/geoip/TarInputStream.java | 27 ++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index 875887786c2c7..e77e96f4c2265 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -262,12 +262,16 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta decompress(databaseTmpGzFile, databaseTmpFile); Path databaseFile = geoipTmpDirectory.resolve(databaseName); + // tarball contains .mmdb, LICENSE.txt, COPYRIGHTS.txt and optional README.txt files. + // we store mmdb file as is and prepend database name to all other entries to avoid conflicts try (TarInputStream is = new TarInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpFile)))) { TarInputStream.TarEntry entry; while ((entry = is.getNextEntry()) != null) { + //there might be ./ entry in tar, we should skip it if (entry.isNotFile()) { continue; } + // flatten structure, remove any directories present from the path (should be ./ only) String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); if (name.startsWith(databaseName)) { Files.copy(is, databaseTmpFile, StandardCopyOption.REPLACE_EXISTING); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java index 51ba768b08cc7..78e398d1242ce 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -23,6 +23,7 @@ class TarInputStream extends FilterInputStream { private TarEntry currentEntry; private long remaining; + private long size; private final byte[] buf = new byte[512]; TarInputStream(InputStream in) { @@ -33,7 +34,7 @@ public TarEntry getNextEntry() throws IOException { if (currentEntry != null) { //go to the end of the current entry skipN(remaining); - long reminder = currentEntry.size % 512; + long reminder = size % 512; if (reminder != 0) { skipN(512 - reminder); } @@ -48,13 +49,17 @@ public TarEntry getNextEntry() throws IOException { if (Arrays.compare(buf, new byte[512]) == 0) { return null; } - String name = getString(345, 155) + getString(0, 100); + + String name = getString(0, 100); String sizeString = getString(124, 12); - remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + size = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + remaining = size; + boolean notFile = (buf[156] != 0 && buf[156] != '0') || name.endsWith("/"); - currentEntry = new TarEntry(name, remaining, notFile); + currentEntry = new TarEntry(name, notFile); if (notFile) { + size = 0; remaining = 0; } return currentEntry; @@ -71,12 +76,12 @@ public int read() throws IOException { @Override public int read(byte[] b, int off, int len) throws IOException { - if (remaining > 0) { - int read = in.read(b, off, remaining > Integer.MAX_VALUE ? len : (int) Math.min(len, remaining)); - remaining -= read; - return read; + if (remaining <= 0) { + return -1; } - return -1; + int read = in.read(b, off, remaining > Integer.MAX_VALUE ? len : (int) Math.min(len, remaining)); + remaining -= read; + return read; } private String getString(int offset, int maxLen) { @@ -99,12 +104,10 @@ private void skipN(long n) throws IOException { static class TarEntry { private final String name; - private final long size; private final boolean notFile; - TarEntry(String name, long size, boolean notFile) { + TarEntry(String name, boolean notFile) { this.name = name; - this.size = size; this.notFile = notFile; } From 1c1317ef8644514f964647b34f2c70f242421b5c Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Fri, 26 Mar 2021 12:55:05 +0100 Subject: [PATCH 16/17] comments and refactorings --- .../org/elasticsearch/ingest/geoip/TarInputStream.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java index 78e398d1242ce..0e1e0d17c7767 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -23,7 +23,7 @@ class TarInputStream extends FilterInputStream { private TarEntry currentEntry; private long remaining; - private long size; + private long reminder; private final byte[] buf = new byte[512]; TarInputStream(InputStream in) { @@ -34,7 +34,6 @@ public TarEntry getNextEntry() throws IOException { if (currentEntry != null) { //go to the end of the current entry skipN(remaining); - long reminder = size % 512; if (reminder != 0) { skipN(512 - reminder); } @@ -53,13 +52,12 @@ public TarEntry getNextEntry() throws IOException { String name = getString(0, 100); String sizeString = getString(124, 12); - size = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); - remaining = size; + remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + reminder = remaining % 512; boolean notFile = (buf[156] != 0 && buf[156] != '0') || name.endsWith("/"); currentEntry = new TarEntry(name, notFile); if (notFile) { - size = 0; remaining = 0; } return currentEntry; From ee5aa36712529b9ed0fc95485470032ae5176f55 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Fri, 26 Mar 2021 13:08:13 +0100 Subject: [PATCH 17/17] refactoring --- .../ingest/geoip/TarInputStream.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java index 0e1e0d17c7767..2b1d4e98bebef 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -51,15 +51,18 @@ public TarEntry getNextEntry() throws IOException { String name = getString(0, 100); - String sizeString = getString(124, 12); - remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); - reminder = remaining % 512; - boolean notFile = (buf[156] != 0 && buf[156] != '0') || name.endsWith("/"); - currentEntry = new TarEntry(name, notFile); - if (notFile) { + + if(notFile){ remaining = 0; + reminder = 0; + } else { + String sizeString = getString(124, 12); + remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + reminder = remaining % 512; } + + currentEntry = new TarEntry(name, notFile); return currentEntry; }