Skip to content

Commit

Permalink
Support IPinfo databases in the ip_location processor (elastic#114735)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joegallo
joegallo committed Oct 15, 2024
1 parent ffcf87c commit bfdb6a1
Show file tree
Hide file tree
Showing 8 changed files with 383 additions and 97 deletions.
2 changes: 2 additions & 0 deletions modules/ingest-geoip/src/main/java/module-info.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@

exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;

exports org.elasticsearch.ingest.geoip to com.maxmind.db;
}
4 changes: 2 additions & 2 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,14 @@ void updateDatabase(Path file, boolean update) {
String databaseFileName = file.getFileName().toString();
try {
if (update) {
logger.info("database file changed [{}], reload database...", file);
logger.info("database file changed [{}], reloading database...", file);
DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null);
DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader);
if (existing != null) {
existing.shutdown();
}
} else {
logger.info("database file removed [{}], close database...", file);
logger.info("database file removed [{}], closing database...", file);
DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName);
assert existing != null;
existing.shutdown();
Expand Down
20 changes: 13 additions & 7 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,13 +196,19 @@ public IpDatabase get() throws IOException {
}

if (Assertions.ENABLED) {
// Only check whether the suffix has changed and not the entire database type.
// To sanity check whether a city db isn't overwriting with a country or asn db.
// For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different,
// by checking just the suffix this assertion doesn't fail.
String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-'));
assert loader.getDatabaseType().endsWith(expectedSuffix)
: "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]";
// Note that the expected suffix might be null for providers that aren't amenable to using dashes as separator for
// determining the database type.
int last = databaseType.lastIndexOf('-');
final String expectedSuffix = last == -1 ? null : databaseType.substring(last);

// If the entire database type matches, then that's a match. Otherwise, if there's a suffix to compare on, then
// check whether the suffix has changed (not the entire database type).
// This is to sanity check, for example, that a city db isn't overwritten with a country or asn db.
// But there are permissible overwrites that make sense, for example overwriting a geolite city db with a geoip city db
// is a valid change, but the db type is slightly different -- by checking just the suffix this assertion won't fail.
final String loaderType = loader.getDatabaseType();
assert loaderType.equals(databaseType) || expectedSuffix == null || loaderType.endsWith(expectedSuffix)
: "database type [" + loaderType + "] doesn't match with expected suffix [" + expectedSuffix + "]";
}
return loader;
}
Expand Down
73 changes: 23 additions & 50 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,16 @@
import org.elasticsearch.core.Nullable;

import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.function.Function;

import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.IPINFO_PREFIX;
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoDatabase;
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoLookup;
import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindDatabase;
import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindLookup;

final class IpDataLookupFactories {

private IpDataLookupFactories() {
Expand All @@ -26,78 +33,44 @@ interface IpDataLookupFactory {
IpDataLookup create(List<String> properties);
}

private static final String CITY_DB_SUFFIX = "-City";
private static final String COUNTRY_DB_SUFFIX = "-Country";
private static final String ASN_DB_SUFFIX = "-ASN";
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
private static final String DOMAIN_DB_SUFFIX = "-Domain";
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
private static final String ISP_DB_SUFFIX = "-ISP";

@Nullable
private static Database getMaxmindDatabase(final String databaseType) {
if (databaseType.endsWith(CITY_DB_SUFFIX)) {
return Database.City;
} else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
return Database.Country;
} else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
return Database.Asn;
} else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
return Database.AnonymousIp;
} else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
return Database.ConnectionType;
} else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
return Database.Domain;
} else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
return Database.Enterprise;
} else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
return Database.Isp;
} else {
return null; // no match was found
}
}

/**
* Parses the passed-in databaseType and return the Database instance that is
* associated with that databaseType.
*
* @param databaseType the database type String from the metadata of the database file
* @return the Database instance that is associated with the databaseType
* @return the Database instance that is associated with the databaseType (or null)
*/
@Nullable
static Database getDatabase(final String databaseType) {
Database database = null;

if (Strings.hasText(databaseType)) {
database = getMaxmindDatabase(databaseType);
final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
database = getIpinfoDatabase(databaseTypeLowerCase); // all lower case!
} else {
// for historical reasons, fall back to assuming maxmind-like type parsing
database = getMaxmindDatabase(databaseType);
}
}

return database;
}

@Nullable
static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
return switch (database) {
case City -> MaxmindIpDataLookups.City::new;
case Country -> MaxmindIpDataLookups.Country::new;
case Asn -> MaxmindIpDataLookups.Asn::new;
case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
case Domain -> MaxmindIpDataLookups.Domain::new;
case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
case Isp -> MaxmindIpDataLookups.Isp::new;
default -> null;
};
}

static IpDataLookupFactory get(final String databaseType, final String databaseFile) {
final Database database = getDatabase(databaseType);
if (database == null) {
throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
}

final Function<Set<Database.Property>, IpDataLookup> factoryMethod = getMaxmindLookup(database);
final Function<Set<Database.Property>, IpDataLookup> factoryMethod;
final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
factoryMethod = getIpinfoLookup(database);
} else {
// for historical reasons, fall back to assuming maxmind-like types
factoryMethod = getMaxmindLookup(database);
}

if (factoryMethod == null) {
throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
Expand Down
79 changes: 79 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,14 @@

import java.io.IOException;
import java.net.InetAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
* A collection of {@link IpDataLookup} implementations for IPinfo databases
Expand All @@ -43,6 +47,81 @@ private IpinfoIpDataLookups() {
// prefix dispatch and checks case-insensitive, so that works out nicely
static final String IPINFO_PREFIX = "ipinfo";

private static final Set<String> IPINFO_TYPE_STOP_WORDS = Set.of(
"ipinfo",
"extended",
"free",
"generic",
"ip",
"sample",
"standard",
"mmdb"
);

/**
* Cleans up the database_type String from an ipinfo database by splitting on punctuation, removing stop words, and then joining
* with an underscore.
* <p>
* e.g. "ipinfo free_foo_sample.mmdb" -> "foo"
*
* @param type the database_type from an ipinfo database
* @return a cleaned up database_type string
*/
// n.b. this is just based on observation of the types from a survey of such databases -- it's like browser user agent sniffing,
// there aren't necessarily any amazing guarantees about this behavior
static String ipinfoTypeCleanup(String type) {
List<String> parts = Arrays.asList(type.split("[ _.]"));
return parts.stream().filter((s) -> IPINFO_TYPE_STOP_WORDS.contains(s) == false).collect(Collectors.joining("_"));
}

@Nullable
static Database getIpinfoDatabase(final String databaseType) {
// for ipinfo the database selection is more along the lines of user-agent sniffing than
// string-based dispatch. the specific database_type strings could change in the future,
// hence the somewhat loose nature of this checking.

final String cleanedType = ipinfoTypeCleanup(databaseType);

// early detection on any of the 'extended' types
if (databaseType.contains("extended")) {
// which are not currently supported
logger.trace("returning null for unsupported database_type [{}]", databaseType);
return null;
}

// early detection on 'country_asn' so the 'country' and 'asn' checks don't get faked out
if (cleanedType.contains("country_asn")) {
// but it's not currently supported
logger.trace("returning null for unsupported database_type [{}]", databaseType);
return null;
}

if (cleanedType.contains("asn")) {
return Database.AsnV2;
} else if (cleanedType.contains("country")) {
return Database.CountryV2;
} else if (cleanedType.contains("location")) { // note: catches 'location' and 'geolocation' ;)
return Database.CityV2;
} else if (cleanedType.contains("privacy")) {
return Database.PrivacyDetection;
} else {
// no match was found
logger.trace("returning null for unsupported database_type [{}]", databaseType);
return null;
}
}

@Nullable
static Function<Set<Database.Property>, IpDataLookup> getIpinfoLookup(final Database database) {
return switch (database) {
case AsnV2 -> IpinfoIpDataLookups.Asn::new;
case CountryV2 -> IpinfoIpDataLookups.Country::new;
case CityV2 -> IpinfoIpDataLookups.Geolocation::new;
case PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new;
default -> null;
};
}

/**
* Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible).
* @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long
Expand Down
55 changes: 55 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import com.maxmind.geoip2.record.Postal;
import com.maxmind.geoip2.record.Subdivision;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.core.Nullable;
Expand All @@ -37,6 +39,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;

/**
* A collection of {@link IpDataLookup} implementations for MaxMind databases
Expand All @@ -47,11 +50,63 @@ private MaxmindIpDataLookups() {
// utility class
}

private static final Logger logger = LogManager.getLogger(MaxmindIpDataLookups.class);

// the actual prefixes from the metadata are cased like the literal strings, but
// prefix dispatch and checks case-insensitive, so the actual constants are lowercase
static final String GEOIP2_PREFIX = "GeoIP2".toLowerCase(Locale.ROOT);
static final String GEOLITE2_PREFIX = "GeoLite2".toLowerCase(Locale.ROOT);

// note: the secondary dispatch on suffix happens to be case sensitive
private static final String CITY_DB_SUFFIX = "-City";
private static final String COUNTRY_DB_SUFFIX = "-Country";
private static final String ASN_DB_SUFFIX = "-ASN";
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
private static final String DOMAIN_DB_SUFFIX = "-Domain";
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
private static final String ISP_DB_SUFFIX = "-ISP";

@Nullable
static Database getMaxmindDatabase(final String databaseType) {
if (databaseType.endsWith(CITY_DB_SUFFIX)) {
return Database.City;
} else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
return Database.Country;
} else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
return Database.Asn;
} else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
return Database.AnonymousIp;
} else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
return Database.ConnectionType;
} else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
return Database.Domain;
} else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
return Database.Enterprise;
} else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
return Database.Isp;
} else {
// no match was found
logger.trace("returning null for unsupported database_type [{}]", databaseType);
return null;
}
}

@Nullable
static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
return switch (database) {
case City -> MaxmindIpDataLookups.City::new;
case Country -> MaxmindIpDataLookups.Country::new;
case Asn -> MaxmindIpDataLookups.Asn::new;
case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
case Domain -> MaxmindIpDataLookups.Domain::new;
case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
case Isp -> MaxmindIpDataLookups.Isp::new;
default -> null;
};
}

static class AnonymousIp extends AbstractBase<AnonymousIpResponse> {
AnonymousIp(final Set<Database.Property> properties) {
super(
Expand Down
Loading

0 comments on commit bfdb6a1

Please sign in to comment.