Skip to content

Commit

Permalink
IPinfo privacy detection support (elastic#114456)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joegallo
joegallo authored Oct 9, 2024
1 parent 19ec3d9 commit ee5be48
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 3 deletions.
14 changes: 12 additions & 2 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,11 @@ enum Database {
Property.POSTAL_CODE
),
Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION)
),;
),
PrivacyDetection(
Set.of(Property.IP, Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE),
Set.of(Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE)
);

private final Set<Property> properties;
private final Set<Property> defaultProperties;
Expand Down Expand Up @@ -262,7 +266,13 @@ enum Property {
TYPE,
POSTAL_CODE,
POSTAL_CONFIDENCE,
ACCURACY_RADIUS;
ACCURACY_RADIUS,
HOSTING,
TOR,
PROXY,
RELAY,
VPN,
SERVICE;

/**
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are
Expand Down
90 changes: 90 additions & 0 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,31 @@ static Long parseAsn(final String asn) {
}
}

/**
* Lax-ly parses a string that contains a boolean into a Boolean (or null, if such parsing isn't possible).
* @param bool a potentially empty (or null) string that is expected to contain a parsable boolean
* @return the parsed boolean
*/
static Boolean parseBoolean(final String bool) {
if (bool == null) {
return null;
} else {
String trimmed = bool.toLowerCase(Locale.ROOT).trim();
if ("true".equals(trimmed)) {
return true;
} else if ("false".equals(trimmed)) {
// "false" can represent false -- this an expected future enhancement in how the database represents booleans
return false;
} else if (trimmed.isEmpty()) {
// empty string can represent false -- this is how the database currently represents 'false' values
return false;
} else {
logger.trace("Unable to parse non-compliant boolean string [{}]", bool);
return null;
}
}
}

/**
* Lax-ly parses a string that contains a double into a Double (or null, if such parsing isn't possible).
* @param latlon a potentially empty (or null) string that is expected to contain a parsable double
Expand Down Expand Up @@ -132,6 +157,22 @@ public GeolocationResult(
}
}

public record PrivacyDetectionResult(Boolean hosting, Boolean proxy, Boolean relay, String service, Boolean tor, Boolean vpn) {
@SuppressWarnings("checkstyle:RedundantModifier")
@MaxMindDbConstructor
public PrivacyDetectionResult(
@MaxMindDbParameter(name = "hosting") String hosting,
// @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
@MaxMindDbParameter(name = "proxy") String proxy,
@MaxMindDbParameter(name = "relay") String relay,
@MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed as booleans
@MaxMindDbParameter(name = "tor") String tor,
@MaxMindDbParameter(name = "vpn") String vpn
) {
this(parseBoolean(hosting), parseBoolean(proxy), parseBoolean(relay), service, parseBoolean(tor), parseBoolean(vpn));
}
}

static class Asn extends AbstractBase<AsnResult> {
Asn(Set<Database.Property> properties) {
super(properties, AsnResult.class);
Expand Down Expand Up @@ -286,6 +327,55 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
}
}

static class PrivacyDetection extends AbstractBase<PrivacyDetectionResult> {
PrivacyDetection(Set<Database.Property> properties) {
super(properties, PrivacyDetectionResult.class);
}

@Override
protected Map<String, Object> transform(final Result<PrivacyDetectionResult> result) {
PrivacyDetectionResult response = result.result;

Map<String, Object> data = new HashMap<>();
for (Database.Property property : this.properties) {
switch (property) {
case IP -> data.put("ip", result.ip);
case HOSTING -> {
if (response.hosting != null) {
data.put("hosting", response.hosting);
}
}
case TOR -> {
if (response.tor != null) {
data.put("tor", response.tor);
}
}
case PROXY -> {
if (response.proxy != null) {
data.put("proxy", response.proxy);
}
}
case RELAY -> {
if (response.relay != null) {
data.put("relay", response.relay);
}
}
case VPN -> {
if (response.vpn != null) {
data.put("vpn", response.vpn);
}
}
case SERVICE -> {
if (Strings.hasText(response.service)) {
data.put("service", response.service);
}
}
}
}
return data;
}
}

/**
* Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the
* getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for
Expand Down
87 changes: 87 additions & 0 deletions ...s/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@
import static java.util.Map.entry;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn;
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean;
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
Expand Down Expand Up @@ -93,6 +95,21 @@ public void testParseAsn() {
assertThat(parseAsn("anythingelse"), nullValue());
}

public void testParseBoolean() {
// expected cases: "true" is true and "" is false
assertThat(parseBoolean("true"), equalTo(true));
assertThat(parseBoolean(""), equalTo(false));
assertThat(parseBoolean("false"), equalTo(false)); // future proofing
// defensive case: null becomes null, this is not expected fwiw
assertThat(parseBoolean(null), nullValue());
// defensive cases: we strip whitespace and ignore case
assertThat(parseBoolean(" "), equalTo(false));
assertThat(parseBoolean(" TrUe "), equalTo(true));
assertThat(parseBoolean(" FaLSE "), equalTo(false));
// bottom case: a non-parsable string is null
assertThat(parseBoolean(randomAlphaOfLength(8)), nullValue());
}

public void testParseLocationDouble() {
// expected case: "123.45" is 123.45
assertThat(parseLocationDouble("123.45"), equalTo(123.45));
Expand Down Expand Up @@ -287,6 +304,76 @@ public void testGeolocationInvariants() {
}
}

public void testPrivacyDetection() throws IOException {
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
Path configDir = tmpDir;
copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb"));

GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
configDatabases.initialize(resourceWatcherService);

// testing the first row in the sample database
try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
Map<String, Object> data = lookup.getData(loader, "1.53.59.33");
assertThat(
data,
equalTo(
Map.ofEntries(
entry("ip", "1.53.59.33"),
entry("hosting", false),
entry("proxy", false),
entry("relay", false),
entry("tor", false),
entry("vpn", true)
)
)
);
}

// testing a row with a non-empty service in the sample database
try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
Map<String, Object> data = lookup.getData(loader, "216.131.74.65");
assertThat(
data,
equalTo(
Map.ofEntries(
entry("ip", "216.131.74.65"),
entry("hosting", true),
entry("proxy", false),
entry("service", "FastVPN"),
entry("relay", false),
entry("tor", false),
entry("vpn", true)
)
)
);
}
}

public void testPrivacyDetectionInvariants() {
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
Path configDir = tmpDir;
copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb"));

{
final Set<String> expectedColumns = Set.of("network", "service", "hosting", "proxy", "relay", "tor", "vpn");

Path databasePath = configDir.resolve("privacy_detection_sample.mmdb");
assertDatabaseInvariants(databasePath, (ip, row) -> {
assertThat(row.keySet(), equalTo(expectedColumns));

for (String booleanColumn : Set.of("hosting", "proxy", "relay", "tor", "vpn")) {
String bool = (String) row.get(booleanColumn);
assertThat(bool, anyOf(equalTo("true"), equalTo(""), equalTo("false")));
assertThat(parseBoolean(bool), notNullValue());
}
});
}
}

private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer<InetAddress, Map<String, Object>> rowConsumer) {
try (Reader reader = new Reader(pathToFile(databasePath))) {
Networks<?> networks = reader.networks(Map.class);
Expand Down
6 changes: 5 additions & 1 deletion modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,11 @@ public class MaxMindSupportTests extends ESTestCase {

private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class);

private static final Set<Database> KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2, Database.CityV2);
private static final Set<Database> KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(
Database.AsnV2,
Database.CityV2,
Database.PrivacyDetection
);

public void testMaxMindSupport() {
for (Database databaseType : Database.values()) {
Expand Down
Binary file added modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb
View file
Binary file not shown.

0 comments on commit ee5be48

Please sign in to comment.