Skip to content

Commit

Permalink
Switch sample files and update code and tests
Browse files Browse the repository at this point in the history 
There are slightly different columns in this sample database than in
the one we had been using. Most of the difference is immaterial, but
the location columns themselves are named slightly differently, and we
need to update our code and tests to match this new schema.
  • Loading branch information
@joegallo
joegallo committed Oct 18, 2024
1 parent c97b29a commit c56f8bd
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 27 deletions.
17 changes: 9 additions & 8 deletions modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,8 +218,8 @@ public record CountryResult(
public record GeolocationResult(
String city,
String country,
Double latitude,
Double longitude,
Double lat,
Double lng,
String postalCode,
String region,
String timezone
Expand All @@ -229,14 +229,15 @@ public record GeolocationResult(
public GeolocationResult(
@MaxMindDbParameter(name = "city") String city,
@MaxMindDbParameter(name = "country") String country,
@MaxMindDbParameter(name = "latitude") String latitude,
@MaxMindDbParameter(name = "longitude") String longitude,
// @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
// @MaxMindDbParameter(name = "geoname_id") String geonameId, // for now we're not exposing this
@MaxMindDbParameter(name = "lat") String lat,
@MaxMindDbParameter(name = "lng") String lng,
@MaxMindDbParameter(name = "postal_code") String postalCode,
@MaxMindDbParameter(name = "region") String region,
// @MaxMindDbParameter(name = "region_code") String regionCode, // for now we're not exposing this
@MaxMindDbParameter(name = "timezone") String timezone
) {
this(city, country, parseLocationDouble(latitude), parseLocationDouble(longitude), postalCode, region, timezone);
this(city, country, parseLocationDouble(lat), parseLocationDouble(lng), postalCode, region, timezone);
}
}

Expand Down Expand Up @@ -395,8 +396,8 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
}
}
case LOCATION -> {
Double latitude = response.latitude;
Double longitude = response.longitude;
Double latitude = response.lat;
Double longitude = response.lng;
if (latitude != null && longitude != null) {
Map<String, Object> locationObject = new HashMap<>();
locationObject.put("lat", latitude);
Expand Down
6 changes: 3 additions & 3 deletions modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,13 @@ public void testMaxmindCity() throws Exception {
}

public void testIpinfoGeolocation() throws Exception {
String ip = "4.36.238.62";
String ip = "72.20.12.220";
GeoIpProcessor processor = new GeoIpProcessor(
IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor
randomAlphaOfLength(10),
null,
"source_field",
loader("ipinfo/ip_geolocation_sample.mmdb"),
loader("ipinfo/ip_geolocation_standard_sample.mmdb"),
() -> true,
"target_field",
getIpinfoGeolocationLookup(),
Expand All @@ -107,7 +107,7 @@ public void testIpinfoGeolocation() throws Exception {
Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(data, notNullValue());
assertThat(data.get("ip"), equalTo(ip));
assertThat(data.get("city_name"), equalTo("Pensacola"));
assertThat(data.get("city_name"), equalTo("Chicago"));
// see IpinfoIpDataLookupsTests for more tests of the data lookup behavior
}

Expand Down
33 changes: 17 additions & 16 deletions ...s/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ public void testCountryFree() {

public void testGeolocationStandard() {
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
String databaseName = "ip_geolocation_sample.mmdb";
String ip = "90.203.36.70";
String databaseName = "ip_geolocation_standard_sample.mmdb";
String ip = "62.69.48.19";
assertExpectedLookupResults(
databaseName,
ip,
Expand All @@ -204,47 +204,48 @@ public void testGeolocationStandard() {
entry("ip", ip),
entry("country_iso_code", "GB"),
entry("region_name", "England"),
entry("city_name", "Birkenhead"),
entry("city_name", "London"),
entry("timezone", "Europe/London"),
entry("postal_code", "CH41"),
entry("location", Map.of("lat", 53.39337, "lon", -3.01479))
entry("postal_code", "E1W"),
entry("location", Map.of("lat", 51.50853, "lon", -0.12574))
)
);
}

public void testGeolocationInvariants() {
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
Path configDir = tmpDir;
copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb"));
copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", configDir.resolve("ip_geolocation_standard_sample.mmdb"));

{
final Set<String> expectedColumns = Set.of(
"network",
"city",
"geoname_id",
"region",
"region_code",
"country",
"postal_code",
"timezone",
"latitude",
"longitude"
"lat",
"lng"
);

Path databasePath = configDir.resolve("ip_geolocation_sample.mmdb");
Path databasePath = configDir.resolve("ip_geolocation_standard_sample.mmdb");
assertDatabaseInvariants(databasePath, (ip, row) -> {
assertThat(row.keySet(), equalTo(expectedColumns));
{
String latitude = (String) row.get("latitude");
String latitude = (String) row.get("lat");
assertThat(latitude, equalTo(latitude.trim()));
Double parsed = parseLocationDouble(latitude);
assertThat(parsed, notNullValue());
assertThat(latitude, equalTo(Double.toString(parsed))); // reverse it
assertThat(Double.parseDouble(latitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it
}
{
String longitude = (String) row.get("longitude");
String longitude = (String) row.get("lng");
assertThat(longitude, equalTo(longitude.trim()));
Double parsed = parseLocationDouble(longitude);
assertThat(parsed, notNullValue());
assertThat(longitude, equalTo(Double.toString(parsed))); // reverse it
assertThat(Double.parseDouble(longitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it
}
});
}
Expand Down Expand Up @@ -391,13 +392,13 @@ public void testDatabaseTypeParsing() throws IOException {
// pedantic about where precisely it should be.

copyDatabase("ipinfo/ip_asn_sample.mmdb", tmpDir.resolve("ip_asn_sample.mmdb"));
copyDatabase("ipinfo/ip_geolocation_sample.mmdb", tmpDir.resolve("ip_geolocation_sample.mmdb"));
copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", tmpDir.resolve("ip_geolocation_standard_sample.mmdb"));
copyDatabase("ipinfo/asn_sample.mmdb", tmpDir.resolve("asn_sample.mmdb"));
copyDatabase("ipinfo/ip_country_sample.mmdb", tmpDir.resolve("ip_country_sample.mmdb"));
copyDatabase("ipinfo/privacy_detection_sample.mmdb", tmpDir.resolve("privacy_detection_sample.mmdb"));

assertThat(parseDatabaseFromType("ip_asn_sample.mmdb"), is(Database.AsnV2));
assertThat(parseDatabaseFromType("ip_geolocation_sample.mmdb"), is(Database.CityV2));
assertThat(parseDatabaseFromType("ip_geolocation_standard_sample.mmdb"), is(Database.CityV2));
assertThat(parseDatabaseFromType("asn_sample.mmdb"), is(Database.AsnV2));
assertThat(parseDatabaseFromType("ip_country_sample.mmdb"), is(Database.CountryV2));
assertThat(parseDatabaseFromType("privacy_detection_sample.mmdb"), is(Database.PrivacyDetection));
Expand Down
Binary file removed modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb
View file
Binary file not shown.
Binary file added modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb
View file
Binary file not shown.

0 comments on commit c56f8bd

Please sign in to comment.