From 3714e9a7e837b2bd51e030b03bf96913e674beeb Mon Sep 17 00:00:00 2001 From: Julien Ruaux Date: Sun, 2 Jun 2019 15:06:25 -0700 Subject: [PATCH] fixed JedisDataException on json null values and added json unit test --- README.adoc | 83 ++++++++++++++----- pom.xml | 4 +- .../redislabs/riot/cli/AbstractCommand.java | 2 +- ...actCollectionRedisImportSubSubCommand.java | 14 ++++ ...AbstractRediSearchImportSubSubCommand.java | 14 ++++ .../AbstractRedisImportSubSubCommand.java | 19 ++--- ...bstractSingleRedisImportSubSubCommand.java | 22 +++++ .../cli/redis/HashImportSubSubCommand.java | 2 +- .../cli/redis/StreamImportSubSubCommand.java | 2 +- .../cli/redis/StringImportSubSubCommand.java | 2 +- .../redis/writer/AbstractRedisCommands.java | 16 +++- .../com/redislabs/riot/AbstractBaseTest.java | 53 ------------ .../java/com/redislabs/riot/BaseTest.java | 36 ++++++++ .../com/redislabs/riot/FileImportTest.java | 19 ----- .../java/com/redislabs/riot/ImportCSV.java | 16 ++++ .../java/com/redislabs/riot/ImportJSON.java | 31 +++++++ .../redislabs/riot/ImportToRediSearch.java | 36 ++++++++ src/test/resources/beers.json | 33 ++++++++ src/test/resources/es_test-index.json | 29 +++++++ 19 files changed, 320 insertions(+), 113 deletions(-) create mode 100644 src/main/java/com/redislabs/riot/cli/redis/AbstractSingleRedisImportSubSubCommand.java delete mode 100644 src/test/java/com/redislabs/riot/AbstractBaseTest.java create mode 100644 src/test/java/com/redislabs/riot/BaseTest.java delete mode 100644 src/test/java/com/redislabs/riot/FileImportTest.java create mode 100644 src/test/java/com/redislabs/riot/ImportCSV.java create mode 100644 src/test/java/com/redislabs/riot/ImportJSON.java create mode 100644 src/test/java/com/redislabs/riot/ImportToRediSearch.java create mode 100644 src/test/resources/beers.json create mode 100644 src/test/resources/es_test-index.json diff --git a/README.adoc b/README.adoc index 0709e1fd5..71b4fab6a 100644 --- a/README.adoc +++ b/README.adoc @@ -20,7 +20,11 @@ Follow the usage help provided with the `--help` option. == Examples -=== Beers +=== Importing Files + +==== CSV + +===== Beers Here is an excerpt of https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv[beers.csv] available at https://github.com/nickhould/craft-beers-dataset[craft-beers-dataset]: [source] @@ -32,29 +36,28 @@ Here is an excerpt of https://raw.githubusercontent.com/nickhould/craft-beers-da 3,0.09,,2263,Sinister,American Double / Imperial IPA,177,12.0 ---- -==== Import to Hashes +* Import beers to Hashes [source,shell] ---- riot import csv --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header hash --keyspace beers --keys id ---- -==== Import to RediSearch Index -. Create an index with redis-cli +* Import beers to RediSearch ++ +.Create an index with redis-cli + [source,shell] ---- FT.CREATE beerIdx SCHEMA abv NUMERIC SORTABLE id TAG name TEXT PHONETIC dm:en style TEXT PHONETIC dm:en brewery_id TAG ounces NUMERIC SORTABLE ---- -. Import data into the index +.Import data into the index + [source,shell] ---- riot import csv --header --included-fields 1 3 4 5 6 --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv search --index beerIdx --keys id ---- -=== OpenFlights - -==== Airports +===== OpenFlights Here is an excerpt of https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat[airports.dat] available at https://github.com/jpatokal/openflights[openflights] ---- @@ -63,28 +66,22 @@ Here is an excerpt of https://raw.githubusercontent.com/jpatokal/openflights/mas 3,"Mount Hagen Kagamuga Airport","Mount Hagen","Papua New Guinea","HGU","AYMH",-5.826789855957031,144.29600524902344,5388,10,"U","Pacific/Port_Moresby","airport","OurAirports" ---- -===== Airport Geo -. Import airports into a geoset -+ +.Import airports into a geoset [source,shell] ---- riot import csv --url https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source geo --keyspace airportgeo --fields AirportID --longitude-field Longitude --latitude-field Latitude ---- -. Query the `airportgeo` geoset using redis-cli -+ +.Query the `airportgeo` geoset using redis-cli [source,shell] ---- -> GEORADIUS airportgeo -122.4194 37.7749 20 mi +GEORADIUS airportgeo -122.4194 37.7749 20 mi 1) "3469" 2) "10360" 3) "8982" 4) "9071" 5) "3453" ----- -+ -[source,shell] ----- -> HGETALL airport:3469 + +HGETALL airport:3469 1) "AirportID" 2) "3469" 3) "ICAO" @@ -115,5 +112,51 @@ riot import csv --url https://raw.githubusercontent.com/jpatokal/openflights/mas 28) "13" ---- - +==== JSON + +===== Beers + +Here is an excerpt of beers.json available at https://github.com/rethinkdb/beerthink/blob/master/data/beers.json[beerthink]: +[source,json] +---- +[ + { + "id": "1", + "brewery_id": "812", + "name": "Hocus Pocus", + "abv": "4.5", + "ibu": "0", + "srm": "0", + "upc": "0", + "filepath": "", + "descript": "Our take on a classic summer ale. A toast to weeds, rays, and summer haze. A light, crisp ale for mowing lawns, hitting lazy fly balls, and communing with nature, Hocus Pocus is offered up as a summer sacrifice to clodless days.\n\nIts malty sweetness finishes tart and crisp and is best apprediated with a wedge of orange.", + "add_user": "0", + "last_mod": "2010-07-22 20:00:20 UTC", + "style_name": "Light American Wheat Ale or Lager", + "cat_name": "Other Style" + }, + { + "id": "6", + "brewery_id": "1385", + "name": "Winter Warmer", + "abv": "5.199999809265137", + "ibu": "0", + "srm": "0", + "upc": "0", + "filepath": "", + "descript": "", + "add_user": "0", + "last_mod": "2010-07-22 20:00:20 UTC", + "style_name": "Old Ale", + "cat_name": "British Ale" + }, + ... +] +---- + +.Import beers into Hashes +[source,shell] +---- +riot import json --url=https://raw.githubusercontent.com/rethinkdb/beerthink/master/data/beers.json hash --keyspace beerjson --keys id +---- diff --git a/pom.xml b/pom.xml index f71e30bf8..0a63bead1 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ com.redislabs riot - 0.2.0 + 0.2.1 jar RIOT Redis Input/Output Tool @@ -58,7 +58,7 @@ com.redislabs lettusearch - 1.1.9 + 1.2.0 org.apache.commons diff --git a/src/main/java/com/redislabs/riot/cli/AbstractCommand.java b/src/main/java/com/redislabs/riot/cli/AbstractCommand.java index 4bee06d5f..e61604ece 100644 --- a/src/main/java/com/redislabs/riot/cli/AbstractCommand.java +++ b/src/main/java/com/redislabs/riot/cli/AbstractCommand.java @@ -106,7 +106,7 @@ public void run(String sourceDescription, AbstractItemCountingItemStreamItemRead builder.setSleep(sleep); Job job = builder.build(); long startTime = System.currentTimeMillis(); - System.out.println("Importing from " + sourceDescription + " to " + targetDescription); + System.out.println("Importing " + sourceDescription + " into " + targetDescription); SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); jobLauncher.setJobRepository(builder.jobRepository()); jobLauncher.setTaskExecutor(new SyncTaskExecutor()); diff --git a/src/main/java/com/redislabs/riot/cli/redis/AbstractCollectionRedisImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/AbstractCollectionRedisImportSubSubCommand.java index 88cb8caee..f62ccb399 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/AbstractCollectionRedisImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/AbstractCollectionRedisImportSubSubCommand.java @@ -6,9 +6,23 @@ public abstract class AbstractCollectionRedisImportSubSubCommand extends AbstractRedisImportSubSubCommand { + @Option(names = "--keyspace", required = true, description = "Redis keyspace prefix.") + private String keyspace; + @Option(names = "--keys", arity = "1..*", description = "Key fields.") + private String[] keys = new String[0]; @Option(names = "--fields", required = true, arity = "1..*", description = "Fields used to build member ids for collection data structures (list, set, zset, geo).") private String[] fields; + @Override + protected String getKeyspace() { + return keyspace; + } + + @Override + protected String[] getKeys() { + return keys; + } + @Override protected AbstractCollectionRedisItemWriter redisItemWriter() { AbstractCollectionRedisItemWriter writer = collectionRedisItemWriter(); diff --git a/src/main/java/com/redislabs/riot/cli/redis/AbstractRediSearchImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/AbstractRediSearchImportSubSubCommand.java index f0a2a00e8..1bc0ea19c 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/AbstractRediSearchImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/AbstractRediSearchImportSubSubCommand.java @@ -13,9 +13,23 @@ public abstract class AbstractRediSearchImportSubSubCommand extends AbstractRedisImportSubSubCommand { + @Option(names = { "--keyspace" }, description = "Document keyspace prefix.") + private String keyspace; + @Option(names = "--keys", required = true, arity = "1..*", description = "Document key fields.") + private String[] keys = new String[0]; @Getter @Option(names = "--index", description = "Name of the RediSearch index", required = true) private String index; + + @Override + protected String getKeyspace() { + return keyspace; + } + + @Override + protected String[] getKeys() { + return keys; + } @Override protected ItemStreamWriter> writer() { diff --git a/src/main/java/com/redislabs/riot/cli/redis/AbstractRedisImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/AbstractRedisImportSubSubCommand.java index da6b0512a..14e77ff35 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/AbstractRedisImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/AbstractRedisImportSubSubCommand.java @@ -4,15 +4,8 @@ import com.redislabs.riot.redis.RedisConverter; import com.redislabs.riot.redis.writer.AbstractRedisItemWriter; -import picocli.CommandLine.Option; - public abstract class AbstractRedisImportSubSubCommand extends AbstractImportSubSubCommand { - @Option(names = { "-s", "--keyspace" }, description = "Redis keyspace prefix.") - private String keyspace; - @Option(names = { "-k", "--keys" }, arity = "1..*", description = "Key fields.") - private String[] keys = new String[0]; - @Override public String getTargetDescription() { return getDataStructure() + " \"" + getKeyspaceDescription() + "\""; @@ -21,8 +14,8 @@ public String getTargetDescription() { protected abstract String getDataStructure(); protected String getKeyspaceDescription() { - String description = keyspace == null ? "" : keyspace; - for (String key : keys) { + String description = getKeyspace() == null ? "" : getKeyspace(); + for (String key : getKeys()) { description += ":<" + key + ">"; } return description; @@ -37,11 +30,15 @@ protected AbstractRedisItemWriter itemWriter() { protected RedisConverter redisConverter() { RedisConverter converter = new RedisConverter(); - converter.setKeyspace(keyspace); - converter.setKeys(keys); + converter.setKeyspace(getKeyspace()); + converter.setKeys(getKeys()); return converter; } protected abstract AbstractRedisItemWriter redisItemWriter(); + protected abstract String getKeyspace(); + + protected abstract String[] getKeys(); + } diff --git a/src/main/java/com/redislabs/riot/cli/redis/AbstractSingleRedisImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/AbstractSingleRedisImportSubSubCommand.java new file mode 100644 index 000000000..0c9e9da50 --- /dev/null +++ b/src/main/java/com/redislabs/riot/cli/redis/AbstractSingleRedisImportSubSubCommand.java @@ -0,0 +1,22 @@ +package com.redislabs.riot.cli.redis; + +import picocli.CommandLine.Option; + +public abstract class AbstractSingleRedisImportSubSubCommand extends AbstractRedisImportSubSubCommand { + + @Option(names = "--keyspace", required = true, description = "Redis keyspace prefix.") + private String keyspace; + @Option(names = "--keys", arity = "1..*", required = true, description = "Key fields.") + private String[] keys = new String[0]; + + @Override + protected String getKeyspace() { + return keyspace; + } + + @Override + protected String[] getKeys() { + return keys; + } + +} diff --git a/src/main/java/com/redislabs/riot/cli/redis/HashImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/HashImportSubSubCommand.java index 5b48a67e4..983631568 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/HashImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/HashImportSubSubCommand.java @@ -6,7 +6,7 @@ import picocli.CommandLine.Command; @Command(name = "hash", description = "Hash data structure") -public class HashImportSubSubCommand extends AbstractRedisImportSubSubCommand { +public class HashImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand { @Override protected AbstractRedisItemWriter redisItemWriter() { diff --git a/src/main/java/com/redislabs/riot/cli/redis/StreamImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/StreamImportSubSubCommand.java index 259d8f29d..ded5a446b 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/StreamImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/StreamImportSubSubCommand.java @@ -7,7 +7,7 @@ import picocli.CommandLine.Option; @Command(name = "stream", description = "Stream data structure") -public class StreamImportSubSubCommand extends AbstractRedisImportSubSubCommand { +public class StreamImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand { @Option(names = "--approximate-trimming", description = "Apply efficient trimming for capped streams using the ~ flag.") private boolean approximateTrimming; diff --git a/src/main/java/com/redislabs/riot/cli/redis/StringImportSubSubCommand.java b/src/main/java/com/redislabs/riot/cli/redis/StringImportSubSubCommand.java index a74c9c280..296b69b3b 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/StringImportSubSubCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/StringImportSubSubCommand.java @@ -8,7 +8,7 @@ import picocli.CommandLine.Option; @Command(name = "string", description = "String data structure") -public class StringImportSubSubCommand extends AbstractRedisImportSubSubCommand { +public class StringImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand { @Option(names = "--format", description = "Serialization format: ${COMPLETION-CANDIDATES}") private StringFormat format = StringFormat.Json; diff --git a/src/main/java/com/redislabs/riot/redis/writer/AbstractRedisCommands.java b/src/main/java/com/redislabs/riot/redis/writer/AbstractRedisCommands.java index 93391f6cf..5ab3a9609 100644 --- a/src/main/java/com/redislabs/riot/redis/writer/AbstractRedisCommands.java +++ b/src/main/java/com/redislabs/riot/redis/writer/AbstractRedisCommands.java @@ -12,10 +12,18 @@ public abstract class AbstractRedisCommands implements RedisCommands { protected Map stringMap(Map item) { Map stringMap = new HashMap(); - for (String key : item.keySet()) { - Object value = item.get(key); - stringMap.put(key, converter.convert(value, String.class)); - } + item.forEach((k, v) -> put(stringMap, k, v)); return stringMap; } + + private void put(Map map, String key, Object value) { + if (value == null) { + return; + } + if (value instanceof Map) { + ((Map) value).forEach((k, v) -> put(map, key + "." + converter.convert(k, String.class), v)); + } else { + map.put(key, converter.convert(value, String.class)); + } + } } diff --git a/src/test/java/com/redislabs/riot/AbstractBaseTest.java b/src/test/java/com/redislabs/riot/AbstractBaseTest.java deleted file mode 100644 index 0f498c866..000000000 --- a/src/test/java/com/redislabs/riot/AbstractBaseTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.redislabs.riot; - -import java.io.IOException; - -import org.junit.After; -import org.junit.Before; - -import com.redislabs.lettusearch.RediSearchClient; -import com.redislabs.lettusearch.RediSearchCommands; -import com.redislabs.lettusearch.StatefulRediSearchConnection; -import com.redislabs.lettusearch.search.Schema; -import com.redislabs.lettusearch.search.Schema.SchemaBuilder; -import com.redislabs.lettusearch.search.field.NumericField; -import com.redislabs.lettusearch.search.field.PhoneticMatcher; -import com.redislabs.lettusearch.search.field.TextField; - -public abstract class AbstractBaseTest { - - protected final static String FIELD_ABV = "abv"; - protected final static String FIELD_ID = "id"; - protected final static String FIELD_NAME = "name"; - protected final static String FIELD_STYLE = "style"; - protected final static String FIELD_OUNCES = "ounces"; - protected final static String INDEX = "beers"; - - private RediSearchClient client; - protected StatefulRediSearchConnection connection; - - @Before - public void setup() throws IOException { - client = RediSearchClient.create("redis://localhost"); - connection = client.connect(); - RediSearchCommands commands = connection.sync(); - commands.flushall(); - SchemaBuilder schema = Schema.builder(); - schema.field(TextField.builder().name(FIELD_NAME).sortable(true).build()); - schema.field(TextField.builder().name(FIELD_STYLE).matcher(PhoneticMatcher.English).sortable(true).build()); - schema.field(NumericField.builder().name(FIELD_ABV).sortable(true).build()); - schema.field(NumericField.builder().name(FIELD_OUNCES).sortable(true).build()); - commands.create(INDEX, schema.build()); - } - - @After - public void teardown() { - if (connection != null) { - connection.close(); - } - if (client != null) { - client.shutdown(); - } - } - -} diff --git a/src/test/java/com/redislabs/riot/BaseTest.java b/src/test/java/com/redislabs/riot/BaseTest.java new file mode 100644 index 000000000..4b1788897 --- /dev/null +++ b/src/test/java/com/redislabs/riot/BaseTest.java @@ -0,0 +1,36 @@ +package com.redislabs.riot; + +import java.io.IOException; + +import org.junit.After; +import org.junit.Before; + +import com.redislabs.lettusearch.RediSearchClient; +import com.redislabs.lettusearch.StatefulRediSearchConnection; + +public class BaseTest { + + private RediSearchClient client; + protected StatefulRediSearchConnection connection; + + @Before + public void setup() throws IOException { + client = RediSearchClient.create("redis://localhost"); + connection = client.connect(); + } + + @After + public void teardown() { + if (connection != null) { + connection.close(); + } + if (client != null) { + client.shutdown(); + } + } + + protected void run(String string) { + RiotApplication.main(string.split(" ")); + } + +} diff --git a/src/test/java/com/redislabs/riot/FileImportTest.java b/src/test/java/com/redislabs/riot/FileImportTest.java deleted file mode 100644 index 509d02c7c..000000000 --- a/src/test/java/com/redislabs/riot/FileImportTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.redislabs.riot; - -import java.util.List; - -import org.junit.Assert; -import org.junit.Test; - -public class FileImportTest extends AbstractBaseTest { - - @Test - public void testFileImport() throws Exception { - String[] args = "import csv --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header hash --keyspace beer --keys id" - .split(" "); - RiotApplication.main(args); - List keys = connection.sync().keys("beer:*"); - Assert.assertEquals(2410, keys.size()); - } - -} diff --git a/src/test/java/com/redislabs/riot/ImportCSV.java b/src/test/java/com/redislabs/riot/ImportCSV.java new file mode 100644 index 000000000..738c41ed7 --- /dev/null +++ b/src/test/java/com/redislabs/riot/ImportCSV.java @@ -0,0 +1,16 @@ +package com.redislabs.riot; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +public class ImportCSV extends BaseTest { + + @Test + public void importHashes() throws Exception { + run("import csv --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header hash --keyspace beer --keys id"); + List keys = connection.sync().keys("beer:*"); + Assert.assertEquals(2410, keys.size()); + } +} diff --git a/src/test/java/com/redislabs/riot/ImportJSON.java b/src/test/java/com/redislabs/riot/ImportJSON.java new file mode 100644 index 000000000..2c406fa95 --- /dev/null +++ b/src/test/java/com/redislabs/riot/ImportJSON.java @@ -0,0 +1,31 @@ +package com.redislabs.riot; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +public class ImportJSON extends BaseTest { + + @Test + public void importJSON() throws MalformedURLException, URISyntaxException { + String url = getClass().getClassLoader().getResource("es_test-index.json").toURI().toURL().toString(); + run("import json --url " + url + " hash --keyspace estest --keys _id"); + Assert.assertEquals(2, connection.sync().keys("estest:*").size()); + Map doc1 = connection.sync().hgetall("estest:doc1"); + Assert.assertEquals("ruan", doc1.get("_source.name")); + Assert.assertEquals("1,3", doc1.get("_source.articles")); + } + + @Test + public void importBeersJson() { + run("import json --url=https://raw.githubusercontent.com/rethinkdb/beerthink/master/data/beers.json hash --keyspace beerjson --keys id"); + List keys = connection.sync().keys("beerjson:*"); + Assert.assertEquals(4432, keys.size()); + Map beer1 = connection.sync().hgetall("beerjson:1"); + Assert.assertEquals("Hocus Pocus", beer1.get("name")); + } +} diff --git a/src/test/java/com/redislabs/riot/ImportToRediSearch.java b/src/test/java/com/redislabs/riot/ImportToRediSearch.java new file mode 100644 index 000000000..3ef8f55f2 --- /dev/null +++ b/src/test/java/com/redislabs/riot/ImportToRediSearch.java @@ -0,0 +1,36 @@ +package com.redislabs.riot; + +import org.junit.Assert; +import org.junit.Test; + +import com.redislabs.lettusearch.RediSearchCommands; +import com.redislabs.lettusearch.search.Schema; +import com.redislabs.lettusearch.search.Schema.SchemaBuilder; +import com.redislabs.lettusearch.search.SearchResults; +import com.redislabs.lettusearch.search.field.NumericField; +import com.redislabs.lettusearch.search.field.PhoneticMatcher; +import com.redislabs.lettusearch.search.field.TextField; + +public class ImportToRediSearch extends BaseTest { + + @Test + public void importBeers() throws Exception { + String FIELD_ABV = "abv"; + String FIELD_NAME = "name"; + String FIELD_STYLE = "style"; + String FIELD_OUNCES = "ounces"; + String INDEX = "beerIdx"; + RediSearchCommands commands = connection.sync(); + commands.flushall(); + SchemaBuilder schema = Schema.builder(); + schema.field(TextField.builder().name(FIELD_NAME).sortable(true).build()); + schema.field(TextField.builder().name(FIELD_STYLE).matcher(PhoneticMatcher.English).sortable(true).build()); + schema.field(NumericField.builder().name(FIELD_ABV).sortable(true).build()); + schema.field(NumericField.builder().name(FIELD_OUNCES).sortable(true).build()); + commands.create(INDEX, schema.build()); + run("import csv --header --included-fields=1,3,4,5,6 --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv search --index beerIdx --keys id"); + SearchResults results = connection.sync().search(INDEX, "*"); + Assert.assertEquals(2410, results.getCount()); + } + +} diff --git a/src/test/resources/beers.json b/src/test/resources/beers.json new file mode 100644 index 000000000..f94665234 --- /dev/null +++ b/src/test/resources/beers.json @@ -0,0 +1,33 @@ +[ + { + "id": "1", + "brewery_id": "812", + "name": "Hocus Pocus", + "abv": "4.5", + "ibu": "0", + "srm": "0", + "upc": "0", + "filepath": "", + "descript": "Our take on a classic summer ale. A toast to weeds, rays, and summer haze. A light, crisp ale for mowing lawns, hitting lazy fly balls, and communing with nature, Hocus Pocus is offered up as a summer sacrifice to clodless days.\n\nIts malty sweetness finishes tart and crisp and is best apprediated with a wedge of orange.", + "add_user": "0", + "last_mod": "2010-07-22 20:00:20 UTC", + "style_name": "Light American Wheat Ale or Lager", + "cat_name": "Other Style" + }, + { + "id": "6", + "brewery_id": "1385", + "name": "Winter Warmer", + "abv": "5.199999809265137", + "ibu": "0", + "srm": "0", + "upc": "0", + "filepath": "", + "descript": "", + "add_user": "0", + "last_mod": "2010-07-22 20:00:20 UTC", + "style_name": "Old Ale", + "cat_name": "British Ale" + } +] + diff --git a/src/test/resources/es_test-index.json b/src/test/resources/es_test-index.json new file mode 100644 index 000000000..9cd78569f --- /dev/null +++ b/src/test/resources/es_test-index.json @@ -0,0 +1,29 @@ +[ + { + "_index": "test-index", + "_type": "docs", + "_id": "doc1", + "_score": 1, + "_source": { + "name": "ruan", + "age": 30, + "articles": [ + "1", + "3" + ] + } + }, + { + "_index": "test-index", + "_type": "docs", + "_id": "doc2", + "_score": 1, + "_source": { + "name": "stefan", + "age": 29, + "articles": [ + "2" + ] + } + } +] \ No newline at end of file