Skip to content

Commit

Permalink
improve field config api for easier index schema creation
Browse files Browse the repository at this point in the history
  • Loading branch information
mdavis95 committed Apr 5, 2022
1 parent df9557f commit c4cb247
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 76 deletions.
28 changes: 28 additions & 0 deletions zulia-client/src/main/java/io/zulia/fields/FieldConfigBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,34 @@ public static FieldConfigBuilder create(String storedFieldName, FieldConfig.Fiel
return new FieldConfigBuilder(storedFieldName, fieldType);
}

public static FieldConfigBuilder createString(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.STRING);
}

public static FieldConfigBuilder createBool(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.BOOL);
}

public static FieldConfigBuilder createDate(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.DATE);
}

public static FieldConfigBuilder createInt(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.NUMERIC_INT);
}

public static FieldConfigBuilder createLong(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.NUMERIC_LONG);
}

public static FieldConfigBuilder createFloat(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.NUMERIC_FLOAT);
}

public static FieldConfigBuilder createDouble(String storedFieldName) {
return create(storedFieldName, FieldConfig.FieldType.NUMERIC_DOUBLE);
}

public FieldConfigBuilder index() {
return indexAs(null, storedFieldName);
}
Expand Down
60 changes: 60 additions & 0 deletions zulia-server/src/test/java/io/zulia/server/test/QueryTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package io.zulia.server.test;

import io.zulia.client.command.builder.ScoredQuery;
import io.zulia.client.command.builder.Search;
import io.zulia.client.command.builder.Sort;
import io.zulia.client.config.ZuliaPoolConfig;
import io.zulia.client.pool.ZuliaWorkPool;
import io.zulia.client.result.SearchResult;
import io.zulia.message.ZuliaQuery;

public class QueryTest {

public static void main(String[] args) throws Exception {
ZuliaWorkPool zuliaWorkPool = new ZuliaWorkPool(new ZuliaPoolConfig().addNode("localhost"));

Search search = new Search("publicgrants20220303");
search.setAmount(500);
search.addQuery(new ScoredQuery("breast AND cancer").addQueryFields("title", "abstract"));
search.addDocumentFields("title", "abstract");

SearchResult searchResult = zuliaWorkPool.search(search);
System.out.println(searchResult.getTotalHits());
System.out.println("breast AND cancer Top 100," + searchResult.getCommandTimeMs());

search.addSort(new Sort("title"));
searchResult = zuliaWorkPool.search(search);
System.out.println("breast AND cancer Top 100 Sort Title," + searchResult.getCommandTimeMs());

search.clearSort();
long start = System.currentTimeMillis();
zuliaWorkPool.searchAllAsDocument(search, document -> {
//no-op
});
long end = System.currentTimeMillis();
System.out.println("breast AND cancer All Matches," + (end-start));

search.setResultFetchType(ZuliaQuery.FetchType.NONE);
search.clearSort();
search.clearLastResult();
search.addSort(new Sort("id"));
start = System.currentTimeMillis();
zuliaWorkPool.searchAllAsScoredResult(search, scoredResult -> {
//no-op
});
end = System.currentTimeMillis();
System.out.println("breast AND cancer All Matches Sort Id (None)," + (end-start));

search.setResultFetchType(ZuliaQuery.FetchType.FULL);
search.clearSort();
search.clearLastResult();
search.addSort(new Sort("id"));
start = System.currentTimeMillis();
zuliaWorkPool.searchAllAsDocument(search, document -> {
//no-op
});
end = System.currentTimeMillis();
System.out.println("breast AND cancer All Matches Sort Id (Full)," + (end-start));

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import io.zulia.client.result.SearchResult;
import io.zulia.doc.ResultDocBuilder;
import io.zulia.fields.FieldConfigBuilder;
import io.zulia.message.ZuliaIndex.FieldConfig.FieldType;
import org.bson.Document;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -49,9 +48,9 @@ public static void initAll() throws Exception {

ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("rating", FieldType.NUMERIC_DOUBLE).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createDouble("rating").index().sort());
indexConfig.setIndexName(ALIAS_TEST_INDEX);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import io.zulia.client.result.SearchResult;
import io.zulia.doc.ResultDocBuilder;
import io.zulia.fields.FieldConfigBuilder;
import io.zulia.message.ZuliaIndex.FieldConfig.FieldType;
import io.zulia.message.ZuliaQuery.FacetCount;
import org.bson.Document;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -51,12 +50,12 @@ public static void initAll() throws Exception {

ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("path", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("date", FieldType.DATE).index().facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("normalFacet", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("normalFacetList", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("path").indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createDate("date").index().facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("normalFacet").indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("normalFacetList").indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.setIndexName(FACET_TEST_INDEX);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down Expand Up @@ -226,11 +225,11 @@ else if (path.getFacet().equals("3")) {
public void reindex() throws Exception {
ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("path", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("path2", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("date", FieldType.DATE).index().facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("path").indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("path2").indexAs(DefaultAnalyzers.LC_KEYWORD).facetHierarchical().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createDate("date").index().facetHierarchical().sort());
indexConfig.setIndexName(FACET_TEST_INDEX);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down
37 changes: 18 additions & 19 deletions zulia-server/src/test/java/io/zulia/server/test/node/SortTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import io.zulia.client.result.SearchResult;
import io.zulia.doc.ResultDocBuilder;
import io.zulia.fields.FieldConfigBuilder;
import io.zulia.message.ZuliaIndex.FieldConfig.FieldType;
import org.bson.Document;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -54,14 +53,14 @@ public void indexingTest() throws Exception {

ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("stars", FieldType.NUMERIC_INT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("starsLong", FieldType.NUMERIC_LONG).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("rating", FieldType.NUMERIC_FLOAT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("ratingDouble", FieldType.NUMERIC_FLOAT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("special", FieldType.BOOL).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("added", FieldType.DATE).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("stars").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createLong("starsLong").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createFloat("rating").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createFloat("ratingDouble").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createBool("special").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createDate("added").index().sort());
indexConfig.setIndexName(INDEX_NAME);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down Expand Up @@ -523,18 +522,18 @@ public void afterRestartRetry() throws Exception {
public void reindexTest() throws Exception {
ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD)
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD)
.sortAs("theId")); //change sort as to be theId instead of just id
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("stars", FieldType.NUMERIC_INT).index()); // no longer sortable
indexConfig.addFieldConfig(FieldConfigBuilder.create("starsLong", FieldType.NUMERIC_LONG).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("stars").index()); // no longer sortable
indexConfig.addFieldConfig(FieldConfigBuilder.createLong("starsLong").index().sort());
//indexConfig.addFieldConfig(FieldConfigBuilder.create("rating", FieldType.NUMERIC_FLOAT).index().sort()); // no longer indexed or sortable
indexConfig.addFieldConfig(FieldConfigBuilder.create("ratingDouble", FieldType.NUMERIC_FLOAT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("special", FieldType.BOOL).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("added", FieldType.DATE).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createFloat("ratingDouble").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createBool("special").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createDate("added").index().sort());
//sort() adds standard string (case senstive sorting with a field name the same as the stored field
//sortAs(LOWERCASE_FOLDING, "otherTitleFolding") add another sortable field with a lowercase and ascii folding filter applied to make case insensitive sort and fancy letter insensitive (gotta be a better term here)
indexConfig.addFieldConfig(FieldConfigBuilder.create("otherTitle", FieldType.STRING).index().sort().sortAs(LOWERCASE_FOLDING, "otherTitleFolding"));
indexConfig.addFieldConfig(FieldConfigBuilder.createString("otherTitle").index().sort().sortAs(LOWERCASE_FOLDING, "otherTitleFolding"));
indexConfig.setIndexName(INDEX_NAME);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down Expand Up @@ -574,8 +573,8 @@ public void multiIndexTest() throws Exception {

ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("magicNumber");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.NUMERIC_INT).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("magicNumber", FieldType.NUMERIC_INT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("id").sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("magicNumber").index().sort());
indexConfig.setIndexName("anotherIndex");
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import io.zulia.doc.ResultDocBuilder;
import io.zulia.fields.FieldConfigBuilder;
import io.zulia.message.ZuliaIndex.FacetAs.DateHandling;
import io.zulia.message.ZuliaIndex.FieldConfig.FieldType;
import io.zulia.message.ZuliaQuery;
import io.zulia.message.ZuliaQuery.FacetCount;
import io.zulia.util.ZuliaUtil;
Expand Down Expand Up @@ -65,16 +64,16 @@ public static void initAll() throws Exception {

ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("issn", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("eissn", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.create("uid", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.create("an", FieldType.NUMERIC_INT).index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("country", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet());
indexConfig.addFieldConfig(FieldConfigBuilder.create("date", FieldType.DATE).index().facetAs(DateHandling.DATE_YYYY_MM_DD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("testList", FieldType.STRING).index());
indexConfig.addFieldConfig(FieldConfigBuilder.create("testBool", FieldType.BOOL).index().facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("issn").indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("eissn").indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.createString("uid").indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("an").index().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("country").indexAs(DefaultAnalyzers.LC_KEYWORD).facet());
indexConfig.addFieldConfig(FieldConfigBuilder.createDate("date").index().facetAs(DateHandling.DATE_YYYY_MM_DD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("testList").index());
indexConfig.addFieldConfig(FieldConfigBuilder.createBool("testBool").index().facet().sort());
indexConfig.setIndexName(FACET_TEST_INDEX);
indexConfig.setNumberOfShards(1);
indexConfig.setShardCommitInterval(20); //force some commits
Expand Down Expand Up @@ -370,16 +369,16 @@ public void termTestBuilder() throws Exception {
public void reindex() throws Exception {
ClientIndexConfig indexConfig = new ClientIndexConfig();
indexConfig.addDefaultSearchField("title");
indexConfig.addFieldConfig(FieldConfigBuilder.create("id", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("title", FieldType.STRING).indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("issn", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("eissn", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet());
indexConfig.addFieldConfig(FieldConfigBuilder.create("uid", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.create("an", FieldType.NUMERIC_INT).index().displayName("Accession Number").sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("country", FieldType.STRING).indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("id").indexAs(DefaultAnalyzers.LC_KEYWORD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("title").indexAs(DefaultAnalyzers.STANDARD).sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("issn").indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("eissn").indexAs(DefaultAnalyzers.LC_KEYWORD).facet());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("uid").indexAs(DefaultAnalyzers.LC_KEYWORD));
indexConfig.addFieldConfig(FieldConfigBuilder.createInt("an").index().displayName("Accession Number").sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("country").indexAs(DefaultAnalyzers.LC_KEYWORD).facet().sort());
indexConfig.addFieldConfig(
FieldConfigBuilder.create("date", FieldType.DATE).index().facetAs(DateHandling.DATE_YYYY_MM_DD).description("The very special data").sort());
indexConfig.addFieldConfig(FieldConfigBuilder.create("testList", FieldType.STRING).index());
FieldConfigBuilder.createDate("date").index().facetAs(DateHandling.DATE_YYYY_MM_DD).description("The very special data").sort());
indexConfig.addFieldConfig(FieldConfigBuilder.createString("testList").index());
indexConfig.setIndexName(FACET_TEST_INDEX);
indexConfig.setNumberOfShards(1);

Expand Down
Loading

0 comments on commit c4cb247

Please sign in to comment.