Skip to content

Commit

Permalink
Merge pull request #42 from zuliaio/termsQueryRefactor
Browse files Browse the repository at this point in the history 
refactor client protocol to allow more advanced types of queries
  • Loading branch information
@payammeyer
payammeyer authored Aug 30, 2020
2 parents 6a01649 + 4c7d476 commit 138e02e
Show file tree
Hide file tree
Showing 8 changed files with 252 additions and 178 deletions.
129 changes: 76 additions & 53 deletions zulia-client/src/main/java/io/zulia/client/command/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.zulia.client.result.QueryResult;
import io.zulia.message.ZuliaQuery;
import io.zulia.message.ZuliaQuery.FieldSort.Direction;
import io.zulia.message.ZuliaQuery.Query.QueryType;
import io.zulia.message.ZuliaServiceGrpc;

import java.util.ArrayList;
Expand All @@ -19,7 +20,6 @@

import static io.zulia.message.ZuliaBase.Similarity;
import static io.zulia.message.ZuliaQuery.AnalysisRequest;
import static io.zulia.message.ZuliaQuery.CosineSimRequest;
import static io.zulia.message.ZuliaQuery.CountRequest;
import static io.zulia.message.ZuliaQuery.Facet;
import static io.zulia.message.ZuliaQuery.FacetRequest;
Expand Down Expand Up @@ -49,8 +49,10 @@ public class Query extends SimpleCommand<QueryRequest, QueryResult> implements M
private List<Facet> drillDowns = Collections.emptyList();
private List<FieldSort> fieldSorts = Collections.emptyList();
private Set<String> queryFields = Collections.emptySet();
private List<ZuliaQuery.Query> filterQueries = Collections.emptyList();
private List<ZuliaQuery.Query> scoredQueries = Collections.emptyList();
private List<ZuliaQuery.Query.Builder> filterQueries = Collections.emptyList();
private List<ZuliaQuery.Query.Builder> scoredQueries = Collections.emptyList();
private List<ZuliaQuery.Query.Builder> termQueries = Collections.emptyList();
private List<ZuliaQuery.Query.Builder> cosineSimQueries = Collections.emptyList();
private List<ZuliaQuery.HighlightRequest> highlightRequests = Collections.emptyList();
private List<ZuliaQuery.AnalysisRequest> analysisRequests = Collections.emptyList();
private Integer minimumNumberShouldMatch;
Expand All @@ -59,7 +61,7 @@ public class Query extends SimpleCommand<QueryRequest, QueryResult> implements M
private Set<String> documentFields = Collections.emptySet();
private Set<String> documentMaskedFields = Collections.emptySet();
private List<FieldSimilarity> fieldSimilarities = Collections.emptyList();
private List<CosineSimRequest> cosineSimRequests = Collections.emptyList();

private Boolean dismax;
private Float dismaxTie;
private Boolean dontCache;
Expand Down Expand Up @@ -200,22 +202,39 @@ private Query addFieldSimilarity(FieldSimilarity fieldSimilarity) {
return this;
}

public Query addCosineSim(String field, double[] vector, double similarity) {
public Query addTermQuery(Iterable<String> terms, String... field) {

ZuliaQuery.Query.Builder builder = ZuliaQuery.Query.newBuilder().addAllQf(Arrays.asList(field)).addAllTerm(terms);
builder.setQueryType(QueryType.TERMS);
addTermQuery(builder);
return this;
}

private Query addTermQuery(ZuliaQuery.Query.Builder termQuery) {
if (termQueries.isEmpty()) {
termQueries = new ArrayList<>();
}
termQueries.add(termQuery);

return this;
}

public Query addCosineSim(double[] vector, double similarity, String... field) {

CosineSimRequest.Builder builder = CosineSimRequest.newBuilder().setField(field).setSimilarity(similarity);
ZuliaQuery.Query.Builder builder = ZuliaQuery.Query.newBuilder().addAllQf(Arrays.asList(field)).setVectorSimilarity(similarity);
for (int i = 0; i < vector.length; i++) {
builder.addVector(vector[i]);
}
addCosineSim(builder.build());
addCosineSim(builder);
return this;
}

private Query addCosineSim(CosineSimRequest cosineSimRequest) {
if (cosineSimRequests.isEmpty()) {
cosineSimRequests = new ArrayList<>();
private Query addCosineSim(ZuliaQuery.Query.Builder cosineSimQuery) {
if (cosineSimQueries.isEmpty()) {
cosineSimQueries = new ArrayList<>();
}

cosineSimRequests.add(cosineSimRequest);
cosineSimQuery.setQueryType(QueryType.VECTOR);
cosineSimQueries.add(cosineSimQuery);

return this;
}
Expand All @@ -240,11 +259,11 @@ public Query addQueryField(String... queryFields) {
return this;
}

public List<ZuliaQuery.Query> getFilterQueries() {
public List<ZuliaQuery.Query.Builder> getFilterQueries() {
return filterQueries;
}

public void setFilterQueries(List<ZuliaQuery.Query> filterQueries) {
public void setFilterQueries(List<ZuliaQuery.Query.Builder> filterQueries) {
this.filterQueries = filterQueries;
}

Expand All @@ -270,6 +289,14 @@ public Query addFilterQuery(String query, Collection<String> queryFields, Operat
}

ZuliaQuery.Query.Builder builder = ZuliaQuery.Query.newBuilder();
builder.setQueryType(QueryType.FILTER);
fillInQuery(query, queryFields, defaultOperator, minimumNumberShouldMatch, builder);
filterQueries.add(builder);
return this;
}

private void fillInQuery(String query, Collection<String> queryFields, Operator defaultOperator, Integer minimumNumberShouldMatch,
ZuliaQuery.Query.Builder builder) {
if (query != null && !query.isEmpty()) {
builder.setQ(query);
}
Expand All @@ -282,53 +309,45 @@ public Query addFilterQuery(String query, Collection<String> queryFields, Operat
if (queryFields != null && !queryFields.isEmpty()) {
builder.addAllQf(queryFields);
}
filterQueries.add(builder.build());
return this;
}

public List<ZuliaQuery.Query> getScoredQueries() {
public List<ZuliaQuery.Query.Builder> getScoredQueries() {
return scoredQueries;
}

public void setScoredQueries(List<ZuliaQuery.Query> scoredQueries) {
public void setScoredQueries(List<ZuliaQuery.Query.Builder> scoredQueries) {
this.scoredQueries = scoredQueries;
}

public Query addScoredQuery(String query) {
return addScoredQuery(query, null, null, null);
return addScoredQuery(query, null, null, null, true);
}

public Query addScoredQuery(String query, Collection<String> queryFields) {
return addScoredQuery(query, queryFields, null, null);
return addScoredQuery(query, queryFields, null, null, true);
}

public Query addScoredQuery(String query, Collection<String> queryFields, Operator defaultOperator) {
return addScoredQuery(query, queryFields, defaultOperator, null);
return addScoredQuery(query, queryFields, defaultOperator, null, true);
}

public Query addScoredQuery(String query, Collection<String> queryFields, Integer minimumNumberShouldMatch) {
return addScoredQuery(query, queryFields, null, minimumNumberShouldMatch);
return addScoredQuery(query, queryFields, null, minimumNumberShouldMatch, true);
}

public Query addScoredQuery(String query, Collection<String> queryFields, Integer minimumNumberShouldMatch, boolean must) {
return addScoredQuery(query, queryFields, null, minimumNumberShouldMatch, must);
}

public Query addScoredQuery(String query, Collection<String> queryFields, Operator defaultOperator, Integer minimumNumberShouldMatch) {
public Query addScoredQuery(String query, Collection<String> queryFields, Operator defaultOperator, Integer minimumNumberShouldMatch, boolean must) {
if (scoredQueries.isEmpty()) {
this.scoredQueries = new ArrayList<>();
}

ZuliaQuery.Query.Builder builder = ZuliaQuery.Query.newBuilder();
if (query != null && !query.isEmpty()) {
builder.setQ(query);
}
if (minimumNumberShouldMatch != null) {
builder.setMm(minimumNumberShouldMatch);
}
if (defaultOperator != null) {
builder.setDefaultOp(defaultOperator);
}
if (queryFields != null && !queryFields.isEmpty()) {
builder.addAllQf(queryFields);
}
scoredQueries.add(builder.build());
builder.setQueryType(must ? QueryType.SCORE_MUST : QueryType.SCORE_SHOULD);
fillInQuery(query, queryFields, defaultOperator, minimumNumberShouldMatch, builder);
scoredQueries.add(builder);
return this;
}

Expand Down Expand Up @@ -523,28 +542,29 @@ public QueryRequest getRequest() {
requestBuilder.setDebug(debug);
}

ZuliaQuery.Query.Builder queryBuilder = ZuliaQuery.Query.newBuilder();
ZuliaQuery.Query.Builder mainQueryBuilder = ZuliaQuery.Query.newBuilder();
if (query != null) {
queryBuilder.setQ(query);
mainQueryBuilder.setQ(query);
}
if (minimumNumberShouldMatch != null) {
queryBuilder.setMm(minimumNumberShouldMatch);
mainQueryBuilder.setMm(minimumNumberShouldMatch);
}
if (!queryFields.isEmpty()) {
queryBuilder.addAllQf(queryFields);
mainQueryBuilder.addAllQf(queryFields);
}
if (defaultOperator != null) {
queryBuilder.setDefaultOp(defaultOperator);
mainQueryBuilder.setDefaultOp(defaultOperator);
}

if (dismax != null) {
queryBuilder.setDismax(dismax);
mainQueryBuilder.setDismax(dismax);
if (dismaxTie != null) {
queryBuilder.setDismaxTie(dismaxTie);
mainQueryBuilder.setDismaxTie(dismaxTie);
}
}

requestBuilder.setQuery(queryBuilder);
mainQueryBuilder.setQueryType(QueryType.SCORE_MUST);
requestBuilder.addQuery(mainQueryBuilder);

if (lastResult != null) {
requestBuilder.setLastResult(lastResult);
Expand All @@ -569,12 +589,20 @@ public QueryRequest getRequest() {

}

if (!filterQueries.isEmpty()) {
requestBuilder.addAllFilterQuery(filterQueries);
for (ZuliaQuery.Query.Builder filterQuery : filterQueries) {
requestBuilder.addQuery(filterQuery.setQueryType(QueryType.FILTER));
}

if (!scoredQueries.isEmpty()) {
requestBuilder.addAllScoredQuery(scoredQueries);
for (ZuliaQuery.Query.Builder cosineSimQuery : cosineSimQueries) {
requestBuilder.addQuery(cosineSimQuery.setQueryType(QueryType.VECTOR));
}

for (ZuliaQuery.Query.Builder termQuery : termQueries) {
requestBuilder.addQuery(termQuery.setQueryType(QueryType.TERMS));
}

for (ZuliaQuery.Query.Builder scoredQuery : scoredQueries) {
requestBuilder.addQuery(scoredQuery);
}

if (!highlightRequests.isEmpty()) {
Expand All @@ -588,11 +616,6 @@ public QueryRequest getRequest() {
if (resultFetchType != null) {
requestBuilder.setResultFetchType(resultFetchType);
}

if (!cosineSimRequests.isEmpty()) {
requestBuilder.addAllCosineSimRequest(cosineSimRequests);
}

if (dontCache != null) {
requestBuilder.setDontCache(dontCache);
}
Expand Down
2 changes: 1 addition & 1 deletion zulia-common/src/main/java/io/zulia/ZuliaConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public interface ZuliaConstants {
String QUERY = "q";
String QUERY_FIELD = "qf";
String FILTER_QUERY = "fq";
String FILTER_QUERY_JSON = "fqJson";
String QUERY_JSON = "qJson";
String ROWS = "rows";
String ID = "id";
String FILE_NAME = "fileName";
Expand Down
32 changes: 19 additions & 13 deletions zulia-common/src/main/proto/zulia_query.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,30 @@ import "zulia_base.proto";
option java_package = "io.zulia.message";

message Query {
enum QueryType {
FILTER = 0;
SCORE_MUST = 1;
SCORE_SHOULD = 2;
TERMS = 3;
VECTOR = 4;
}

enum Operator {
OR = 0;
AND = 1;
}

string q = 1;
Operator defaultOp = 2; // default OR
repeated string qf = 3;
uint32 mm = 4; // default 0
bool dismax = 5;
float dismaxTie = 6; // default 0
QueryType queryType = 1;
string q = 2;
Operator defaultOp = 3; // default OR
repeated string qf = 4;
uint32 mm = 5; // default 0
bool dismax = 6;
float dismaxTie = 7; // default 0

repeated string term = 8;
double vectorSimilarity = 9;
repeated double vector = 10;

}

Expand Down Expand Up @@ -158,13 +171,6 @@ message AnalysisResult {
repeated Term terms = 3;
}


message CosineSimRequest {
string field = 1;
double similarity = 2;
repeated double vector = 3;
}

message FieldSimilarity {
string field = 1;
Similarity similarity = 2;
Expand Down
33 changes: 15 additions & 18 deletions zulia-common/src/main/proto/zulia_service.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,25 +52,22 @@ message InternalQueryRequest {

message QueryRequest {
repeated string index = 1;
Query query = 2;
repeated Query query = 2;
uint32 amount = 3;
repeated Query filterQuery = 4;
FacetRequest facetRequest = 5;
SortRequest sortRequest = 6;
uint32 start = 7;
LastResult lastResult = 8;
FetchType resultFetchType = 9;
repeated string documentFields = 10;
repeated string documentMaskedFields = 11;
bool fetchFull = 12;
repeated FieldSimilarity fieldSimilarity = 13;
repeated HighlightRequest highlightRequest = 14;
repeated AnalysisRequest analysisRequest = 15;
bool debug = 16;
repeated CosineSimRequest cosineSimRequest = 17;
bool dontCache = 18;
MasterSlaveSettings masterSlaveSettings = 19;
repeated Query scoredQuery = 20;
FacetRequest facetRequest = 4;
SortRequest sortRequest = 5;
uint32 start = 6;
LastResult lastResult = 7;
FetchType resultFetchType = 8;
repeated string documentFields = 9;
repeated string documentMaskedFields = 10;
bool fetchFull = 11;
repeated FieldSimilarity fieldSimilarity = 12;
repeated HighlightRequest highlightRequest = 13;
repeated AnalysisRequest analysisRequest = 14;
bool debug = 15;
bool dontCache = 16;
MasterSlaveSettings masterSlaveSettings = 17;
}

message QueryResponse {
Expand Down
Loading

0 comments on commit 138e02e

Please sign in to comment.