Merge pull request #47 from zuliaio/stats

Stats

zulia-analyzer/src/main/java/io/zulia/server/field/FieldTypeUtil.java

@@ -26,8 +26,16 @@ public static boolean isDateFieldType(FieldConfig.FieldType fieldType) {
 	}
 
 	public static boolean isNumericOrDateFieldType(FieldConfig.FieldType fieldType) {
+		return isNumericFieldType(fieldType) || isDateFieldType(fieldType);
+	}
+
+	public static boolean isNumericFieldType(FieldConfig.FieldType fieldType) {
 		return isNumericIntFieldType(fieldType) || isNumericLongFieldType(fieldType) || isNumericFloatFieldType(fieldType) || isNumericDoubleFieldType(
-				fieldType) || isDateFieldType(fieldType);
+				fieldType);
+	}
+
+	public static boolean isNumericFloatingPointFieldType(FieldConfig.FieldType fieldType) {
+		return isNumericFloatFieldType(fieldType) || isNumericDoubleFieldType(fieldType);
 	}
 
 	public static boolean isBooleanFieldType(FieldConfig.FieldType fieldType) {

zulia-client/src/main/java/io/zulia/client/command/builder/NumericStat.java

@@ -0,0 +1,17 @@
+package io.zulia.client.command.builder;
+
+import io.zulia.message.ZuliaQuery.StatRequest;
+
+public class NumericStat implements StatBuilder {
+
+	private final StatRequest.Builder statRequestBuilder;
+
+	public NumericStat(String numericField) {
+		statRequestBuilder = StatRequest.newBuilder().setNumericField(numericField);
+	}
+
+	@Override
+	public StatRequest getStatRequest() {
+		return statRequestBuilder.build();
+	}
+}

zulia-client/src/main/java/io/zulia/client/command/builder/Search.java

@@ -182,6 +182,16 @@ public Search clearFacetCount() {
 		return this;
 	}
 
+	public Search addStat(StatBuilder statBuilder) {
+		facetRequest.addStatRequest(statBuilder.getStatRequest());
+		return this;
+	}
+
+	public Search clearStat() {
+		facetRequest.clearStatRequest();
+		return this;
+	}
+
 	public Search addAnalysis(AnalysisBuilder analysisBuilder) {
 		queryRequest.addAnalysisRequest(analysisBuilder.getAnalysis());
 		return this;

zulia-client/src/main/java/io/zulia/client/command/builder/StatBuilder.java

@@ -0,0 +1,8 @@
+package io.zulia.client.command.builder;
+
+import io.zulia.message.ZuliaQuery;
+
+public interface StatBuilder {
+
+	ZuliaQuery.StatRequest getStatRequest();
+}

zulia-client/src/main/java/io/zulia/client/command/builder/StatFacet.java

@@ -0,0 +1,39 @@
+package io.zulia.client.command.builder;
+
+import io.zulia.message.ZuliaQuery;
+import io.zulia.message.ZuliaQuery.StatRequest;
+
+import java.util.Arrays;
+
+public class StatFacet implements StatBuilder {
+
+	private final StatRequest.Builder statRequestBuilder;
+
+	public StatFacet(String numericField, String facetField) {
+		statRequestBuilder = StatRequest.newBuilder().setNumericField(numericField).setFacetField(ZuliaQuery.Facet.newBuilder().setLabel(facetField).build());
+	}
+
+	public StatFacet(String numericField, String facetField, String... path) {
+		this(numericField, facetField, Arrays.asList(path));
+	}
+
+	public StatFacet(String numericField, String facetField, Iterable<String> path) {
+		statRequestBuilder = StatRequest.newBuilder().setNumericField(numericField)
+				.setFacetField(ZuliaQuery.Facet.newBuilder().setLabel(facetField).addAllPath(path).build());
+	}
+
+	public StatFacet setTopN(int topN) {
+		statRequestBuilder.setMaxFacets(topN);
+		return this;
+	}
+
+	public StatFacet setTopNShard(int topNShard) {
+		statRequestBuilder.setShardFacets(topNShard);
+		return this;
+	}
+
+	@Override
+	public StatRequest getStatRequest() {
+		return statRequestBuilder.build();
+	}
+}

zulia-client/src/main/java/io/zulia/client/result/QueryResult.java

@@ -5,8 +5,10 @@
 import io.zulia.message.ZuliaQuery.AnalysisResult;
 import io.zulia.message.ZuliaQuery.FacetCount;
 import io.zulia.message.ZuliaQuery.FacetGroup;
+import io.zulia.message.ZuliaQuery.FacetStats;
 import io.zulia.message.ZuliaQuery.LastResult;
 import io.zulia.message.ZuliaQuery.ScoredResult;
+import io.zulia.message.ZuliaQuery.StatGroup;
 import io.zulia.message.ZuliaServiceOuterClass.QueryResponse;
 import io.zulia.util.ResultHelper;
 import io.zulia.util.ZuliaUtil;
@@ -132,6 +134,44 @@ public int getFacetGroupCount() {
 		return queryResponse.getFacetGroupCount();
 	}
 
+	public List<StatGroup> getStatGroups() {
+		return queryResponse.getStatGroupList();
+	}
+
+	public FacetStats getNumericFieldStat(String numericFieldName) {
+		for (StatGroup sg : queryResponse.getStatGroupList()) {
+			if (numericFieldName.equals(sg.getStatRequest().getNumericField()) && sg.getStatRequest().getFacetField().getLabel().isEmpty()) {
+				return sg.getGlobalStats();
+			}
+		}
+		return null;
+	}
+
+	public List<FacetStats> getFacetFieldStat(String numericFieldName, String facetField) {
+		if (facetField == null) {
+			facetField = "";
+		}
+		for (StatGroup sg : queryResponse.getStatGroupList()) {
+			if (numericFieldName.equals(sg.getStatRequest().getNumericField()) && facetField.equals(sg.getStatRequest().getFacetField().getLabel())) {
+				return sg.getFacetStatsList();
+			}
+		}
+		return null;
+	}
+
+	public List<FacetStats> getFacetFieldStat(String numericFieldName, String facetField, List<String> paths) {
+		if (facetField == null) {
+			facetField = "";
+		}
+		for (StatGroup sg : queryResponse.getStatGroupList()) {
+			if (numericFieldName.equals(sg.getStatRequest().getNumericField()) && facetField.equals(sg.getStatRequest().getFacetField().getLabel())
+					&& paths.equals(sg.getStatRequest().getFacetField().getPathList())) {
+				return sg.getFacetStatsList();
+			}
+		}
+		return null;
+	}
+
 	public List<AnalysisResult> getSummaryAnalysisResults() {
 		return queryResponse.getAnalysisResultList();
 	}

zulia-common/src/main/proto/zulia_query.proto

@@ -48,6 +48,14 @@ message Facet {
 message FacetRequest {
     repeated CountRequest countRequest = 1;
     repeated Facet drillDown = 2;
+    repeated StatRequest statRequest = 3;
+}
+
+message StatRequest {
+    string numericField = 1;
+    Facet facetField = 2;
+    uint32 maxFacets = 3; // default 10, set to -1 to get all
+    uint32 shardFacets = 4; // defaults to maxFacets * 10, ignored for single shard indexes, set to -1 to get all
 }
 
 message CountRequest {
@@ -56,7 +64,6 @@ message CountRequest {
     uint32 shardFacets = 3; // defaults to maxFacets * 10, ignored for single shard indexes, set to -1 to get all
 }
 
-
 message FacetCount {
     string facet = 1;
     uint64 count = 2;
@@ -70,6 +77,22 @@ message FacetGroup {
     uint64 maxValuePossibleMissing = 4; // default 0
 }
 
+message FacetStats {
+    string facet = 1;
+    SortValue min = 2;
+    SortValue max = 3;
+    SortValue sum = 4;
+    uint64 docCount = 5;
+    uint64 valueCount = 6;
+}
+
+
+message StatGroup {
+    StatRequest statRequest = 1;
+    FacetStats globalStats = 2;
+    repeated FacetStats facetStats = 3;
+}
+
 message SortRequest {
     repeated FieldSort fieldSort = 1;
 }
@@ -197,4 +220,5 @@ message ShardQueryResponse {
     ScoredResult next = 5;
     repeated FacetGroup facetGroup = 6;
     repeated AnalysisResult analysisResult = 7;
+    repeated StatGroup statGroup = 8;
 }

zulia-common/src/main/proto/zulia_service.proto

@@ -76,6 +76,7 @@ message QueryResponse {
     LastResult lastResult = 3;
     repeated FacetGroup facetGroup = 4;
     repeated AnalysisResult analysisResult = 5;
+    repeated StatGroup statGroup = 6;
 }
 
 message InternalQueryResponse {

zulia-server/src/main/java/io/zulia/server/index/ShardReader.java

@@ -16,6 +16,7 @@
 import io.zulia.server.field.FieldTypeUtil;
 import io.zulia.server.search.QueryCacheKey;
 import io.zulia.server.search.QueryResultCache;
+import io.zulia.server.search.TaxonomyStatsHandler;
 import io.zulia.server.search.ZuliaQueryParser;
 import io.zulia.server.util.FieldAndSubFields;
 import io.zulia.util.ResultHelper;
@@ -24,7 +25,6 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetResult;
-import org.apache.lucene.facet.Facets;
 import org.apache.lucene.facet.FacetsCollector;
 import org.apache.lucene.facet.FacetsConfig;
 import org.apache.lucene.facet.LabelAndValue;
@@ -121,10 +121,6 @@ public void close() throws Exception {
 		taxoReader.close();
 	}
 
-	public Facets getFacets(FacetsCollector facetsCollector) throws IOException {
-		return new FastTaxonomyFacetCounts(taxoReader, facetsConfig, facetsCollector);
-	}
-
 	public int getTotalFacets() {
 		return taxoReader.getSize();
 	}
@@ -192,10 +188,21 @@ public ZuliaQuery.ShardQueryResponse queryShard(Query query, Map<String, ZuliaBa
 		ZuliaQuery.ShardQueryResponse.Builder shardQueryReponseBuilder = ZuliaQuery.ShardQueryResponse.newBuilder();
 
 		try {
-			if ((facetRequest != null) && !facetRequest.getCountRequestList().isEmpty()) {
+			boolean hasFacetRequests = (facetRequest != null) && !facetRequest.getCountRequestList().isEmpty();
+			boolean hasStatRequests = (facetRequest != null) && !facetRequest.getStatRequestList().isEmpty();
 
-				searchWithFacets(facetRequest, query, indexSearcher, collector, shardQueryReponseBuilder);
+			if (hasFacetRequests || hasStatRequests) {
+				FacetsCollector facetsCollector = new FacetsCollector();
+				indexSearcher.search(query, MultiCollector.wrap(collector, facetsCollector));
 
+				if (hasFacetRequests) {
+					List<ZuliaQuery.FacetGroup> facetGroups = handleFacets(facetRequest.getCountRequestList(), facetsCollector);
+					shardQueryReponseBuilder.addAllFacetGroup(facetGroups);
+				}
+				if (hasStatRequests) {
+					List<ZuliaQuery.StatGroup> statGroups = handleStats(facetRequest.getStatRequestList(), facetsCollector);
+					shardQueryReponseBuilder.addAllStatGroup(statGroups);
+				}
 			}
 			else {
 				indexSearcher.search(query, collector);
@@ -262,6 +269,65 @@ public ZuliaQuery.ShardQueryResponse queryShard(Query query, Map<String, ZuliaBa
 
 	}
 
+	private List<ZuliaQuery.StatGroup> handleStats(List<ZuliaQuery.StatRequest> statRequestList, FacetsCollector facetsCollector) throws IOException {
+		List<ZuliaQuery.StatGroup> statGroups = new ArrayList<>();
+
+		TaxonomyStatsHandler facets = new TaxonomyStatsHandler(taxoReader, facetsCollector, statRequestList, indexConfig);
+
+		for (ZuliaQuery.StatRequest statRequest : statRequestList) {
+
+			ZuliaQuery.StatGroup.Builder statGroupBuilder = ZuliaQuery.StatGroup.newBuilder();
+			statGroupBuilder.setStatRequest(statRequest);
+			String label = statRequest.getFacetField().getLabel();
+			if (!label.isEmpty()) {
+
+				int numOfFacets;
+				if (indexConfig.getNumberOfShards() > 1) {
+					if (statRequest.getShardFacets() > 0) {
+						numOfFacets = statRequest.getShardFacets();
+					}
+					else if (statRequest.getShardFacets() == 0) {
+						numOfFacets = statRequest.getMaxFacets() * 10;
+					}
+					else {
+						numOfFacets = getTotalFacets();
+					}
+				}
+				else {
+					if (statRequest.getMaxFacets() > 0) {
+						numOfFacets = statRequest.getMaxFacets();
+					}
+					else {
+						numOfFacets = getTotalFacets();
+					}
+				}
+
+				if (indexConfig.isHierarchicalFacet(label)) {
+					List<ZuliaQuery.FacetStats> topChildren = facets.getTopChildren(statRequest.getNumericField(), numOfFacets, label,
+							statRequest.getFacetField().getPathList().toArray(new String[0]));
+					statGroupBuilder.addAllFacetStats(topChildren);
+				}
+				else {
+					List<ZuliaQuery.FacetStats> topChildren = facets.getTopChildren(statRequest.getNumericField(), numOfFacets, label,
+							statRequest.getFacetField().getPathList().toArray(new String[0]));
+					if (topChildren != null) {
+						statGroupBuilder.addAllFacetStats(topChildren);
+					}
+				}
+
+			}
+			else {
+				ZuliaQuery.FacetStats globalStats = facets.getGlobalStatsForNumericField(statRequest.getNumericField());
+				statGroupBuilder.setGlobalStats(globalStats);
+			}
+
+			statGroups.add(statGroupBuilder.build());
+
+		}
+
+		return statGroups;
+	}
+
 	private List<AnalysisHandler> getAnalysisHandlerList(List<ZuliaQuery.AnalysisRequest> analysisRequests) throws Exception {
 		if (analysisRequests.isEmpty()) {
 			return Collections.emptyList();
@@ -349,14 +415,12 @@ else if (ZuliaBase.Similarity.TF.equals(similarity)) {
 		};
 	}
 
-	private void searchWithFacets(ZuliaQuery.FacetRequest facetRequest, Query q, IndexSearcher indexSearcher, TopDocsCollector<?> collector,
-			ZuliaQuery.ShardQueryResponse.Builder segmentReponseBuilder) throws Exception {
-		FacetsCollector facetsCollector = new FacetsCollector();
-		indexSearcher.search(q, MultiCollector.wrap(collector, facetsCollector));
+	private List<ZuliaQuery.FacetGroup> handleFacets(List<ZuliaQuery.CountRequest> countRequests, FacetsCollector facetsCollector) throws IOException {
+		FastTaxonomyFacetCounts facets = new FastTaxonomyFacetCounts(taxoReader, facetsConfig, facetsCollector);
 
-		Facets facets = getFacets(facetsCollector);
+		List<ZuliaQuery.FacetGroup> facetGroups = new ArrayList<>();
 
-		for (ZuliaQuery.CountRequest countRequest : facetRequest.getCountRequestList()) {
+		for (ZuliaQuery.CountRequest countRequest : countRequests) {
 
 			ZuliaQuery.Facet facetField = countRequest.getFacetField();
 			String label = facetField.getLabel();
@@ -427,8 +491,9 @@ else if (countRequest.getShardFacets() == 0) {
 					fg.addFacetCount(facetCountBuilder);
 				}
 			}
-			segmentReponseBuilder.addFacetGroup(fg);
+			facetGroups.add(fg.build());
 		}
+		return facetGroups;
 	}
 
 	private TopDocsCollector<?> getSortingCollector(ZuliaQuery.SortRequest sortRequest, int hasMoreAmount, FieldDoc after) throws Exception {