Skip to content

Commit

Permalink
fix all combinations of numeric stat and stat faceting to work
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdavis95
mdavis95 committed Apr 17, 2022
1 parent c4cb247 commit 2ee5a6a
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 67 deletions.
59 changes: 36 additions & 23 deletions zulia-server/src/main/java/io/zulia/server/search/TaxonomyStatsHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ private void sumValues(List<MatchingDocs> matchingDocs, List<String> fieldsList)

// temp storage of the doc values for a document
// because we iterate through the values twice (and the iterator is not resetable) and we do not want to have to create a new array each time
int[] documentValuesBuffer = new int[0];
int[] ordinalDocValues = new int[0];
long[] numericValues = new long[0];

for (MatchingDocs hits : matchingDocs) {

Expand All @@ -120,22 +121,38 @@ private void sumValues(List<MatchingDocs> matchingDocs, List<String> fieldsList)

for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = docs.nextDoc()) {

int ordinalCount = -1;
if (ordinalValues != null) {

ordinalCount = ordinalValues.docValueCount();

if (ordinalCount > ordinalDocValues.length) {
ordinalDocValues = new int[ordinalCount];
}

for (int i = 0; i < ordinalCount; i++) {
int ordIndex = (int) ordinalValues.nextValue();
ordinalDocValues[i] = ordIndex;
}
}

for (int f = 0; f < fieldsList.size(); f++) {

SortedNumericDocValues functionValue = functionValues[f];
ZuliaIndex.FieldConfig.FieldType fieldType = fieldTypes.get(f);
if (functionValue.advanceExact(doc)) {
if (ordinalValues != null) {

int ordinalCount = ordinalValues.docValueCount();
if ( functionValue.docValueCount() > numericValues.length) {
numericValues = new long[functionValue.docValueCount()];
}
for (int j = 0; j < functionValue.docValueCount(); j++) {
long value = functionValue.nextValue();
numericValues[j] = value;
}

if (ordinalCount > documentValuesBuffer.length) {
documentValuesBuffer = new int[ordinalCount];
}
if (ordinalCount != -1) {

for (int i = 0; i < ordinalCount; i++) {
int ordIndex = (int) ordinalValues.nextValue();
documentValuesBuffer[i] = ordIndex;
int ordIndex = ordinalDocValues[i];
Stats stats = fieldFacetStats[f][ordIndex];
if (stats == null) {
stats = new Stats(FieldTypeUtil.isNumericFloatingPointFieldType(fieldType));
Expand All @@ -144,23 +161,26 @@ private void sumValues(List<MatchingDocs> matchingDocs, List<String> fieldsList)
stats.newDoc(true);
}
for (int j = 0; j < functionValue.docValueCount(); j++) {
long value = functionValue.nextValue();
for (int i = 0; i < ordinalCount; i++) {
int ordIndex = documentValuesBuffer[i];
int ordIndex = ordinalDocValues[i];
Stats stats = fieldFacetStats[f][ordIndex];
addUpValue(fieldType, value, stats);
addUpValue(fieldType, numericValues[j], stats);
}

}
}
if (fieldStats != null) {
docValuesForDocument(functionValue, fieldType, fieldStats[f]);
Stats stats = fieldStats[f];
stats.newDoc(true);
for (int j = 0; j < functionValue.docValueCount(); j++) {
addUpValue(fieldType, numericValues[j], stats);
}
}
}
else {
if (ordinalValues != null) {
int ordinalCount = ordinalValues.docValueCount();
if (ordinalCount != -1) {
for (int i = 0; i < ordinalCount; i++) {
int ordIndex = (int) ordinalValues.nextValue();
int ordIndex = ordinalDocValues[i];
Stats stats = fieldFacetStats[f][ordIndex];
if (stats == null) {
stats = new Stats(FieldTypeUtil.isNumericFloatingPointFieldType(fieldType));
Expand Down Expand Up @@ -197,13 +217,6 @@ else if (FieldTypeUtil.isBooleanFieldType(fieldType)) {
}
}

private void docValuesForDocument(SortedNumericDocValues functionValue, ZuliaIndex.FieldConfig.FieldType fieldType, Stats stats) throws IOException {
stats.newDoc(true);
for (int j = 0; j < functionValue.docValueCount(); j++) {
long value = functionValue.nextValue();
addUpValue(fieldType, value, stats);
}
}

public ZuliaQuery.FacetStats getGlobalStatsForNumericField(String field) {
int fieldIndex = fieldsList.indexOf(field);
Expand Down
136 changes: 92 additions & 44 deletions zulia-server/src/test/java/io/zulia/server/test/node/StatTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,19 +106,45 @@ public void statTest() throws Exception {
System.out.println("Hits: " + searchResult.getTotalHits());

FacetStats ratingStat = searchResult.getNumericFieldStat("rating");
ratingTest(ratingStat);

search.clearStat();
search.addStat(new StatFacet("rating", "normalFacet"));
searchResult = zuliaWorkPool.search(search);
ratingNormalTest(searchResult);

search.clearStat();
search.addStat(new StatFacet("rating", "pathFacet"));
searchResult = zuliaWorkPool.search(search);

ratingPathTest(searchResult);

search.clearStat();
search.addStat(new StatFacet("rating", "normalFacet"));
search.addStat(new StatFacet("rating", "pathFacet"));
searchResult = zuliaWorkPool.search(search);
ratingNormalTest(searchResult);
ratingPathTest(searchResult);

search = new Search(STAT_TEST_INDEX);
search.addStat(new StatFacet("authorCount", "pathFacet"));
Search finalSearch = search;
Assertions.assertThrows(Exception.class, () -> zuliaWorkPool.search(finalSearch),
"Expecting: Search: Numeric field <authorCount> must be indexed as a SORTABLE numeric field");

}

private void ratingTest(FacetStats ratingStat) {
Assertions.assertEquals(0.5, ratingStat.getMin().getDoubleValue(), 0.001);
Assertions.assertEquals(3.5, ratingStat.getMax().getDoubleValue(), 0.001);

Assertions.assertEquals(10.5 * repeatCount, ratingStat.getSum().getDoubleValue(), 0.001);
Assertions.assertEquals(4L * repeatCount, ratingStat.getDocCount());
Assertions.assertEquals(6L * repeatCount, ratingStat.getAllDocCount());
Assertions.assertEquals(5L * repeatCount, ratingStat.getValueCount());
}

search.clearStat();
search.addStat(new StatFacet("rating", "normalFacet"));
searchResult = zuliaWorkPool.search(search);

private void ratingNormalTest(SearchResult searchResult) {
List<FacetStats> ratingByFacet = searchResult.getFacetFieldStat("rating", "normalFacet");

for (FacetStats facetStats : ratingByFacet) {
Expand All @@ -142,11 +168,9 @@ else if (facetStats.getFacet().equals("bar")) {
throw new AssertionFailedError("Unexpect facet <" + facetStats.getFacet() + ">");
}
}
}

search.clearStat();
search.addStat(new StatFacet("rating", "pathFacet"));
searchResult = zuliaWorkPool.search(search);

private void ratingPathTest(SearchResult searchResult) {
List<FacetStats> ratingByPathFacet = searchResult.getFacetFieldStat("rating", "pathFacet");

for (FacetStats facetStats : ratingByPathFacet) {
Expand Down Expand Up @@ -177,13 +201,6 @@ else if (facetStats.getFacet().equals("top3")) {
throw new AssertionFailedError("Unexpect facet <" + facetStats.getFacet() + ">");
}
}

search = new Search(STAT_TEST_INDEX);
search.addStat(new StatFacet("authorCount", "pathFacet"));
Search finalSearch = search;
Assertions.assertThrows(Exception.class, () -> zuliaWorkPool.search(finalSearch),
"Expecting: Search: Numeric field <authorCount> must be indexed as a SORTABLE numeric field");

}

@Test
Expand Down Expand Up @@ -226,35 +243,67 @@ public void confirm() throws Exception {
search.addStat(new NumericStat("authorCount"));

SearchResult searchResult = zuliaWorkPool.search(search);
authorCountTest(searchResult);

FacetStats authorCountStats = searchResult.getNumericFieldStat("authorCount");
search.clearStat();
search.addStat(new StatFacet("authorCount", "normalFacet"));
searchResult = zuliaWorkPool.search(search);

Assertions.assertEquals(2, authorCountStats.getMin().getLongValue());
Assertions.assertEquals(5, authorCountStats.getMax().getLongValue());
authorCountNormalFacetTest(searchResult);

search.clearStat();
search.addStat(new StatFacet("authorCount", "pathFacet"));
searchResult = zuliaWorkPool.search(search);
authorCountPathFacetTest(searchResult);

Assertions.assertEquals(20L * repeatCount, authorCountStats.getSum().getLongValue());
Assertions.assertEquals(6L * repeatCount, authorCountStats.getDocCount());
Assertions.assertEquals(6L * repeatCount, authorCountStats.getValueCount());

search.clearStat();
search.addStat(new NumericStat("authorCount"));
search.addStat(new StatFacet("authorCount", "normalFacet"));
search.addStat(new StatFacet("authorCount", "pathFacet"));
searchResult = zuliaWorkPool.search(search);
authorCountTest(searchResult);
authorCountNormalFacetTest(searchResult);
authorCountPathFacetTest(searchResult);

List<FacetStats> authorCountByNormalFacet = searchResult.getFacetFieldStat("authorCount", "normalFacet");

for (FacetStats facetStats : authorCountByNormalFacet) {
if (facetStats.getFacet().equals("foo")) {
search.clearStat();
search.addStat(new StatFacet("authorCount", "pathFacet"));
search.addStat(new NumericStat("authorCount"));
search.addStat(new StatFacet("authorCount", "normalFacet"));
search.addStat(new StatFacet("rating", "normalFacet"));
search.addStat(new StatFacet("rating", "pathFacet"));
searchResult = zuliaWorkPool.search(search);
authorCountTest(searchResult);
authorCountNormalFacetTest(searchResult);
authorCountPathFacetTest(searchResult);
ratingNormalTest(searchResult);
ratingPathTest(searchResult);
}

private void authorCountPathFacetTest(SearchResult searchResult) {
List<FacetStats> authorCountPathFacet = searchResult.getFacetFieldStat("authorCount", "pathFacet");
for (FacetStats facetStats : authorCountPathFacet) {
if (facetStats.getFacet().equals("top1")) {
Assertions.assertEquals(2L, facetStats.getMin().getLongValue());
Assertions.assertEquals(4L, facetStats.getMax().getLongValue());
Assertions.assertEquals(9L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(3L * repeatCount, facetStats.getDocCount());
Assertions.assertEquals(3L * repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(3L * repeatCount, facetStats.getValueCount());
}
else if (facetStats.getFacet().equals("bar")) {
else if (facetStats.getFacet().equals("top2")) {
Assertions.assertEquals(2L, facetStats.getMin().getLongValue());
Assertions.assertEquals(2L, facetStats.getMax().getLongValue());
Assertions.assertEquals(2L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(repeatCount, facetStats.getDocCount());
Assertions.assertEquals(repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(repeatCount, facetStats.getValueCount());
}
else if (facetStats.getFacet().equals("top3")) {
Assertions.assertEquals(4L, facetStats.getMin().getLongValue());
Assertions.assertEquals(5L, facetStats.getMax().getLongValue());
Assertions.assertEquals(7L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(9L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(2L * repeatCount, facetStats.getDocCount());
Assertions.assertEquals(2L * repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(2L * repeatCount, facetStats.getValueCount());
Expand All @@ -263,34 +312,23 @@ else if (facetStats.getFacet().equals("bar")) {
throw new AssertionFailedError("Unexpected facet <" + facetStats.getFacet() + ">");
}
}
}

search.clearStat();
search.addStat(new StatFacet("authorCount", "pathFacet"));
searchResult = zuliaWorkPool.search(search);

List<FacetStats> ratingByPathFacet = searchResult.getFacetFieldStat("authorCount", "pathFacet");

for (FacetStats facetStats : ratingByPathFacet) {
if (facetStats.getFacet().equals("top1")) {
private void authorCountNormalFacetTest(SearchResult searchResult) {
List<FacetStats> authorCountByNormalFacet = searchResult.getFacetFieldStat("authorCount", "normalFacet");
for (FacetStats facetStats : authorCountByNormalFacet) {
if (facetStats.getFacet().equals("foo")) {
Assertions.assertEquals(2L, facetStats.getMin().getLongValue());
Assertions.assertEquals(4L, facetStats.getMax().getLongValue());
Assertions.assertEquals(9L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(3L * repeatCount, facetStats.getDocCount());
Assertions.assertEquals(3L * repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(3L * repeatCount, facetStats.getValueCount());
}
else if (facetStats.getFacet().equals("top2")) {
else if (facetStats.getFacet().equals("bar")) {
Assertions.assertEquals(2L, facetStats.getMin().getLongValue());
Assertions.assertEquals(2L, facetStats.getMax().getLongValue());
Assertions.assertEquals(2L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(repeatCount, facetStats.getDocCount());
Assertions.assertEquals(repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(repeatCount, facetStats.getValueCount());
}
else if (facetStats.getFacet().equals("top3")) {
Assertions.assertEquals(4L, facetStats.getMin().getLongValue());
Assertions.assertEquals(5L, facetStats.getMax().getLongValue());
Assertions.assertEquals(9L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(7L * repeatCount, facetStats.getSum().getLongValue());
Assertions.assertEquals(2L * repeatCount, facetStats.getDocCount());
Assertions.assertEquals(2L * repeatCount, facetStats.getAllDocCount());
Assertions.assertEquals(2L * repeatCount, facetStats.getValueCount());
Expand All @@ -301,6 +339,16 @@ else if (facetStats.getFacet().equals("top3")) {
}
}

private void authorCountTest(SearchResult searchResult) {
FacetStats authorCountStats = searchResult.getNumericFieldStat("authorCount");
Assertions.assertEquals(2, authorCountStats.getMin().getLongValue());
Assertions.assertEquals(5, authorCountStats.getMax().getLongValue());

Assertions.assertEquals(20L * repeatCount, authorCountStats.getSum().getLongValue());
Assertions.assertEquals(6L * repeatCount, authorCountStats.getDocCount());
Assertions.assertEquals(6L * repeatCount, authorCountStats.getValueCount());
}

@Test
@Order(7)
public void shutdown() throws Exception {
Expand Down

0 comments on commit 2ee5a6a

Please sign in to comment.