sort results of search index by matched count (#653)

implement: #630

Change-Id: I6cab6741f2e0dfcdb731ffc099334508e33ca2e1

hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraFeatures.java

@@ -75,6 +75,11 @@ public boolean supportsQueryByPage() {
         return true;
     }
 
+    @Override
+    public boolean supportsQuerySortByInputIds() {
+        return false;
+    }
+
     @Override
     public boolean supportsDeleteEdgeByLabel() {
         return true;

hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java

@@ -40,6 +40,7 @@
 import com.baidu.hugegraph.backend.store.BackendTable;
 import com.baidu.hugegraph.backend.store.Shard;
 import com.baidu.hugegraph.exception.NotFoundException;
+import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.ExtendableIterator;
 import com.baidu.hugegraph.type.HugeType;
 import com.baidu.hugegraph.type.define.HugeKeys;
@@ -303,7 +304,7 @@ protected Clause relation2Cql(Relation relation) {
             //    return QueryBuilder.like(key, value);
             case NEQ:
             default:
-                throw new AssertionError("Unsupported relation: " + relation);
+                throw new NotSupportException("relation '%s'", relation);
         }
     }
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolder.java

@@ -28,7 +28,7 @@
 import com.baidu.hugegraph.util.InsertionOrderUtil;
 import com.google.common.collect.ImmutableSet;
 
-public final class IdHolder {
+public class IdHolder {
 
     private final ConditionQuery query;
     private final Function<ConditionQuery, PageIds> idsFetcher;
@@ -67,6 +67,7 @@ public IdHolder(ConditionQuery query,
     }
 
     public void merge(Set<Id> ids) {
+        E.checkNotNull(this.ids, "ids");
         this.ids.addAll(ids);
     }
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java

@@ -24,7 +24,7 @@
 
 import com.baidu.hugegraph.util.E;
 
-public final class IdHolderList extends ArrayList<IdHolder> {
+public class IdHolderList extends ArrayList<IdHolder> {
 
     private static final long serialVersionUID = -738694176552424990L;
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java

@@ -24,6 +24,7 @@
 
 import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.backend.store.BackendEntry;
+import com.baidu.hugegraph.backend.tx.AbstractTransaction.QueryResults;
 import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.Metadatable;
 import com.baidu.hugegraph.util.E;
@@ -32,15 +33,19 @@ public class PageEntryIterator implements Iterator<BackendEntry>, Metadatable {
 
     private final QueryList queries;
     private final long pageSize;
-    private QueryList.PageIterator results;
-    private PageInfo pageInfo;
+    private final PageInfo pageInfo;
+    private final QueryResults queryResults; // for upper layer
+
+    private QueryList.PageIterator pageResults;
     private long remaining;
 
     public PageEntryIterator(QueryList queries, long pageSize) {
         this.queries = queries;
         this.pageSize = pageSize;
-        this.results = QueryList.PageIterator.EMPTY;
         this.pageInfo = this.parsePageState();
+        this.queryResults = new QueryResults(this);
+
+        this.pageResults = QueryList.PageIterator.EMPTY;
         this.remaining = queries.parent().limit();
     }
 
@@ -55,7 +60,7 @@ private PageInfo parsePageState() {
 
     @Override
     public boolean hasNext() {
-        if (this.results.iterator().hasNext()) {
+        if (this.pageResults.get().hasNext()) {
             return true;
         }
         return this.fetch();
@@ -71,16 +76,17 @@ private boolean fetch() {
         if (this.remaining != Query.NO_LIMIT && this.remaining < pageSize) {
             pageSize = this.remaining;
         }
-        this.results = this.queries.fetchNext(this.pageInfo, pageSize);
-        assert this.results != null;
+        this.pageResults = this.queries.fetchNext(this.pageInfo, pageSize);
+        assert this.pageResults != null;
+        this.queryResults.setQuery(this.pageResults.query());
 
-        if (this.results.iterator().hasNext()) {
-            if (!this.results.hasNextPage()) {
+        if (this.pageResults.get().hasNext()) {
+            if (!this.pageResults.hasNextPage()) {
                 this.pageInfo.increase();
             } else {
-                this.pageInfo.page(this.results.page());
+                this.pageInfo.page(this.pageResults.page());
             }
-            this.remaining -= this.results.total();
+            this.remaining -= this.pageResults.total();
             return true;
         } else {
             this.pageInfo.increase();
@@ -93,7 +99,7 @@ public BackendEntry next() {
         if (!this.hasNext()) {
             throw new NoSuchElementException();
         }
-        return this.results.iterator().next();
+        return this.pageResults.get().next();
     }
 
     @Override
@@ -106,4 +112,8 @@ public Object metadata(String meta, Object... args) {
         }
         throw new NotSupportException("Invalid meta '%s'", meta);
     }
+
+    public QueryResults results() {
+        return this.queryResults;
+    }
 }

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java

@@ -26,30 +26,29 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.backend.query.IdQuery;
 import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.backend.store.BackendEntry;
+import com.baidu.hugegraph.backend.tx.AbstractTransaction.QueryResults;
 import com.baidu.hugegraph.config.CoreOptions;
-import com.baidu.hugegraph.iterator.FlatMapperIterator;
 import com.baidu.hugegraph.util.Bytes;
 import com.baidu.hugegraph.util.CollectionUtil;
 import com.baidu.hugegraph.util.E;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
 public final class QueryList {
 
     private final HugeGraph graph;
     private final Query parent;
     // The size of each page fetched by the inner page
-    private final Function<Query, Iterator<BackendEntry>> fetcher;
+    private final Function<Query, QueryResults> fetcher;
     private final List<QueryHolder> queries;
 
     public QueryList(HugeGraph graph, Query parent,
-                     Function<Query, Iterator<BackendEntry>> fetcher) {
+                     Function<Query, QueryResults> fetcher) {
         this.graph = graph;
         this.parent = parent;
         this.fetcher = fetcher;
@@ -60,7 +59,7 @@ protected Query parent() {
         return this.parent;
     }
 
-    protected Function<Query, Iterator<BackendEntry>> fetcher() {
+    protected Function<Query, QueryResults> fetcher() {
         return this.fetcher;
     }
 
@@ -93,21 +92,21 @@ public boolean empty() {
         return this.queries.isEmpty();
     }
 
-    public Iterator<BackendEntry> fetch() {
+    public QueryResults fetch() {
         assert !this.queries.isEmpty();
         if (this.parent.paging()) {
             int pageSize = this.graph.configuration()
                                      .get(CoreOptions.QUERY_PAGE_SIZE);
-            return new PageEntryIterator(this, pageSize);
-        } else {
-            return this.fetchAll();
+            PageEntryIterator iterator = new PageEntryIterator(this, pageSize);
+            /*
+             * NOTE: PageEntryIterator query will change every fetch time.
+             * TODO: sort results by input ids in each page.
+             */
+            return iterator.results();
         }
-    }
 
-    protected Iterator<BackendEntry> fetchAll() {
-        return new FlatMapperIterator<>(this.queries.iterator(), q -> {
-            return q.iterator();
-        });
+        // Fetch all results once
+        return QueryResults.flatMap(this.queries.iterator(), q -> q.iterator());
     }
 
     protected PageIterator fetchNext(PageInfo pageInfo, long pageSize) {
@@ -157,7 +156,7 @@ private interface QueryHolder {
          * For non-paging situation
          * @return          BackendEntry iterator
          */
-        public Iterator<BackendEntry> iterator();
+        public QueryResults iterator();
 
         /**
          * For paging situation
@@ -183,7 +182,7 @@ public OptimizedQuery(Query query) {
         }
 
         @Override
-        public Iterator<BackendEntry> iterator() {
+        public QueryResults iterator() {
             return fetcher().apply(this.query);
         }
 
@@ -196,11 +195,11 @@ public PageIterator iterator(int index, String page, long pageSize) {
             if (this.query.limit() == Query.NO_LIMIT) {
                 query.limit(pageSize);
             }
-            Iterator<BackendEntry> iterator = fetcher().apply(query);
+            QueryResults results = fetcher().apply(query);
             // Must iterate all entries before get the next page
-            List<BackendEntry> results = IteratorUtils.list(iterator);
-            return new PageIterator(results.iterator(),
-                                    PageInfo.pageState(iterator));
+            return new PageIterator(results.list().iterator(),
+                                    results.queries(),
+                                    PageInfo.pageState(results.iterator()));
         }
 
         @Override
@@ -222,23 +221,28 @@ public IndexQuery(List<IdHolder> holders) {
         }
 
         @Override
-        public Iterator<BackendEntry> iterator() {
-            return new FlatMapperIterator<>(this.holders.iterator(), holder -> {
-                if (holder.ids().isEmpty()) {
-                    return null;
-                }
-                Set<Id> ids = holder.ids();
-                if (parent().limit() != Query.NO_LIMIT &&
-                    ids.size() > parent().limit()) {
-                    /*
-                     * Avoid too many ids in one time query,
-                     * Assume it will get one result by each id
-                     */
-                    ids = CollectionUtil.subSet(ids, 0, (int) parent().limit());
-                }
-                IdQuery query = new IdQuery(parent(), ids);
-                return fetcher().apply(query);
-            });
+        public QueryResults iterator() {
+            if (this.holders.size() == 1) {
+                return this.each(this.holders.get(0));
+            }
+            return QueryResults.flatMap(this.holders.iterator(), this::each);
+        }
+
+        private QueryResults each(IdHolder holder) {
+            Set<Id> ids = holder.ids();
+            if (ids.isEmpty()) {
+                return null;
+            }
+            if (parent().limit() != Query.NO_LIMIT &&
+                ids.size() > parent().limit()) {
+                /*
+                 * Avoid too many ids in one time query,
+                 * Assume it will get one result by each id
+                 */
+                ids = CollectionUtil.subSet(ids, 0, (int) parent().limit());
+            }
+            IdQuery query = new IdQuery(parent(), ids);
+            return fetcher().apply(query);
         }
 
         @Override
@@ -249,7 +253,8 @@ public PageIterator iterator(int index, String page, long pageSize) {
                 return PageIterator.EMPTY;
             }
             IdQuery query = new IdQuery(parent(), pageIds.ids());
-            return new PageIterator(fetcher().apply(query),
+            QueryResults results = fetcher().apply(query);
+            return new PageIterator(results.iterator(), results.queries(),
                                     pageIds.pageState());
         }
 
@@ -263,18 +268,21 @@ public static class PageIterator {
 
         public static final PageIterator EMPTY = new PageIterator(
                                                  Collections.emptyIterator(),
+                                                 ImmutableList.of(Query.NONE),
                                                  PageState.EMPTY);
 
         private final Iterator<BackendEntry> iterator;
+        private final List<Query> queries;
         private final PageState pageState;
 
         public PageIterator(Iterator<BackendEntry> iterator,
-                            PageState pageState) {
+                            List<Query> queries, PageState pageState) {
             this.iterator = iterator;
+            this.queries = queries;
             this.pageState = pageState;
         }
 
-        public Iterator<BackendEntry> iterator() {
+        public Iterator<BackendEntry> get() {
             return this.iterator;
         }
 
@@ -283,6 +291,12 @@ public boolean hasNextPage() {
                                  PageState.EMPTY_BYTES);
         }
 
+        public Query query() {
+            E.checkState(this.queries.size() == 1,
+                         "Expect queries size 1, but got: %s", this.queries);
+            return this.queries.get(0);
+        }
+
         public String page() {
             return this.pageState.toString();
         }