Skip to content

Commit

Permalink
feat(index): support remove left range index
Browse files Browse the repository at this point in the history
  • Loading branch information
jadepeng committed Jun 30, 2021
1 parent 0d32e16 commit 1a503fe
Show file tree
Hide file tree
Showing 6 changed files with 356 additions and 43 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package com.baidu.hugegraph.backend.query;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
Expand All @@ -37,12 +38,14 @@
import com.baidu.hugegraph.backend.query.Condition.RelationType;
import com.baidu.hugegraph.perf.PerfUtil.Watched;
import com.baidu.hugegraph.structure.HugeElement;
import com.baidu.hugegraph.structure.HugeProperty;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.HugeKeys;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.LongEncoding;
import com.baidu.hugegraph.util.NumericUtil;
import com.google.common.base.Function;
import com.google.common.collect.Sets;

public final class ConditionQuery extends IdQuery {

Expand All @@ -51,6 +54,7 @@ public final class ConditionQuery extends IdQuery {

private OptimizedType optimizedType = OptimizedType.NONE;
private Function<HugeElement, Boolean> resultsFilter = null;
private Element2IndexValueMap element2IndexValueMap = null;

public ConditionQuery(HugeType resultType) {
super(resultType);
Expand Down Expand Up @@ -134,6 +138,21 @@ public void resetConditions() {
this.conditions = new LinkedHashSet<>();
}

public void recordIndexValue(Id propertyId, Id id, Object indexValue) {
if (this.element2IndexValueMap == null) {
this.element2IndexValueMap = new Element2IndexValueMap(propertyId);
}
assert this.element2IndexValueMap.propertyId.equals(propertyId);
this.element2IndexValueMap.addIndexValue(id, indexValue);
}

public LeftIndex getLeftIndex(Id elementId) {
if (this.element2IndexValueMap == null) {
return null;
}
return this.element2IndexValueMap.getLeftIndex(elementId);
}

public List<Condition.Relation> relations() {
List<Condition.Relation> relations = new ArrayList<>();
for (Condition c : this.conditions) {
Expand Down Expand Up @@ -389,7 +408,6 @@ public boolean hasNeqCondition() {
return false;
}


public boolean matchUserpropKeys(List<Id> keys) {
Set<Id> conditionKeys = this.userpropKeys();
return keys.size() > 0 && conditionKeys.containsAll(keys);
Expand Down Expand Up @@ -429,6 +447,11 @@ public boolean test(HugeElement element) {
if (!cond.test(element)) {
return false;
}

if (this.element2IndexValueMap != null &&
!this.element2IndexValueMap.validRangeIndex(element, cond)) {
return false;
}
}
return true;
}
Expand Down Expand Up @@ -522,4 +545,115 @@ public enum OptimizedType {
INDEX,
INDEX_FILTER
}

public static final class Element2IndexValueMap {

private final Id propertyId;
private final Map<Id, LeftIndex> leftIndexMap;
private final Map<Id, Set<Object>> element2IndexValues;

public Element2IndexValueMap(Id propertyId) {
this.propertyId = propertyId;
this.element2IndexValues = new HashMap<>();
this.leftIndexMap = new HashMap<>();
}

public void addIndexValue(Id id, Object indexValue) {
if (this.element2IndexValues.containsKey(id)) {
this.element2IndexValues.get(id).add(indexValue);
} else {
this.element2IndexValues.put(id, Sets.newHashSet(indexValue));
}
}

public Set<Object> getIndexValues(Id elementId){
return this.element2IndexValues.get(elementId);
}

public void addLeftIndex(Id indexField, Set<Object> indexValues,
Id elementId) {
this.leftIndexMap.put(elementId, new LeftIndex(indexValues, indexField));
}

public LeftIndex getLeftIndex(Id elementId) {
return this.leftIndexMap.get(elementId);
}

public boolean validRangeIndex(HugeElement element, Condition cond) {
// Not UserpropRelation
if (!(cond instanceof Condition.UserpropRelation)) {
return true;
}

Condition.UserpropRelation propRelation =
(Condition.UserpropRelation) cond;
Id propId = propRelation.key();
Set<Object> fieldValues = this.getIndexValues(element.id());
if (fieldValues == null) {
// Not range index
return true;
}

HugeProperty<Object> hugeProperty = element.getProperty(propId);
if (hugeProperty == null) {
// Property value has been deleted
this.addLeftIndex(propId, fieldValues, element.id());
return false;
}

/*
* NOTE: If success remove means has correct index,
* we should add left index values to left index map
* waiting to be removed
*/
boolean hasRightValue = removeValue(fieldValues, hugeProperty.value());
if (fieldValues.size() > 0) {
this.addLeftIndex(propId, fieldValues, element.id());
}
return hasRightValue;
}

private static boolean removeValue(Set<Object> values, Object value){
for (Object compareValue : values) {
if (numberEquals(compareValue, value)) {
values.remove(compareValue);
return true;
}
}
return false;
}

private static boolean numberEquals(Object number1, Object number2) {
// Same class compare directly
if (number1.getClass().equals(number2.getClass())) {
return number1.equals(number2);
}

// Otherwise convert to BigDecimal to make two numbers comparable
Number n1 = NumericUtil.convertToNumber(number1);
Number n2 = NumericUtil.convertToNumber(number2);
BigDecimal b1 = new BigDecimal(n1.doubleValue());
BigDecimal b2 = new BigDecimal(n2.doubleValue());
return b1.compareTo(b2) == 0;
}
}

public static final class LeftIndex {

private final Set<Object> indexFieldValues;
private final Id indexField;

public LeftIndex(Set<Object> indexFieldValues, Id indexField) {
this.indexFieldValues = indexFieldValues;
this.indexField = indexField;
}

public Set<Object> indexFieldValues() {
return indexFieldValues;
}

public Id indexField() {
return indexField;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public GraphIndexTransaction(HugeGraphParams graph, BackendStore store) {

protected Id asyncRemoveIndexLeft(ConditionQuery query,
HugeElement element) {
LOG.info("Remove left index: {}, query: {}", element, query);
RemoveLeftIndexJob job = new RemoveLeftIndexJob(query, element);
HugeTask<?> task = EphemeralJobBuilder.of(this.graph())
.name(element.id().asString())
Expand Down Expand Up @@ -613,6 +614,7 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel,
this.removeExpiredIndexIfNeeded(index, query.showExpired());
ids.addAll(index.elementIds());
Query.checkForceCapacity(ids.size());
this.recordIndexValue(query, index, indexLabel);
}
return ids;
} finally {
Expand All @@ -621,6 +623,21 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel,
});
}

private void recordIndexValue(ConditionQuery query, HugeIndex index,
IndexLabel indexLabel) {
if (!shouldRecordIndexValue(query, index)) {
return;
}

ConditionQuery originQuery =
((ConditionQuery) query.originQuery());
Id fieldId = index.indexLabel().indexField();
for (Id id : index.elementIds()) {
Object value = indexLabel.validValue(index.fieldValues());
originQuery.recordIndexValue(fieldId, id, value);
}
}

@Watched(prefix = "index")
private PageIds doIndexQueryOnce(IndexLabel indexLabel,
ConditionQuery query) {
Expand All @@ -642,6 +659,7 @@ private PageIds doIndexQueryOnce(IndexLabel indexLabel,
break;
}
Query.checkForceCapacity(ids.size());
this.recordIndexValue(query, index, indexLabel);
}
// If there is no data, the entries is not a Metadatable object
if (ids.isEmpty()) {
Expand Down Expand Up @@ -1037,6 +1055,13 @@ private static <T> boolean cmn(List<T> all, int m, int n,
return false;
}

private static boolean shouldRecordIndexValue(ConditionQuery query,
HugeIndex index) {
// Currently, only range index has problems
return query.originQuery() instanceof ConditionQuery &&
index.indexLabel().indexType().isRange();
}

private static IndexQueries constructJointSecondaryQueries(
ConditionQuery query,
List<IndexLabel> ils) {
Expand Down Expand Up @@ -1538,13 +1563,15 @@ public static class RemoveLeftIndexJob extends EphemeralJob<Object> {
private final ConditionQuery query;
private final HugeElement element;
private GraphIndexTransaction tx;
private ConditionQuery.LeftIndex leftIndex;

private RemoveLeftIndexJob(ConditionQuery query, HugeElement element) {
E.checkArgumentNotNull(query, "query");
E.checkArgumentNotNull(element, "element");
this.query = query;
this.element = element;
this.tx = null;
this.leftIndex = query.getLeftIndex(element.id());
}

@Override
Expand Down Expand Up @@ -1598,55 +1625,42 @@ protected long removeIndexLeft(ConditionQuery query,

private long processRangeIndexLeft(ConditionQuery query,
HugeElement element) {
GraphIndexTransaction tx = this.tx;
AbstractSerializer serializer = tx.serializer;
long count = 0;
// Construct index ConditionQuery
Set<MatchedIndex> matchedIndexes = tx.collectMatchedIndexes(query);
IndexQueries queries = null;
Id elementLabelId = element.schemaLabel().id();
for (MatchedIndex index : matchedIndexes) {
if (index.schemaLabel().id().equals(elementLabelId)) {
queries = index.constructIndexQueries(query);
break;
}
if (this.leftIndex == null) {
return count;
}

E.checkState(queries != null,
"Can't construct left-index query for '%s'", query);
Set<Object> indexValues = leftIndex.indexFieldValues();
IndexLabel indexLabel = findMatchedIndexLabel(query);
assert indexLabel != null;

for (ConditionQuery q : queries.values()) {
if (!q.resultType().isRangeIndex()) {
continue;
}
// Query and delete index equals element id
Iterator<BackendEntry> it = tx.query(q).iterator();
try {
while (it.hasNext()) {
HugeIndex index = serializer.readIndex(graph(), q,
it.next());
tx.removeExpiredIndexIfNeeded(index, q.showExpired());
if (index.elementIds().contains(element.id())) {
index.resetElementIds();
index.elementIds(element.id());
tx.doEliminate(serializer.writeIndex(index));
tx.commit();
// If deleted by error, re-add deleted index again
if (this.deletedByError(query, element)) {
tx.doAppend(serializer.writeIndex(index));
tx.commit();
} else {
count++;
}
}
}
} finally {
CloseableIterator.closeIterator(it);
}
AbstractSerializer serializer = tx.serializer;
for (Object value : indexValues) {
HugeIndex index = new HugeIndex(this.graph(), indexLabel);
index.elementIds(element.id());
index.fieldValues(value);
tx.doEliminate(serializer.writeIndex(index));
count++;
}
// commit by batch
tx.commit();
return count;
}

private IndexLabel findMatchedIndexLabel(ConditionQuery query) {
IndexLabel indexLabel = null;
Set<MatchedIndex> matchedIndexes = tx.collectMatchedIndexes(query);
for (MatchedIndex index : matchedIndexes) {
for (IndexLabel label : index.indexLabels()){
if (label.indexField().equals(leftIndex.indexField())){
indexLabel = label;
break;
}
}
}
return indexLabel;
}

private long processSecondaryOrSearchIndexLeft(ConditionQuery query,
HugeElement element) {
Map<PropertyKey, Object> incorrectPKs = InsertionOrderUtil.newMap();
Expand Down
Loading

0 comments on commit 1a503fe

Please sign in to comment.