Skip to content

Commit

Permalink
[GAIA-IR] create ffi object from java side to avoid invoking ffi inte…
Browse files Browse the repository at this point in the history 
…rface which can lead memory leak problems (#1925)

* [IR Compiler] create ffi object from java side

* [IR Core] remove ffi object creation from native side

* [IR Compiler] update unit tests to fix compile error
  • Loading branch information
@shirly121
shirly121 authored Aug 4, 2022
1 parent 9284f69 commit a0da2a9
Show file tree
Hide file tree
Showing 36 changed files with 237 additions and 341 deletions.
4 changes: 2 additions & 2 deletions ...ctive_engine/executor/ir/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public Pointer apply(InterOpBase baseOp) {
for (int i = 0; i < ffiIds.size(); ++i) {
FfiResult e2 =
irCoreLib.orEquivPredicate(
idsPredicate, irCoreLib.asIdKey(), ffiIds.get(i));
idsPredicate, ArgUtils.asKey(ArgUtils.ID), ffiIds.get(i));
if (e2.code != ResultCode.Success) {
throw new InterOpIllegalArgException(
baseOp.getClass(), "ids", "orEquivPredicate returns " + e2.msg);
Expand Down Expand Up @@ -509,7 +509,7 @@ public Pointer apply(InterOpBase baseOp) {
AS_NONE_OP {
public Pointer apply(InterOpBase baseOp) {
Pointer asPtr = irCoreLib.initAsOperator();
FfiResult error = irCoreLib.setAsAlias(asPtr, ArgUtils.asFfiNoneAlias());
FfiResult error = irCoreLib.setAsAlias(asPtr, ArgUtils.asNoneAlias());
if (error != null && error.code != ResultCode.Success) {
throw new AppendInterOpException(baseOp.getClass(), error.msg);
}
Expand Down
2 changes: 1 addition & 1 deletion ...ecutor/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgAggFn.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public ArgAggFn(FfiAggOpt aggregate, FfiAlias.ByValue alias) {
this.aggregate = aggregate;
this.alias = alias;
// set to none by default
this.var = ArgUtils.asFfiNoneVar();
this.var = ArgUtils.asNoneVar();
}

public FfiVariable.ByValue getVar() {
Expand Down
106 changes: 63 additions & 43 deletions ...ecutor/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,75 +17,95 @@
package com.alibaba.graphscope.common.intermediate;

import com.alibaba.graphscope.common.exception.OpArgIllegalException;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
import com.alibaba.graphscope.common.jna.type.*;

import org.apache.commons.lang3.StringUtils;

public class ArgUtils {
private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE;
private static String LABEL = "~label";
private static String ID = "~id";
private static String LEN = "~len";
public static String LABEL = "~label";
public static String ID = "~id";
public static String LEN = "~len";
public static String PROPERTY_ALL = "~all";

public static FfiConst.ByValue asFfiConst(int id) {
return irCoreLib.int32AsConst(id);
public static FfiConst.ByValue asConst(int id) {
FfiConst.ByValue ffiConst = new FfiConst.ByValue();
ffiConst.dataType = FfiDataType.I32;
ffiConst.int32 = id;
return ffiConst;
}

public static FfiConst.ByValue asConst(long id) {
FfiConst.ByValue ffiConst = new FfiConst.ByValue();
ffiConst.dataType = FfiDataType.I64;
ffiConst.int64 = id;
return ffiConst;
}

public static FfiConst.ByValue asFfiConst(long id) {
return irCoreLib.int64AsConst(id);
public static FfiConst.ByValue asConst(String str) {
FfiConst.ByValue ffiConst = new FfiConst.ByValue();
ffiConst.dataType = FfiDataType.Str;
ffiConst.cstr = str;
return ffiConst;
}

// "" indicates NONE
public static FfiProperty.ByValue asFfiProperty(String property) {
if (property.isEmpty()) {
return irCoreLib.asNoneKey();
} else if (property.equals(LABEL)) {
return irCoreLib.asLabelKey();
} else if (property.equals(ID)) {
return irCoreLib.asIdKey();
} else if (property.equals(LEN)) {
return irCoreLib.asLenKey();
} else {
return irCoreLib.asPropertyKey(irCoreLib.cstrAsNameOrId(property));
// "" or null indicates NONE or HEAD
public static FfiNameOrId.ByValue asNameOrId(String tag) {
FfiNameOrId.ByValue ffiName = new FfiNameOrId.ByValue();
if (!StringUtils.isEmpty(tag)) {
ffiName.name = tag;
ffiName.opt = FfiNameIdOpt.Name;
}
return ffiName;
}

public static FfiNameOrId.ByValue asNoneNameOrId() {
return new FfiNameOrId.ByValue();
}

// "" indicates NONE or HEAD
public static FfiNameOrId.ByValue asFfiTag(String tag) {
if (tag.isEmpty()) {
return irCoreLib.noneNameOrId();
} else {
return irCoreLib.cstrAsNameOrId(tag);
// "" or null indicates NONE
public static FfiProperty.ByValue asKey(String property) {
FfiProperty.ByValue ffiProperty = new FfiProperty.ByValue();
if (!StringUtils.isEmpty(property)) {
if (property.equals(LABEL)) {
ffiProperty.opt = FfiPropertyOpt.Label;
} else if (property.equals(ID)) {
ffiProperty.opt = FfiPropertyOpt.Id;
} else if (property.equals(LEN)) {
ffiProperty.opt = FfiPropertyOpt.Len;
} else {
ffiProperty.opt = FfiPropertyOpt.Key;
ffiProperty.key = asNameOrId(property);
}
}
return ffiProperty;
}

public static FfiNameOrId.ByValue asFfiNoneTag() {
return irCoreLib.noneNameOrId();
public static FfiProperty.ByValue asNoneKey() {
return new FfiProperty.ByValue();
}

public static FfiVariable.ByValue asFfiVar(String tag, String property) {
FfiNameOrId.ByValue ffiTag = asFfiTag(tag);
FfiProperty.ByValue ffiProperty = asFfiProperty(property);
return irCoreLib.asVar(ffiTag, ffiProperty);
public static FfiVariable.ByValue asVar(String tag, String property) {
FfiVariable.ByValue ffiVar = new FfiVariable.ByValue();
ffiVar.tag = asNameOrId(tag);
ffiVar.property = asKey(property);
return ffiVar;
}

public static FfiVariable.ByValue asFfiNoneVar() {
return irCoreLib.asNoneVar();
public static FfiVariable.ByValue asNoneVar() {
return new FfiVariable.ByValue();
}

public static FfiAlias.ByValue asFfiNoneAlias() {
FfiNameOrId.ByValue alias = irCoreLib.noneNameOrId();
public static FfiAlias.ByValue asAlias(String aliasName, boolean isQueryGiven) {
FfiAlias.ByValue ffiAlias = new FfiAlias.ByValue();
ffiAlias.alias = alias;
ffiAlias.isQueryGiven = false;
ffiAlias.alias = asNameOrId(aliasName);
ffiAlias.isQueryGiven = isQueryGiven;
return ffiAlias;
}

public static FfiAlias.ByValue asFfiAlias(String aliasName, boolean isQueryGiven) {
FfiNameOrId.ByValue alias = irCoreLib.cstrAsNameOrId(aliasName);
public static FfiAlias.ByValue asNoneAlias() {
FfiAlias.ByValue ffiAlias = new FfiAlias.ByValue();
ffiAlias.alias = alias;
ffiAlias.isQueryGiven = isQueryGiven;
ffiAlias.alias = asNoneNameOrId();
ffiAlias.isQueryGiven = false;
return ffiAlias;
}

Expand Down
4 changes: 2 additions & 2 deletions ...r/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/MatchSentence.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ public class MatchSentence {

public MatchSentence(
String startTag, String endTag, InterOpCollection binders, FfiJoinKind joinKind) {
this.startTag = ArgUtils.asFfiAlias(startTag, true);
this.endTag = ArgUtils.asFfiAlias(endTag, true);
this.startTag = ArgUtils.asAlias(startTag, true);
this.endTag = ArgUtils.asAlias(endTag, true);
this.joinKind = joinKind;
this.binders = binders;
}
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkOutputProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ private SinkArg getSinkArg(InterOpCollection opCollection) {
} else if (cur instanceof ExpandOp
|| cur instanceof ScanFusionOp
|| cur instanceof GetVOp) {
sinkArg.addColumnName(ArgUtils.asFfiNoneTag());
sinkArg.addColumnName(ArgUtils.asNoneNameOrId());
break;
} else if (cur instanceof ProjectOp) {
ProjectOp op = (ProjectOp) cur;
Expand Down
33 changes: 0 additions & 33 deletions ...e/executor/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,39 +180,6 @@ FfiResult.ByValue appendPathxpdOperator(
FfiResult.ByValue appendPatternOperator(
Pointer plan, Pointer pattern, int parent, IntByReference oprIdx);

FfiNameOrId.ByValue noneNameOrId();

default FfiNameOrId.ByValue cstrAsNameOrId(String name) {
FfiNameOrId.ByValue ffiName = new FfiNameOrId.ByValue();
ffiName.opt = FfiNameIdOpt.Name;
ffiName.name = name;
return ffiName;
}

FfiConst.ByValue cstrAsConst(String value);

FfiConst.ByValue int32AsConst(int value);

FfiConst.ByValue int64AsConst(long value);

FfiProperty.ByValue asNoneKey();

FfiProperty.ByValue asLabelKey();

FfiProperty.ByValue asIdKey();

FfiProperty.ByValue asLenKey();

FfiProperty.ByValue asPropertyKey(FfiNameOrId.ByValue key);

FfiVariable.ByValue asVarTagOnly(FfiNameOrId.ByValue tag);

FfiVariable.ByValue asVarPropertyOnly(FfiProperty.ByValue property);

FfiVariable.ByValue asVar(FfiNameOrId.ByValue tag, FfiProperty.ByValue property);

FfiVariable.ByValue asNoneVar();

void destroyFfiData(FfiData.ByValue value);

FfiResult.ByValue setSchema(String schemaJson);
Expand Down
2 changes: 1 addition & 1 deletion ...xecutor/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameOrId.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ private FfiNameOrId() {

public static class ByValue extends FfiNameOrId implements Structure.ByValue {}

public FfiNameIdOpt opt;
public FfiNameIdOpt opt = FfiNameIdOpt.None;
public String name;
public int nameId;

Expand Down
4 changes: 2 additions & 2 deletions ...xecutor/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiProperty.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public FfiProperty() {

public static class ByValue extends FfiProperty implements Structure.ByValue {}

public FfiPropertyOpt opt;
public FfiNameOrId.ByValue key;
public FfiPropertyOpt opt = FfiPropertyOpt.None;
public FfiNameOrId.ByValue key = new FfiNameOrId.ByValue();

@Override
public boolean equals(Object o) {
Expand Down
4 changes: 2 additions & 2 deletions ...xecutor/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
public class FfiVariable extends Structure {
public static class ByValue extends FfiVariable implements Structure.ByValue {}

public FfiNameOrId.ByValue tag;
public FfiProperty.ByValue property;
public FfiNameOrId.ByValue tag = new FfiNameOrId.ByValue();
public FfiProperty.ByValue property = new FfiProperty.ByValue();

@Override
public boolean equals(Object o) {
Expand Down
2 changes: 1 addition & 1 deletion ...or/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/InterOpCollectionBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public InterOpCollection build() throws OpArgIllegalException, UnsupportedStepEx
}
if (!step.getLabels().isEmpty()) {
String label = (String) step.getLabels().iterator().next();
op.setAlias(new OpArg(ArgUtils.asFfiAlias(label, true)));
op.setAlias(new OpArg(ArgUtils.asAlias(label, true)));
}
}
opCollection.appendInterOp(op);
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransformFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public String apply(Step arg) {
public String apply(Step arg) {
WhereTraversalStep.WhereEndStep endStep = (WhereTraversalStep.WhereEndStep) arg;
String matchTag = endStep.getScopeKeys().iterator().next();
P predicate = P.eq(ArgUtils.asFfiVar(matchTag, ""));
P predicate = P.eq(ArgUtils.asVar(matchTag, ""));
return flatPredicate("@", predicate);
}
}
Expand Down
20 changes: 11 additions & 9 deletions ...compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/StepTransformFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public InterOpBase apply(Step step) {
// set labels
List<String> labels = scanFusionStep.getGraphLabels();
for (String label : labels) {
params.addTable(ArgUtils.asFfiTag(label));
params.addTable(ArgUtils.asNameOrId(label));
}
List<HasContainer> containers = scanFusionStep.getHasContainers();
if (!containers.isEmpty()) {
Expand Down Expand Up @@ -159,7 +159,7 @@ public InterOpBase apply(Step step) {
String[] labels = ((VertexStep) step).getEdgeLabels();
if (labels.length > 0) {
for (String label : labels) {
params.addTable(ArgUtils.asFfiTag(label));
params.addTable(ArgUtils.asNameOrId(label));
}
}
op.setParams(params);
Expand Down Expand Up @@ -191,7 +191,7 @@ public InterOpBase apply(Step step) {
new OpArg<>(
expr,
(String expr1) -> {
FfiAlias.ByValue alias = ArgUtils.asFfiNoneAlias();
FfiAlias.ByValue alias = ArgUtils.asNoneAlias();
return Arrays.asList(Pair.with(expr1, alias));
}));
return op;
Expand Down Expand Up @@ -269,7 +269,7 @@ public InterOpBase apply(Step step) {
selectKey,
(String key) -> {
String expr = "@" + selectKey;
FfiAlias.ByValue alias = ArgUtils.asFfiNoneAlias();
FfiAlias.ByValue alias = ArgUtils.asNoneAlias();
return Collections.singletonList(Pair.with(expr, alias));
}));
return op;
Expand Down Expand Up @@ -317,7 +317,7 @@ public InterOpBase apply(Step step) {
|| mapTraversal instanceof ValueTraversal) {
// by head
String defaultExpr = "@";
FfiAlias.ByValue defaultAlias = ArgUtils.asFfiNoneAlias();
FfiAlias.ByValue defaultAlias = ArgUtils.asNoneAlias();
if (mapTraversal instanceof ValueTraversal) {
defaultExpr = "@." + ((ValueTraversal) mapTraversal).getPropertyKey();
}
Expand Down Expand Up @@ -531,7 +531,7 @@ public InterOpBase apply(Step step) {
exprStep.getExpr(),
(String expr) ->
Arrays.asList(
Pair.with(expr, ArgUtils.asFfiNoneAlias()))));
Pair.with(expr, ArgUtils.asNoneAlias()))));
return projectOp;
case FILTER:
default:
Expand Down Expand Up @@ -593,9 +593,9 @@ public InterOpBase apply(Step step) {
.map(
(id) -> {
if (id instanceof Integer) {
return ArgUtils.asFfiConst((Integer) id);
return ArgUtils.asConst((Integer) id);
} else if (id instanceof Long) {
return ArgUtils.asFfiConst((Long) id);
return ArgUtils.asConst((Long) id);
} else {
throw new OpArgIllegalException(
OpArgIllegalException.Cause.UNSUPPORTED_TYPE,
Expand All @@ -607,6 +607,8 @@ public InterOpBase apply(Step step) {
protected Function<ScanFusionStep, List<FfiNameOrId.ByValue>> LABELS_FROM_STEP =
(ScanFusionStep step) -> {
List<String> labels = step.getGraphLabels();
return labels.stream().map(k -> ArgUtils.asFfiTag(k)).collect(Collectors.toList());
return labels.stream()
.map(k -> ArgUtils.asNameOrId(k))
.collect(Collectors.toList());
};
}
4 changes: 2 additions & 2 deletions ...iler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ default FfiVariable.ByValue getExpressionAsVar(String expr) {
String tag = (splitExpr[0].length() > 0) ? splitExpr[0].substring(1) : splitExpr[0];
// property "" indicates none
String property = (splitExpr.length > 1) ? splitExpr[1] : "";
return ArgUtils.asFfiVar(tag, property);
return ArgUtils.asVar(tag, property);
}

default Map<String, Traversal.Admin> getProjectTraversals(TraversalParent parent) {
Expand Down Expand Up @@ -265,7 +265,7 @@ default ArgAggFn getAggFn(Step step, int stepIdx) {
Set<String> labels = step.getLabels();
if (labels != null && !labels.isEmpty()) {
String label = labels.iterator().next();
alias = ArgUtils.asFfiAlias(label, true);
alias = ArgUtils.asAlias(label, true);
}
// set to NONE by default
return new ArgAggFn(aggOpt, alias);
Expand Down
6 changes: 2 additions & 4 deletions ...c/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransformFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public List<InterOpBase> apply(TraversalParent parent) {
// optimize: if there is only one expression, alias with NONE
if (projectExprWithAlias.size() == 1) {
Pair single = projectExprWithAlias.get(0);
projectExprWithAlias.set(0, single.setAt1(ArgUtils.asFfiNoneAlias()));
projectExprWithAlias.set(0, single.setAt1(ArgUtils.asNoneAlias()));
}
ProjectOp op = new ProjectOp();
op.setExprWithAlias(new OpArg(projectExprWithAlias));
Expand Down Expand Up @@ -287,9 +287,7 @@ public List<InterOpBase> apply(TraversalParent parent) {
String queryAlias = (String) endStep.getLabels().iterator().next();
if (groupKeyVarWithAlias.size() == 1) {
Pair newPair =
groupKeyVarWithAlias
.get(0)
.setAt1(ArgUtils.asFfiAlias(queryAlias, true));
groupKeyVarWithAlias.get(0).setAt1(ArgUtils.asAlias(queryAlias, true));
groupKeyVarWithAlias.set(0, newPair);
} else if (groupKeyVarWithAlias.size() > 1) {
throw new OpArgIllegalException(
Expand Down
2 changes: 1 addition & 1 deletion ...r/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public static FfiAlias.ByValue getFfiAlias(AliasArg prefix) {
int stepIdx = prefix.getStepIdx();
int subTraversalId = prefix.getSubTraversalIdx();
String alias = prefix.getPrefix() + "_" + stepIdx + "_" + subTraversalId;
return ArgUtils.asFfiAlias(alias, false);
return ArgUtils.asAlias(alias, false);
}

// prefix is as the gremlin result key which is used to display
Expand Down
Loading

0 comments on commit a0da2a9

Please sign in to comment.