Skip to content

Commit

Permalink
[ARROW-6355][Java] Wrapper visit parameters into a pojo
Browse files Browse the repository at this point in the history
  • Loading branch information
@liyafan82
liyafan82 committed Aug 29, 2019
1 parent 53c1e0b commit 7482414
Show file tree
Hide file tree
Showing 9 changed files with 495 additions and 413 deletions.
13 changes: 9 additions & 4 deletions java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.compare.RangeEqualsParameter;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;

import io.netty.buffer.ArrowBuf;
Expand All @@ -43,11 +44,15 @@ public static <V extends ValueVector> void populateRunStartIndicators(V vector,
runStarts.setZero(0, bufSize);

BitVectorHelper.setValidityBitToOne(runStarts, 0);

RangeEqualsVisitor visitor = new RangeEqualsVisitor();
RangeEqualsParameter param = new RangeEqualsParameter()
.setRight(vector)
.setLeft(vector)
.setLength(1)
.setTypeCheckNeeded(false);
for (int i = 1; i < vector.getValueCount(); i++) {
RangeEqualsVisitor visitor = new RangeEqualsVisitor(
vector, i - 1, i, /* length */1, /* need check type*/false);
if (!visitor.equals(vector)) {
param.setLeftStart(i).setRightStart(i - 1);
if (!visitor.rangeEquals(param)) {
BitVectorHelper.setValidityBitToOne(runStarts, i);
}
}
Expand Down
197 changes: 28 additions & 169 deletions java/vector/src/main/java/org/apache/arrow/vector/compare/ApproxEqualsVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,9 @@

package org.apache.arrow.vector.compare;

import java.util.List;

import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.Float4Vector;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.complex.UnionVector;

/**
* Visitor to compare floating point.
Expand All @@ -54,28 +45,6 @@ public ApproxEqualsVisitor(float floatEpsilon, double doubleEpsilon) {
this.doubleEpsilon = doubleEpsilon;
}

public ApproxEqualsVisitor(ValueVector right, float epsilon) {
this (right, epsilon, epsilon, true);
}

public ApproxEqualsVisitor(ValueVector right, float floatEpsilon, double doubleEpsilon) {
this (right, floatEpsilon, doubleEpsilon, true);
}

public ApproxEqualsVisitor(ValueVector right, float floatEpsilon, double doubleEpsilon, boolean typeCheckNeeded) {
this (right, floatEpsilon, doubleEpsilon, typeCheckNeeded, 0, 0, right.getValueCount());
}

/**
* Construct an instance.
*/
public ApproxEqualsVisitor(ValueVector right, float floatEpsilon, double doubleEpsilon, boolean typeCheckNeeded,
int leftStart, int rightStart, int length) {
this.floatEpsilon = floatEpsilon;
this.doubleEpsilon = doubleEpsilon;
set(right, rightStart, leftStart, length, typeCheckNeeded);
}

public void setFloatDiffFunction(DiffFunction<Float> floatDiffFunction) {
this.floatDiffFunction = floatDiffFunction;
}
Expand All @@ -85,152 +54,39 @@ public void setDoubleDiffFunction(DiffFunction<Double> doubleDiffFunction) {
}

@Override
public Boolean visit(BaseFixedWidthVector left, Void value) {
public Boolean visit(BaseFixedWidthVector left, RangeEqualsParameter parameter) {
parameter.setLeft(left);
if (left instanceof Float4Vector) {
return validate(left) && float4ApproxEquals((Float4Vector) left);
return parameter.validate() && float4ApproxEquals(parameter);
} else if (left instanceof Float8Vector) {
return validate(left) && float8ApproxEquals((Float8Vector) left);
return parameter.validate() && float8ApproxEquals(parameter);
} else {
return super.visit(left, value);
return super.visit(left, parameter);
}
}

@Override
protected boolean compareUnionVectors(UnionVector left) {

UnionVector rightVector = (UnionVector) right;

List<FieldVector> leftChildren = left.getChildrenFromFields();
List<FieldVector> rightChildren = rightVector.getChildrenFromFields();

if (leftChildren.size() != rightChildren.size()) {
return false;
}

ApproxEqualsVisitor visitor = new ApproxEqualsVisitor(floatEpsilon, doubleEpsilon);
for (int k = 0; k < leftChildren.size(); k++) {
visitor.set(rightChildren.get(k), 0, 0, rightChildren.get(k).getValueCount(), true);
if (!leftChildren.get(k).accept(visitor, null)) {
return false;
}
}
return true;
}

@Override
protected boolean compareStructVectors(NonNullableStructVector left) {

NonNullableStructVector rightVector = (NonNullableStructVector) right;

if (!left.getChildFieldNames().equals(rightVector.getChildFieldNames())) {
return false;
}

ApproxEqualsVisitor visitor = new ApproxEqualsVisitor(floatEpsilon, doubleEpsilon);
for (String name : left.getChildFieldNames()) {
visitor.set(rightVector.getChild(name), 0, 0, rightVector.getChild(name).getValueCount(), true);
if (!left.getChild(name).accept(visitor, null)) {
return false;
}
}

return true;
}

@Override
protected boolean compareListVectors(ListVector left) {
ApproxEqualsVisitor visitor = new ApproxEqualsVisitor(floatEpsilon, doubleEpsilon);
for (int i = 0; i < length; i++) {
int leftIndex = leftStart + i;
int rightIndex = rightStart + i;

boolean isNull = left.isNull(leftIndex);
if (isNull != right.isNull(rightIndex)) {
return false;
}

int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH;

if (!isNull) {
final int startIndexLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth);
final int endIndexLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth);

final int startIndexRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth);
final int endIndexRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth);

if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) {
return false;
}

ValueVector leftDataVector = left.getDataVector();
ValueVector rightDataVector = ((ListVector)right).getDataVector();

visitor.set(rightDataVector, startIndexRight, startIndexLeft, endIndexLeft - startIndexLeft, true);
if (!leftDataVector.accept(visitor, null)) {
return false;
}
}
}
return true;
protected ApproxEqualsVisitor createInnerVisitor() {
return new ApproxEqualsVisitor(floatEpsilon, doubleEpsilon);
}

protected boolean compareFixedSizeListVectors(FixedSizeListVector left) {
private boolean float4ApproxEquals(RangeEqualsParameter parameter) {
Float4Vector leftVector = (Float4Vector) parameter.getLeft();
Float4Vector rightVector = (Float4Vector) parameter.getRight();

if (left.getListSize() != ((FixedSizeListVector)right).getListSize()) {
return false;
}
for (int i = 0; i < parameter.getLength(); i++) {
int leftIndex = parameter.getLeftStart() + i;
int rightIndex = parameter.getRightStart() + i;

ApproxEqualsVisitor visitor = new ApproxEqualsVisitor(floatEpsilon, doubleEpsilon);
for (int i = 0; i < length; i++) {
int leftIndex = leftStart + i;
int rightIndex = rightStart + i;
boolean isNull = leftVector.isNull(leftIndex);

boolean isNull = left.isNull(leftIndex);
if (isNull != right.isNull(rightIndex)) {
if (isNull != rightVector.isNull(rightIndex)) {
return false;
}

int listSize = left.getListSize();

if (!isNull) {
final int startIndexLeft = leftIndex * listSize;
final int endIndexLeft = (leftIndex + 1) * listSize;

final int startIndexRight = rightIndex * listSize;
final int endIndexRight = (rightIndex + 1) * listSize;

if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) {
return false;
}

ValueVector leftDataVector = left.getDataVector();
ValueVector rightDataVector = ((FixedSizeListVector)right).getDataVector();

visitor.set(rightDataVector, rightStart, leftStart, endIndexLeft - startIndexLeft, true);
if (!leftDataVector.accept(visitor, null)) {
return false;
}
}
}
return true;
}

private boolean float4ApproxEquals(Float4Vector left) {

for (int i = 0; i < length; i++) {
int leftIndex = leftStart + i;
int rightIndex = rightStart + i;

boolean isNull = left.isNull(leftIndex);

if (isNull != right.isNull(rightIndex)) {
return false;
}

if (!isNull) {

Float leftValue = left.get(leftIndex);
Float rightValue = ((Float4Vector)right).get(rightIndex);
Float leftValue = leftVector.get(leftIndex);
Float rightValue = rightVector.get(rightIndex);
if (leftValue.isNaN()) {
return rightValue.isNaN();
}
Expand All @@ -245,21 +101,24 @@ private boolean float4ApproxEquals(Float4Vector left) {
return true;
}

private boolean float8ApproxEquals(Float8Vector left) {
for (int i = 0; i < length; i++) {
int leftIndex = leftStart + i;
int rightIndex = rightStart + i;
private boolean float8ApproxEquals(RangeEqualsParameter parameter) {
Float8Vector leftVector = (Float8Vector) parameter.getLeft();
Float8Vector rightVector = (Float8Vector) parameter.getRight();

for (int i = 0; i < parameter.getLength(); i++) {
int leftIndex = parameter.getLeftStart() + i;
int rightIndex = parameter.getRightStart() + i;

boolean isNull = left.isNull(leftIndex);
boolean isNull = leftVector.isNull(leftIndex);

if (isNull != right.isNull(rightIndex)) {
if (isNull != rightVector.isNull(rightIndex)) {
return false;
}

if (!isNull) {

Double leftValue = left.get(leftIndex);
Double rightValue = ((Float8Vector)right).get(rightIndex);
Double leftValue = leftVector.get(leftIndex);
Double rightValue = rightVector.get(rightIndex);
if (leftValue.isNaN()) {
return rightValue.isNaN();
}
Expand Down
Loading

0 comments on commit 7482414

Please sign in to comment.