Skip to content

Commit

Permalink
Bugfix: Type pointers which started at >4GB were not serialized corre…
Browse files Browse the repository at this point in the history
…ctly
  • Loading branch information
dkoszewnik committed Jul 7, 2022
1 parent 0885bb9 commit d0fe0f2
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2014 Netflix, Inc.
* Copyright 2014-2022 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -67,7 +67,7 @@ private long[] deserializeLongPointerArray(DataInputStream dis) throws IOExcepti
long currentPointer = 0;

for(int i=0;i<numNodes;i++) {
int vInt = reader.readVInt();
long vInt = reader.readVLong();
if(vInt == -1) {
pointers[i] = -1;
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2014 Netflix, Inc.
* Copyright 2014-2022 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -58,7 +58,12 @@ private void serializePointerArray(DataOutputStream dos, long pointers[]) throws
if(pointers[i] == -1) {
buf.writeVInt(-1);
} else {
buf.writeVInt((int)(pointers[i] - currentPointer));
long delta = pointers[i] - currentPointer;
if(delta >= 0xFFFFFFFFL) {
buf.writeVLong(delta);
} else {
buf.writeVInt((int)delta);
}
currentPointer = pointers[i];
}
}
Expand Down
21 changes: 21 additions & 0 deletions src/main/java/com/netflix/nfgraph/util/ByteArrayBuffer.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,27 @@ public void writeVInt(int value) {
writeByte((byte)(value & 0x7F));
}
}

/**
* Writes a variable-byte encoded integer to the byte array.
*/
public void writeVLong(long value) {
if(value < 0) {
writeByte((byte)0x80);
return;
} else {
if(value > 0xFFFFFFFFFFFFFFL) writeByte((byte)(0x80 | ((value >>> 56) & 0x7FL)));
if(value > 0x1FFFFFFFFFFFFL) writeByte((byte)(0x80 | ((value >>> 49) & 0x7FL)));
if(value > 0x3FFFFFFFFFFL) writeByte((byte)(0x80 | ((value >>> 42) & 0x7FL)));
if(value > 0x7FFFFFFFFL) writeByte((byte)(0x80 | ((value >>> 35) & 0x7FL)));
if(value > 0xFFFFFFFL) writeByte((byte)(0x80 | ((value >>> 28) & 0x7FL)));
if(value > 0x1FFFFFL) writeByte((byte)(0x80 | ((value >>> 21) & 0x7FL)));
if(value > 0x3FFFL) writeByte((byte)(0x80 | ((value >>> 14) & 0x7FL)));
if(value > 0x7FL) writeByte((byte)(0x80 | ((value >>> 7) & 0x7FL)));

writeByte((byte)(value & 0x7F));
}
}

/**
* The current length of the written data, in bytes.
Expand Down
24 changes: 23 additions & 1 deletion src/main/java/com/netflix/nfgraph/util/ByteArrayReader.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2013 Netflix, Inc.
* Copyright 2013-2022 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -91,6 +91,28 @@ public int readVInt() {
return value;
}

/**
* @return a variable-byte long at the current offset. The offset is incremented by the size of the returned long.
*/
public long readVLong() {
if(pointer >= endByte)
return -1;

byte b = readByte();

if(b == (byte) 0x80)
return -1;

long value = b & 0x7F;
while ((b & 0x80) != 0) {
b = readByte();
value <<= 7;
value |= (b & 0x7F);
}

return value;
}

/**
* @return the byte at the current offset. The offset is incremented by one.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,32 @@ public void pointersMightStartGreaterThan2GB() throws IOException {
for(int i=0;i<ptrs.length;i++) {
Assert.assertEquals(ptrs[i], deserialized.getPointer("Test", i));
}

Assert.assertTrue(deserialized instanceof NFCompressedGraphIntPointers);
}

@Test
public void pointersMightStartGreaterThan4GB() throws IOException {
NFCompressedGraphLongPointers pointers = new NFCompressedGraphLongPointers();

long bigStartVal = Integer.MAX_VALUE;
bigStartVal *= 5;

long[] ptrs = new long[] { bigStartVal, bigStartVal + 10, bigStartVal + 20, bigStartVal + 100 };
pointers.addPointers("Test", ptrs);

NFCompressedGraphPointersSerializer serializer = new NFCompressedGraphPointersSerializer(pointers, bigStartVal + 125);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
serializer.serializePointers(new DataOutputStream(baos));

NFCompressedGraphPointersDeserializer deserializer = new NFCompressedGraphPointersDeserializer();
NFCompressedGraphPointers deserialized = deserializer.deserializePointers(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));

for(int i=0;i<ptrs.length;i++) {
Assert.assertEquals(ptrs[i], deserialized.getPointer("Test", i));
}

Assert.assertTrue(deserialized instanceof NFCompressedGraphLongPointers);
}


Expand Down

0 comments on commit d0fe0f2

Please sign in to comment.