Over account ram usage of array-based vectors (#104159)

This is part of the effort to enable heap attack tests. Currently, when creating Blocks from Vectors, we don't account for the extra bytes used by VectorBlock alongside the Vector. In heap attack tests, the unaccounted memory can be significant, as seen in the manyEval test, where the unaccounted bytes is 40, while the vector is just 128 bytes, resulting in an OOM instead of triggering the circuit breaker. Ideally, Vector#asBlock should adjust the breaker, and callers should release the returned block. While I tried to implement this, there are many places in our tests where we use Vector#asBlock without releasing the block. This is possible because Vector#asBlock doesn't increase the reference to the vector. In the long run, I think we should remove Vector#asBlock and force that callers use newBlockFromVector from the BlockFactory. This would provide clearer semantics and enable proper tracking of blocks in assertions. This PR adds the extra bytes used by VectorBlock to ArrayVector, allowing us to run heap-attack tests.
elastic · Jan 9, 2024 · a5ab9f3 · a5ab9f3
1 parent 7c48efc
commit a5ab9f3
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 19 deletions.
diff --git a/...sql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java b/...sql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java
@@ -17,7 +17,9 @@
  */
 final class BooleanArrayVector extends AbstractVector implements BooleanVector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class);
 
     private final boolean[] values;
 

diff --git a/...ql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java b/...ql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java
@@ -19,7 +19,9 @@
  */
 final class BytesRefArrayVector extends AbstractVector implements BytesRefVector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance(BytesRefVectorBlock.class);
 
     private final BytesRefArray values;
 

diff --git a/...esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java b/...esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java
@@ -17,7 +17,9 @@
  */
 final class DoubleArrayVector extends AbstractVector implements DoubleVector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class);
 
     private final double[] values;
 

diff --git a/...in/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java b/...in/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java
@@ -17,7 +17,9 @@
  */
 final class IntArrayVector extends AbstractVector implements IntVector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class);
 
     private final int[] values;
 

diff --git a/...n/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java b/...n/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java
@@ -17,7 +17,9 @@
  */
 final class LongArrayVector extends AbstractVector implements LongVector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class);
 
     private final long[] values;
 

diff --git a/...ck/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st b/...ck/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st
@@ -28,7 +28,9 @@ $endif$
  */
 final class $Type$ArrayVector extends AbstractVector implements $Type$Vector {
 
-    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayVector.class)
+        // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector.
+        + RamUsageEstimator.shallowSizeOfInstance($Type$VectorBlock.class);
 
 $if(BytesRef)$
     private final BytesRefArray values;

diff --git a/...lugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/...lugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.tests.util.RamUsageTester;
 import org.apache.lucene.tests.util.RamUsageTester.Accumulator;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.util.BigArray;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BytesRefArray;
@@ -41,7 +42,9 @@ public class BlockAccountingTests extends ComputeTestCase {
     public void testBooleanVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newBooleanArrayVector(new boolean[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            BooleanVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newBooleanArrayVector(new boolean[] { randomBoolean() }, 1);
@@ -59,7 +62,9 @@ public void testBooleanVector() {
     public void testIntVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newIntArrayVector(new int[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            IntVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newIntArrayVector(new int[] { randomInt() }, 1);
@@ -77,7 +82,9 @@ public void testIntVector() {
     public void testLongVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newLongArrayVector(new long[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            LongVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newLongArrayVector(new long[] { randomLong() }, 1);
@@ -96,7 +103,9 @@ public void testLongVector() {
     public void testDoubleVector() {
         BlockFactory blockFactory = blockFactory();
         Vector empty = blockFactory.newDoubleArrayVector(new double[] {}, 0);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            DoubleVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Vector emptyPlusOne = blockFactory.newDoubleArrayVector(new double[] { randomDouble() }, 1);
@@ -118,7 +127,9 @@ public void testBytesRefVector() {
         var emptyArray = new BytesRefArray(0, blockFactory.bigArrays());
         var arrayWithOne = new BytesRefArray(0, blockFactory.bigArrays());
         Vector emptyVector = blockFactory.newBytesRefArrayVector(emptyArray, 0);
-        long expectedEmptyVectorUsed = RamUsageTester.ramUsed(emptyVector, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyVectorUsed = RamUsageTester.ramUsed(emptyVector, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            BytesRefVectorBlock.class
+        );
         assertThat(emptyVector.ramBytesUsed(), is(expectedEmptyVectorUsed));
 
         var bytesRef = new BytesRef(randomAlphaOfLengthBetween(1, 16));
@@ -135,7 +146,9 @@ public void testBytesRefVector() {
     public void testBooleanBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            BooleanVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new BooleanArrayBlock(
@@ -181,14 +194,18 @@ public void testBooleanBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            BooleanVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), lessThanOrEqualTo(expectedEmptyUsed));
     }
 
     public void testIntBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new IntArrayBlock(new int[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            IntVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new IntArrayBlock(
@@ -225,14 +242,18 @@ public void testIntBlock() {
     public void testIntBlockWithNullFirstValues() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new IntArrayBlock(new int[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            IntVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }
 
     public void testLongBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new LongArrayBlock(new long[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            LongVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new LongArrayBlock(
@@ -278,14 +299,18 @@ public void testLongBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            LongVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }
 
     public void testDoubleBlock() {
         BlockFactory blockFactory = blockFactory();
         Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory);
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            DoubleVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
 
         Block emptyPlusOne = new DoubleArrayBlock(
@@ -331,7 +356,9 @@ public void testDoubleBlockWithNullFirstValues() {
             Block.MvOrdering.UNORDERED,
             blockFactory()
         );
-        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR);
+        long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR) + RamUsageEstimator.shallowSizeOfInstance(
+            DoubleVectorBlock.class
+        );
         assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed));
     }