datacolumns retrival fix

ChainSafe · Sep 12, 2024 · 2b10e4d · 2b10e4d
1 parent 56c8c6e
commit 2b10e4d
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 22 deletions.
diff --git a/packages/beacon-node/src/db/repositories/dataColumnSidecars.ts b/packages/beacon-node/src/db/repositories/dataColumnSidecars.ts
@@ -10,7 +10,7 @@ export const dataColumnSidecarsWrapperSsz = new ContainerType(
   {
     blockRoot: ssz.Root,
     slot: ssz.Slot,
-    dataColumnsLen: ssz.Uint16,
+    dataColumnsLen: ssz.Uint8,
     dataColumnsSize: ssz.UintNum64,
     // // each byte[i] tells what index (1 based) the column i is stored, 0 means not custodied
     // max value to represent will be 128 which can be represented in a byte
@@ -24,10 +24,9 @@ export type DataColumnSidecarsWrapper = ValueOf<typeof dataColumnSidecarsWrapper
 export const BLOCK_ROOT_IN_WRAPPER_INDEX = 0;
 export const BLOCK_SLOT_IN_WRAPPER_INDEX = 32;
 export const NUM_COLUMNS_IN_WRAPPER_INDEX = 40;
-export const COLUMN_SIZE_IN_WRAPPER_INDEX = 42;
-export const CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX = 50;
-export const DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX =
-  CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS + 4 + 4 * NUMBER_OF_COLUMNS;
+export const COLUMN_SIZE_IN_WRAPPER_INDEX = 41;
+export const CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX = 49;
+export const DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX = CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS + 4;
 
 /**
  * dataColumnSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlock.message))

diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts
@@ -8,6 +8,7 @@ import {
   DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX,
   COLUMN_SIZE_IN_WRAPPER_INDEX,
   CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
+  NUM_COLUMNS_IN_WRAPPER_INDEX,
 } from "../../../db/repositories/dataColumnSidecars.js";
 
 export async function* onDataColumnSidecarsByRange(
@@ -78,16 +79,21 @@ export function* iterateDataColumnBytesFromWrapper(
   blockSlot: Slot,
   columns: ColumnIndex[]
 ): Iterable<ResponseOutgoing> {
+  const retrivedColumnsLen = ssz.Uint8.deserialize(
+    dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX)
+  );
   const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice(
     COLUMN_SIZE_IN_WRAPPER_INDEX,
     CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX
   );
   const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
-  const allDataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX);
   const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice(
     CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
     CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS
   );
+  const allDataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(
+    DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen
+  );
 
   const columnsLen = allDataColumnSidecarsBytes.length / columnsSize;
   // no columns possibly no blob

diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts
@@ -8,6 +8,7 @@ import {
   DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX,
   CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
   COLUMN_SIZE_IN_WRAPPER_INDEX,
+  NUM_COLUMNS_IN_WRAPPER_INDEX,
 } from "../../../db/repositories/dataColumnSidecars.js";
 
 export async function* onDataColumnSidecarsByRoot(
@@ -47,12 +48,17 @@ export async function* onDataColumnSidecarsByRoot(
         throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`);
       }
 
+      const retrivedColumnsLen = ssz.Uint8.deserialize(
+        dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX)
+      );
       const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice(
         COLUMN_SIZE_IN_WRAPPER_INDEX,
         CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX
       );
       const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
-      const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX);
+      const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(
+        DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen
+      );
 
       const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice(
         CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,

diff --git a/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts b/packages/beacon-node/test/unit/db/api/repositories/dataColumn.test.ts
@@ -12,6 +12,7 @@ import {
   DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX,
   COLUMN_SIZE_IN_WRAPPER_INDEX,
   CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
+  NUM_COLUMNS_IN_WRAPPER_INDEX,
 } from "../../../../../src/db/repositories/dataColumnSidecars.js";
 import {testLogger} from "../../../../utils/logger.js";
 import {computeDataColumnSidecars} from "../../../../../src/util/blobs.js";
@@ -54,12 +55,12 @@ describe("block archive repository", function () {
     singedBlock.message.body.blobKzgCommitments.push(commitment);
     singedBlock.message.body.blobKzgCommitments.push(commitment);
     singedBlock.message.body.blobKzgCommitments.push(commitment);
-    const dataColumnSidecars = computeDataColumnSidecars(config, singedBlock, {
+    const allDataColumnSidecars = computeDataColumnSidecars(config, singedBlock, {
       blobs: [blob, blob, blob],
       kzgProofs: [commitment, commitment, commitment],
     });
-    for (let j = 0; j < dataColumnSidecars.length; j++) {
-      dataColumnSidecars[j].index = j;
+    for (let j = 0; j < allDataColumnSidecars.length; j++) {
+      allDataColumnSidecars[j].index = j;
     }
 
     const blobKzgCommitmentsLen = 3;
@@ -68,39 +69,50 @@ describe("block archive repository", function () {
       blobKzgCommitmentsLen *
         (ssz.peerdas.Cell.fixedSize + ssz.deneb.KZGCommitment.fixedSize + ssz.deneb.KZGProof.fixedSize);
 
-    const numColumns = NUMBER_OF_COLUMNS;
-    const blobsLen = (singedBlock.message as peerdas.BeaconBlock).body.blobKzgCommitments.length;
-
-    // const dataColumnsSize = ssz.peerdas.DataColumnSidecar.minSize + blobsLen * (ssz.peerdas.Cell.fixedSize + ssz.deneb.KZGCommitment.fixedSize + ssz.deneb.KZGProof.fixedSize);
+    const dataColumnSidecars = allDataColumnSidecars.slice(0, 7);
+    const dataColumnsLen = dataColumnSidecars.length;
+    const dataColumnsIndex = Array.from({length: NUMBER_OF_COLUMNS}, (_v, _i) => 0);
+    for (let i = 0; i < dataColumnsLen; i++) {
+      dataColumnsIndex[i] = i + 1;
+    }
+    dataColumnsIndex[127] = 19;
 
-    // const dataColumnsLen = blockInput.blockData;
     const writeData = {
       blockRoot,
       slot,
-      blobsLen,
-      columnsSize,
-      dataColumnsIndex: new ByteVectorType(NUMBER_OF_COLUMNS),
-      dataColumnSidecars: ssz.peerdas.DataColumnSidecars,
+      dataColumnsLen,
+      dataColumnsSize: columnsSize,
+      dataColumnsIndex,
+      dataColumnSidecars,
     };
 
     await dataColumnRepo.add(writeData);
     const retrievedBinary = await dataColumnRepo.getBinary(blockRoot);
+    const lastIndex = retrievedBinary.findIndex((i) => i === 19);
     if (!retrievedBinary) throw Error("get by root returned null");
 
     const retrieved = dataColumnSidecarsWrapperSsz.deserialize(retrievedBinary);
     expect(dataColumnSidecarsWrapperSsz.equals(retrieved, writeData)).toBe(true);
 
+    const retrivedColumnsLen = ssz.Uint8.deserialize(
+      retrievedBinary.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX)
+    );
+    expect(retrivedColumnsLen === dataColumnsLen).toBe(true);
+
     const retrievedColumnsSizeBytes = retrievedBinary.slice(
       COLUMN_SIZE_IN_WRAPPER_INDEX,
       CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX
     );
 
     const retrievedColumnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
     expect(retrievedColumnsSize === columnsSize).toBe(true);
-    const dataColumnSidecarsBytes = retrievedBinary.slice(DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX);
-    expect(dataColumnSidecarsBytes.length === columnsSize * numColumns).toBe(true);
+    const dataColumnSidecarsBytes = retrievedBinary.slice(
+      DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen
+    );
+    // console.log({dataColumnSidecarsBytes: dataColumnSidecarsBytes.length, computeLen: dataColumnSidecarsBytes.length/columnsSize, dataColumnsLen, dataColumnSidecars: dataColumnSidecars.length, retrievedColumnsSize, columnsSize, allDataColumnSidecars: allDataColumnSidecars.length, lastIndex, DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX, retrivedColumnsLen})
+    expect(dataColumnSidecarsBytes.length === columnsSize * dataColumnsLen).toBe(true);
 
-    for (let j = 0; j < numColumns; j++) {
+    for (let j = 0; j < dataColumnsLen; j++) {
       const dataColumnBytes = dataColumnSidecarsBytes.slice(j * columnsSize, (j + 1) * columnsSize);
       const retrivedDataColumnSidecar = ssz.peerdas.DataColumnSidecar.deserialize(dataColumnBytes);
       const index = retrivedDataColumnSidecar.index;