Implement Number64UintArray and tree_newTreeFromUint64Deltas (#159)

* Implement Number64UintArray and tree_newTreeFromUint64Deltas

* Create a separate Number64ListType

* Fix lint

* Address PR comment: more comments

* Use for(;;) pattern and update subtreeFillToContents perf test

* Make hash object methods applicable to basic types

Add optional tree_* hash object methods to BasicType
Remove FieldInfo.isNumber64Type
Remove separate number64_* functions
Rename applyDeltas methods

Co-authored-by: Cayman <[email protected]>

.gitignore

@@ -4,6 +4,7 @@ dist/
 lib/
 node_modules/
 .idea/
+.vscode
 yarn-error.log
 
 test/spec-tests-data

src/types/basic/abstract.ts

@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable @typescript-eslint/no-unused-vars */
+import {HashObject} from "@chainsafe/as-sha256";
 import {isTypeOf, Type} from "../type";
 
 export const BASIC_TYPE = Symbol.for("ssz/BasicType");
@@ -65,6 +66,8 @@ export abstract class BasicType<T> extends Type<T> {
   }
 
   abstract struct_deserializeFromBytes(data: Uint8Array, offset: number): T;
+  struct_deserializeFromHashObject?(data: HashObject, byteOffset: number): T;
+  struct_serializeToHashObject?(value: T, output: HashObject, byteOffset: number): number;
 
   struct_hashTreeRoot(value: T): Uint8Array {
     const output = new Uint8Array(32);

src/types/basic/uint.ts

@@ -2,6 +2,7 @@ import {Json} from "../../interface";
 import {bigIntPow} from "../../util/bigInt";
 import {isTypeOf, Type} from "../type";
 import {BasicType} from "./abstract";
+import {HashObject} from "@chainsafe/as-sha256";
 
 export interface IUintOptions {
   byteLength: number;
@@ -32,13 +33,18 @@ export abstract class UintType<T> extends BasicType<T> {
 }
 
 export const NUMBER_UINT_TYPE = Symbol.for("ssz/NumberUintType");
+export const NUMBER_64_UINT_TYPE = Symbol.for("ssz/Number64UintType");
 
 const BIGINT_4_BYTES = BigInt(32);
 
 export function isNumberUintType(type: Type<unknown>): type is NumberUintType {
   return isTypeOf(type, NUMBER_UINT_TYPE);
 }
 
+export function isNumber64UintType(type: Type<unknown>): type is Number64UintType {
+  return isTypeOf(type, NUMBER_64_UINT_TYPE);
+}
+
 export class NumberUintType extends UintType<number> {
   _maxBigInt?: BigInt;
 
@@ -128,6 +134,98 @@ export class NumberUintType extends UintType<number> {
   }
 }
 
+const TWO_POWER_32 = 2 ** 32;
+
+/**
+ * For 64 bit number, we want to operator on HashObject
+ * over bytes to improve performance.
+ */
+export class Number64UintType extends NumberUintType {
+  constructor() {
+    super({byteLength: 8});
+    this._typeSymbols.add(NUMBER_64_UINT_TYPE);
+  }
+
+  /**
+   * TODO: move this logic all the way to persistent-merkle-tree?
+   * That's save us 1 time to traverse the tree in the applyDelta scenario
+   */
+  struct_deserializeFromHashObject(data: HashObject, byteOffset: number): number {
+    const numberOffset = Math.floor(byteOffset / 8);
+    // a chunk contains 4 items
+    if (numberOffset < 0 || numberOffset > 3) {
+      throw new Error(`Invalid numberOffset ${numberOffset}`);
+    }
+    let low32Number = 0;
+    let high32Number = 0;
+
+    switch (numberOffset) {
+      case 0:
+        low32Number = data.h0 & 0xffffffff;
+        high32Number = data.h1 & 0xffffffff;
+        break;
+      case 1:
+        low32Number = data.h2 & 0xffffffff;
+        high32Number = data.h3 & 0xffffffff;
+        break;
+      case 2:
+        low32Number = data.h4 & 0xffffffff;
+        high32Number = data.h5 & 0xffffffff;
+        break;
+      case 3:
+        low32Number = data.h6 & 0xffffffff;
+        high32Number = data.h7 & 0xffffffff;
+        break;
+      default:
+        throw new Error(`Invalid offset ${numberOffset}`);
+    }
+    if (low32Number < 0) low32Number = low32Number >>> 0;
+    if (high32Number === 0) {
+      return low32Number;
+    } else if (high32Number < 0) {
+      high32Number = high32Number >>> 0;
+    }
+    if (low32Number === 0xffffffff && high32Number === 0xffffffff) {
+      return Infinity;
+    }
+    return high32Number * TWO_POWER_32 + low32Number;
+  }
+
+  struct_serializeToHashObject(value: number, output: HashObject, byteOffset: number): number {
+    const numberOffset = Math.floor(byteOffset / 8);
+    let low32Number: number;
+    let high32Number: number;
+    if (value !== Infinity) {
+      low32Number = value & 0xffffffff;
+      high32Number = Math.floor(value / TWO_POWER_32) & 0xffffffff;
+    } else {
+      low32Number = 0xffffffff;
+      high32Number = 0xffffffff;
+    }
+    switch (numberOffset) {
+      case 0:
+        output.h0 = low32Number;
+        output.h1 = high32Number;
+        break;
+      case 1:
+        output.h2 = low32Number;
+        output.h3 = high32Number;
+        break;
+      case 2:
+        output.h4 = low32Number;
+        output.h5 = high32Number;
+        break;
+      case 3:
+        output.h6 = low32Number;
+        output.h7 = high32Number;
+        break;
+      default:
+        throw new Error(`Invalid offset ${numberOffset}`);
+    }
+    return numberOffset + 1;
+  }
+}
+
 export const BIGINT_UINT_TYPE = Symbol.for("ssz/BigIntUintType");
 
 export function isBigIntUintType(type: Type<unknown>): type is BigIntUintType {

src/types/composite/container.ts

@@ -15,6 +15,8 @@ import {SszErrorPath} from "../../util/errorPath";
 import {toExpectedCase} from "../../util/json";
 import {isTreeBacked} from "../../backings/tree/treeValue";
 import {basicTypeToLeafNode} from "../../util/basic";
+import {Number64UintType, NumberUintType} from "../basic";
+import {newHashObject} from "../../util/hash";
 
 export interface IContainerOptions {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -457,12 +459,16 @@ export class ContainerType<T extends ObjectLike = ObjectLike> extends CompositeT
     if (!fieldInfo) {
       return undefined;
     }
-    const {isBasic, gindex} = fieldInfo;
-    if (isBasic) {
-      const chunk = target.getRoot(gindex);
+    if (fieldInfo.isBasic) {
+      // Number64Uint wants to work on HashObject to improve performance
+      if ((fieldType as NumberUintType).struct_deserializeFromHashObject) {
+        const hashObject = target.getHashObject(fieldInfo.gindex);
+        return (fieldType as Number64UintType).struct_deserializeFromHashObject(hashObject, 0);
+      }
+      const chunk = target.getRoot(fieldInfo.gindex);
       return fieldType.struct_deserializeFromBytes(chunk, 0);
     } else {
-      return target.getSubtree(gindex);
+      return target.getSubtree(fieldInfo.gindex);
     }
   }
 
@@ -472,14 +478,20 @@ export class ContainerType<T extends ObjectLike = ObjectLike> extends CompositeT
     if (!fieldInfo) {
       throw new Error("Invalid container field name");
     }
-    const {isBasic, gindex} = fieldInfo;
-    if (isBasic) {
+    if (fieldInfo.isBasic) {
+      // Number64Uint wants to work on HashObject to improve performance
+      if ((fieldType as Number64UintType).struct_serializeToHashObject) {
+        const hashObject = newHashObject();
+        (fieldType as Number64UintType).struct_serializeToHashObject(value as number, hashObject, 0);
+        target.setHashObject(fieldInfo.gindex, hashObject);
+        return true;
+      }
       const chunk = new Uint8Array(32);
       fieldType.struct_serializeToBytes(value, chunk, 0);
-      target.setRoot(gindex, chunk);
+      target.setRoot(fieldInfo.gindex, chunk);
       return true;
     } else {
-      target.setSubtree(gindex, value as Tree);
+      target.setSubtree(fieldInfo.gindex, value as Tree);
       return true;
     }
   }

src/types/composite/list.ts

@@ -1,9 +1,19 @@
 import {Json, List} from "../../interface";
 import {IArrayOptions, BasicArrayType, CompositeArrayType} from "./array";
-import {isBasicType, number32Type} from "../basic";
+import {isBasicType, isNumber64UintType, number32Type} from "../basic";
 import {IJsonOptions, isTypeOf, Type} from "../type";
 import {mixInLength} from "../../util/compat";
-import {BranchNode, concatGindices, Gindex, Node, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree";
+import {cloneHashObject} from "../../util/hash";
+import {
+  BranchNode,
+  concatGindices,
+  Gindex,
+  LeafNode,
+  Node,
+  subtreeFillToContents,
+  Tree,
+  zeroNode,
+} from "@chainsafe/persistent-merkle-tree";
 import {isTreeBacked} from "../../backings/tree/treeValue";
 
 /**
@@ -39,7 +49,9 @@ export function isListType<T extends List<any> = List<any>>(type: Type<unknown>)
 export const ListType: ListTypeConstructor =
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   (function ListType<T extends List<any> = List<any>>(options: IListOptions): ListType<T> {
-    if (isBasicType(options.elementType)) {
+    if (isNumber64UintType(options.elementType)) {
+      return new Number64ListType(options);
+    } else if (isBasicType(options.elementType)) {
       return new BasicListType(options);
     } else {
       return new CompositeListType(options);
@@ -220,6 +232,92 @@ export class BasicListType<T extends List<unknown> = List<unknown>> extends Basi
   }
 }
 
+/** For Number64UintType, it takes 64 / 8 = 8 bytes per item, each chunk has 32 bytes = 4 items */
+const NUMBER64_LIST_NUM_ITEMS_PER_CHUNK = 4;
+
+/**
+ * An optimization for Number64 using HashObject and new method to work with deltas.
+ */
+export class Number64ListType<T extends List<number> = List<number>> extends BasicListType<T> {
+  constructor(options: IListOptions) {
+    super(options);
+  }
+
+  /** @override */
+  tree_getValueAtIndex(target: Tree, index: number): number {
+    const chunkGindex = this.getGindexAtChunkIndex(this.getChunkIndex(index));
+    const hashObject = target.getHashObject(chunkGindex);
+    // 4 items per chunk
+    const offsetInChunk = (index % 4) * 8;
+    return this.elementType.struct_deserializeFromHashObject!(hashObject, offsetInChunk) as number;
+  }
+
+  /** @override */
+  tree_setValueAtIndex(target: Tree, index: number, value: number, expand = false): boolean {
+    const chunkGindex = this.getGindexAtChunkIndex(this.getChunkIndex(index));
+    const hashObject = cloneHashObject(target.getHashObject(chunkGindex));
+    // 4 items per chunk
+    const offsetInChunk = (index % 4) * 8;
+    this.elementType.struct_serializeToHashObject!(value as number, hashObject, offsetInChunk);
+    target.setHashObject(chunkGindex, hashObject, expand);
+    return true;
+  }
+
+  /**
+   * delta > 0 increments the underlying value, delta < 0 decrements the underlying value
+   * returns the new value
+   **/
+  tree_applyDeltaAtIndex(target: Tree, index: number, delta: number): number {
+    const chunkGindex = this.getGindexAtChunkIndex(this.getChunkIndex(index));
+    const hashObject = cloneHashObject(target.getHashObject(chunkGindex));
+    // 4 items per chunk
+    const offsetInChunk = (index % 4) * 8;
+
+    let value = this.elementType.struct_deserializeFromHashObject!(hashObject, offsetInChunk) as number;
+    value += delta;
+    if (value < 0) value = 0;
+    this.elementType.struct_serializeToHashObject!(value as number, hashObject, offsetInChunk);
+    target.setHashObject(chunkGindex, hashObject);
+
+    return value;
+  }
+
+  /**
+   * delta > 0 means an increasement, delta < 0 means a decreasement
+   * returns the new tree and new values
+   **/
+  tree_newTreeFromDeltas(target: Tree, deltas: number[]): [Tree, number[]] {
+    if (deltas.length !== this.tree_getLength(target)) {
+      throw new Error(`Expect delta length ${this.tree_getLength(target)}, actual ${deltas.length}`);
+    }
+    const chunkDepth = this.getChunkDepth();
+    const length = deltas.length;
+    let nodeIdx = 0;
+    const newLeafNodes: LeafNode[] = [];
+    const newValues: number[] = [];
+    const chunkCount = Math.ceil(length / NUMBER64_LIST_NUM_ITEMS_PER_CHUNK);
+    const currentNodes = target.getNodesAtDepth(chunkDepth, 0, chunkCount);
+    for (let i = 0; i < currentNodes.length; i++) {
+      const node = currentNodes[i];
+      const hashObject = cloneHashObject(node);
+      for (let offset = 0; offset < NUMBER64_LIST_NUM_ITEMS_PER_CHUNK; offset++) {
+        const index = nodeIdx * NUMBER64_LIST_NUM_ITEMS_PER_CHUNK + offset;
+        if (index >= length) break;
+        let value =
+          (this.elementType.struct_deserializeFromHashObject!(hashObject, offset * 8) as number) + deltas[index];
+        if (value < 0) value = 0;
+        newValues.push(value);
+        // mutate hashObject at offset
+        this.elementType.struct_serializeToHashObject!(value, hashObject, offset * 8);
+      }
+      newLeafNodes.push(new LeafNode(hashObject));
+      nodeIdx++;
+    }
+    const newRootNode = subtreeFillToContents(newLeafNodes, chunkDepth);
+    return [new Tree(newRootNode), newValues];
+  }
+}
+
 export class CompositeListType<T extends List<unknown> = List<unknown>> extends CompositeArrayType<T> {
   limit: number;
 

src/util/hash.ts

@@ -1,9 +1,61 @@
 /** @module ssz */
-import SHA256 from "@chainsafe/as-sha256";
+import SHA256, {HashObject} from "@chainsafe/as-sha256";
 
 /**
  * Hash used for hashTreeRoot
  */
 export function hash(...inputs: Uint8Array[]): Uint8Array {
   return SHA256.digest(Buffer.concat(inputs));
 }
+
+/**
+ * A temporary HashObject is needed in a lot of places, this HashObject is then
+ * applied to persistent-merkle-tree, it'll make a copy so it's safe to mutate it after that.
+ * It means that we could use a shared HashObject instead of having to always allocate
+ * a new one to save memory. This temporary HashObject is always allocated by cloneHashObject()
+ * or newHashObject() below.
+ **/
+const sharedHashObject: HashObject = {
+  h0: 0,
+  h1: 0,
+  h2: 0,
+  h3: 0,
+  h4: 0,
+  h5: 0,
+  h6: 0,
+  h7: 0,
+};
+
+/**
+ * Clone a hash object using sharedHashObject, after doing this we usually
+ * apply HashObject to the Tree which make a copy there so it's safe to mutate
+ * this HashObject after that.
+ **/
+export function cloneHashObject(hashObject: HashObject): HashObject {
+  sharedHashObject.h0 = hashObject.h0;
+  sharedHashObject.h1 = hashObject.h1;
+  sharedHashObject.h2 = hashObject.h2;
+  sharedHashObject.h3 = hashObject.h3;
+  sharedHashObject.h4 = hashObject.h4;
+  sharedHashObject.h5 = hashObject.h5;
+  sharedHashObject.h6 = hashObject.h6;
+  sharedHashObject.h7 = hashObject.h7;
+  return sharedHashObject;
+}
+
+/**
+ * Reset and return sharedHashObject, after doing this we usually
+ * apply HashObject to the Tree which make a copy there so it's safe to mutate
+ * this HashObject after that.
+ **/
+export function newHashObject(): HashObject {
+  sharedHashObject.h0 = 0;
+  sharedHashObject.h1 = 0;
+  sharedHashObject.h2 = 0;
+  sharedHashObject.h3 = 0;
+  sharedHashObject.h4 = 0;
+  sharedHashObject.h5 = 0;
+  sharedHashObject.h6 = 0;
+  sharedHashObject.h7 = 0;
+  return sharedHashObject;
+}