feat: add pushToField to Store and World

packages/store/gas-report.txt

@@ -14,9 +14,9 @@
 (test/Gas.t.sol) | pass abi encoded bytes to external contract [someContract.doSomethingWithBytes(abiEncoded)]: 6554
 (test/Gas.t.sol) | pass custom encoded bytes to external contract [someContract.doSomethingWithBytes(customEncoded)]: 1381
 (test/Mixed.t.sol) | store Mixed struct in storage (native solidity) [testMixed = mixed]: 92050
-(test/Mixed.t.sol) | register Mixed schema [Mixed.registerSchema()]: 35047
-(test/Mixed.t.sol) | set record in Mixed [Mixed.set({ key: key, u32: 1, u128: 2, a32: a32, s: s })]: 111032
-(test/Mixed.t.sol) | get record from Mixed [MixedData memory mixed = Mixed.get(key)]: 13167
+(test/Mixed.t.sol) | register Mixed schema [Mixed.registerSchema()]: 35069
+(test/Mixed.t.sol) | set record in Mixed [Mixed.set({ key: key, u32: 1, u128: 2, a32: a32, s: s })]: 111076
+(test/Mixed.t.sol) | get record from Mixed [MixedData memory mixed = Mixed.get(key)]: 13189
 (test/PackedCounter.t.sol) | get value at index of PackedCounter [packedCounter.atIndex(3)]: 261
 (test/PackedCounter.t.sol) | set value at index of PackedCounter [packedCounter = packedCounter.setAtIndex(2, 5)]: 799
 (test/PackedCounter.t.sol) | pack uint16 array into PackedCounter [PackedCounter packedCounter = PackedCounterLib.pack(counters)]: 2152
@@ -47,39 +47,41 @@
 (test/StoreCore.t.sol) | access non-existing record [bytes memory data1 = StoreCore.getRecord(table, key)]: 7091
 (test/StoreCore.t.sol) | access static field of non-existing record [bytes memory data2 = StoreCore.getField(table, key, 0)]: 2750
 (test/StoreCore.t.sol) | access dynamic field of non-existing record [bytes memory data3 = StoreCore.getField(table, key, 1)]: 3364
-(test/StoreCore.t.sol) | delete record (complex data, 3 slots) [StoreCore.deleteRecord(table, key)]: 10287
+(test/StoreCore.t.sol) | delete record (complex data, 3 slots) [StoreCore.deleteRecord(table, key)]: 10299
 (test/StoreCore.t.sol) | Check for existence of table (existent) [StoreCore.hasTable(table)]: 941
 (test/StoreCore.t.sol) | check for existence of table (non-existent) [StoreCore.hasTable(table2)]: 2963
-(test/StoreCore.t.sol) | register subscriber [StoreCore.registerStoreHook(table, subscriber)]: 66016
-(test/StoreCore.t.sol) | set record on table with subscriber [StoreCore.setRecord(table, key, data)]: 72428
-(test/StoreCore.t.sol) | set static field on table with subscriber [StoreCore.setField(table, key, 0, data)]: 28127
-(test/StoreCore.t.sol) | delete record on table with subscriber [StoreCore.deleteRecord(table, key)]: 23007
-(test/StoreCore.t.sol) | register subscriber [StoreCore.registerStoreHook(table, subscriber)]: 66016
-(test/StoreCore.t.sol) | set (dynamic) record on table with subscriber [StoreCore.setRecord(table, key, data)]: 165835
-(test/StoreCore.t.sol) | set (dynamic) field on table with subscriber [StoreCore.setField(table, key, 1, arrayDataBytes)]: 31173
-(test/StoreCore.t.sol) | delete (dynamic) record on table with subscriber [StoreCore.deleteRecord(table, key)]: 24484
+(test/StoreCore.t.sol) | register subscriber [StoreCore.registerStoreHook(table, subscriber)]: 66063
+(test/StoreCore.t.sol) | set record on table with subscriber [StoreCore.setRecord(table, key, data)]: 72464
+(test/StoreCore.t.sol) | set static field on table with subscriber [StoreCore.setField(table, key, 0, data)]: 28163
+(test/StoreCore.t.sol) | delete record on table with subscriber [StoreCore.deleteRecord(table, key)]: 23043
+(test/StoreCore.t.sol) | register subscriber [StoreCore.registerStoreHook(table, subscriber)]: 66063
+(test/StoreCore.t.sol) | set (dynamic) record on table with subscriber [StoreCore.setRecord(table, key, data)]: 165871
+(test/StoreCore.t.sol) | set (dynamic) field on table with subscriber [StoreCore.setField(table, key, 1, arrayDataBytes)]: 31231
+(test/StoreCore.t.sol) | delete (dynamic) record on table with subscriber [StoreCore.deleteRecord(table, key)]: 24520
+(test/StoreCore.t.sol) | push to field (1 slot, 1 uint32 item) [StoreCore.pushToField(table, key, 1, secondDataToPush)]: 16253
+(test/StoreCore.t.sol) | push to field (2 slots, 10 uint32 items) [StoreCore.pushToField(table, key, 2, thirdDataToPush)]: 38966
 (test/StoreCore.t.sol) | StoreCore: register schema [StoreCore.registerSchema(table, schema)]: 29967
 (test/StoreCore.t.sol) | StoreCore: get schema (warm) [Schema loadedSchema = StoreCore.getSchema(table)]: 909
-(test/StoreCore.t.sol) | set complex record with dynamic data (4 slots) [StoreCore.setRecord(table, key, data)]: 106905
+(test/StoreCore.t.sol) | set complex record with dynamic data (4 slots) [StoreCore.setRecord(table, key, data)]: 106917
 (test/StoreCore.t.sol) | get complex record with dynamic data (4 slots) [bytes memory loadedData = StoreCore.getRecord(table, key)]: 6222
 (test/StoreCore.t.sol) | compare: Set complex record with dynamic data using native solidity [testStruct = _testStruct]: 116842
 (test/StoreCore.t.sol) | compare: Set complex record with dynamic data using abi.encode [testMapping[1234] = abi.encode(_testStruct)]: 267376
-(test/StoreCore.t.sol) | set dynamic length of dynamic index 0 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 0, 10)]: 23591
-(test/StoreCore.t.sol) | set dynamic length of dynamic index 1 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 1, 99)]: 1708
-(test/StoreCore.t.sol) | reduce dynamic length of dynamic index 0 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 0, 5)]: 1697
-(test/StoreCore.t.sol) | set static field (1 slot) [StoreCore.setField(table, key, 0, abi.encodePacked(firstDataBytes))]: 37316
+(test/StoreCore.t.sol) | set dynamic length of dynamic index 0 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 0, 10)]: 23602
+(test/StoreCore.t.sol) | set dynamic length of dynamic index 1 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 1, 99)]: 1719
+(test/StoreCore.t.sol) | reduce dynamic length of dynamic index 0 [StoreCoreInternal._setDynamicDataLengthAtIndex(table, key, 0, 5)]: 1708
+(test/StoreCore.t.sol) | set static field (1 slot) [StoreCore.setField(table, key, 0, abi.encodePacked(firstDataBytes))]: 37328
 (test/StoreCore.t.sol) | get static field (1 slot) [bytes memory loadedData = StoreCore.getField(table, key, 0)]: 2755
-(test/StoreCore.t.sol) | set static field (overlap 2 slot) [StoreCore.setField(table, key, 1, abi.encodePacked(secondDataBytes))]: 32326
+(test/StoreCore.t.sol) | set static field (overlap 2 slot) [StoreCore.setField(table, key, 1, abi.encodePacked(secondDataBytes))]: 32338
 (test/StoreCore.t.sol) | get static field (overlap 2 slot) [loadedData = StoreCore.getField(table, key, 1)]: 3642
-(test/StoreCore.t.sol) | set dynamic field (1 slot, first dynamic field) [StoreCore.setField(table, key, 2, thirdDataBytes)]: 54608
+(test/StoreCore.t.sol) | set dynamic field (1 slot, first dynamic field) [StoreCore.setField(table, key, 2, thirdDataBytes)]: 54631
 (test/StoreCore.t.sol) | get dynamic field (1 slot, first dynamic field) [loadedData = StoreCore.getField(table, key, 2)]: 3589
-(test/StoreCore.t.sol) | set dynamic field (1 slot, second dynamic field) [StoreCore.setField(table, key, 3, fourthDataBytes)]: 34740
+(test/StoreCore.t.sol) | set dynamic field (1 slot, second dynamic field) [StoreCore.setField(table, key, 3, fourthDataBytes)]: 34763
 (test/StoreCore.t.sol) | get dynamic field (1 slot, second dynamic field) [loadedData = StoreCore.getField(table, key, 3)]: 3606
-(test/StoreCore.t.sol) | set static record (1 slot) [StoreCore.setRecord(table, key, data)]: 36596
+(test/StoreCore.t.sol) | set static record (1 slot) [StoreCore.setRecord(table, key, data)]: 36608
 (test/StoreCore.t.sol) | get static record (1 slot) [bytes memory loadedData = StoreCore.getRecord(table, key, schema)]: 1324
-(test/StoreCore.t.sol) | set static record (2 slots) [StoreCore.setRecord(table, key, data)]: 59161
+(test/StoreCore.t.sol) | set static record (2 slots) [StoreCore.setRecord(table, key, data)]: 59173
 (test/StoreCore.t.sol) | get static record (2 slots) [bytes memory loadedData = StoreCore.getRecord(table, key, schema)]: 1569
-(test/StoreCore.t.sol) | StoreCore: set table metadata [StoreCore.setMetadata(table, tableName, fieldNames)]: 250520
+(test/StoreCore.t.sol) | StoreCore: set table metadata [StoreCore.setMetadata(table, tableName, fieldNames)]: 250544
 (test/StoreMetadata.t.sol) | set record in StoreMetadataTable [StoreMetadata.set({ tableId: tableId, tableName: tableName, abiEncodedFieldNames: abi.encode(fieldNames) })]: 249248
 (test/StoreMetadata.t.sol) | get record from StoreMetadataTable [StoreMetadataData memory metadata = StoreMetadata.get(tableId)]: 11934
 (test/StoreSwitch.t.sol) | check if delegatecall [isDelegate = StoreSwitch.isDelegateCall()]: 693

packages/store/src/IStore.sol

@@ -33,6 +33,14 @@ interface IStore {
     bytes calldata data
   ) external;
 
+  // Push encoded items to the dynamic field at schema index
+  function pushToField(
+    uint256 table,
+    bytes32[] calldata key,
+    uint8 schemaIndex,
+    bytes calldata dataToPush
+  ) external;
+
   // Register hooks to be called when a record or field is set or deleted
   function registerStoreHook(uint256 table, IStoreHook hooks) external;
 

packages/store/src/StoreCore.sol

@@ -23,8 +23,8 @@ library StoreCore {
   error StoreCore_TableNotFound(uint256 table);
   error StoreCore_NotImplemented();
   error StoreCore_InvalidDataLength(uint256 expected, uint256 received);
-  error StoreCore_NoDynamicField();
   error StoreCore_InvalidFieldNamesLength(uint256 expected, uint256 received);
+  error StoreCore_NotDynamicField();
 
   /**
    * Initialize internal tables.
@@ -202,7 +202,6 @@ library StoreCore {
   }
 
   function deleteRecord(uint256 table, bytes32[] memory key) internal {
-    // Get schema for this table
     Schema schema = getSchema(table);
 
     // Emit event to notify indexers
@@ -227,6 +226,37 @@ library StoreCore {
     Storage.store({ storagePointer: dynamicDataLengthLocation, data: bytes32(0) });
   }
 
+  function pushToField(
+    uint256 table,
+    bytes32[] memory key,
+    uint8 schemaIndex,
+    bytes memory dataToPush
+  ) internal {
+    Schema schema = getSchema(table);
+
+    if (schemaIndex < schema.numStaticFields()) {
+      revert StoreCore_NotDynamicField();
+    }
+
+    // TODO add push-specific event and hook to avoid the storage read? (https://github.com/latticexyz/mud/issues/444)
+    bytes memory fullData = abi.encodePacked(
+      StoreCoreInternal._getDynamicField(table, key, schemaIndex, schema),
+      dataToPush
+    );
+
+    // Emit event to notify indexers
+    emit StoreSetField(table, key, schemaIndex, fullData);
+
+    // Call onSetField hooks (before actually modifying the state, so observers have access to the previous state if needed)
+    address[] memory hooks = Hooks.get(bytes32(table));
+    for (uint256 i = 0; i < hooks.length; i++) {
+      IStoreHook hook = IStoreHook(hooks[i]);
+      hook.onSetField(table, key, schemaIndex, fullData);
+    }
+
+    StoreCoreInternal._pushToDynamicField(table, key, schema, schemaIndex, dataToPush);
+  }
+
   /************************************************************************
    *
    *    GET DATA
@@ -389,6 +419,34 @@ library StoreCoreInternal {
     Storage.store({ storagePointer: dynamicDataLocation, data: data });
   }
 
+  function _pushToDynamicField(
+    uint256 table,
+    bytes32[] memory key,
+    Schema schema,
+    uint8 schemaIndex,
+    bytes memory dataToPush
+  ) internal {
+    uint8 dynamicSchemaIndex = schemaIndex - schema.numStaticFields();
+
+    // Load dynamic data length from storage
+    uint256 dynamicSchemaLengthSlot = _getDynamicDataLengthLocation(table, key);
+    PackedCounter encodedLengths = PackedCounter.wrap(Storage.load({ storagePointer: dynamicSchemaLengthSlot }));
+
+    // Update the encoded length
+    uint256 oldFieldLength = encodedLengths.atIndex(dynamicSchemaIndex);
+    encodedLengths = encodedLengths.setAtIndex(dynamicSchemaIndex, oldFieldLength + dataToPush.length);
+
+    // Set the new length
+    Storage.store({ storagePointer: dynamicSchemaLengthSlot, data: encodedLengths.unwrap() });
+
+    // Append `dataToPush` to the end of the data in storage
+    uint256 dynamicDataLocation = _getDynamicDataLocation(table, key, dynamicSchemaIndex);
+    dynamicDataLocation += oldFieldLength / 32;
+    // offset for new data (old data never has an offset because each dynamic field starts at a different storage slot)
+    uint256 offset = oldFieldLength % 32;
+    Storage.store({ storagePointer: dynamicDataLocation, offset: offset, data: dataToPush });
+  }
+
   /************************************************************************
    *
    *    GET DATA

packages/store/src/StoreSwitch.sol

@@ -75,6 +75,19 @@ library StoreSwitch {
     }
   }
 
+  function pushToField(
+    uint256 table,
+    bytes32[] memory key,
+    uint8 fieldIndex,
+    bytes memory dataToPush
+  ) internal {
+    if (isDelegateCall()) {
+      StoreCore.pushToField(table, key, fieldIndex, dataToPush);
+    } else {
+      IStore(msg.sender).pushToField(table, key, fieldIndex, dataToPush);
+    }
+  }
+
   function deleteRecord(uint256 table, bytes32[] memory key) internal {
     if (isDelegateCall()) {
       StoreCore.deleteRecord(table, key);

packages/store/src/StoreView.sol

@@ -51,6 +51,18 @@ contract StoreView is Store {
     revert StoreView_NotImplemented();
   }
 
+  /**
+   * Not implemented in StoreView
+   */
+  function pushToField(
+    uint256,
+    bytes32[] calldata,
+    uint8,
+    bytes calldata
+  ) public virtual {
+    revert StoreView_NotImplemented();
+  }
+
   /**
    * Not implemented in StoreView
    */

packages/store/test/StoreCore.t.sol

@@ -44,6 +44,16 @@ contract StoreCoreTest is Test, StoreView {
     StoreCore.setField(table, key, schemaIndex, data);
   }
 
+  // Expose an external pushToField function for testing purposes of indexers (see testHooks)
+  function pushToField(
+    uint256 table,
+    bytes32[] calldata key,
+    uint8 schemaIndex,
+    bytes calldata dataToPush
+  ) public override {
+    StoreCore.pushToField(table, key, schemaIndex, dataToPush);
+  }
+
   // Expose an external deleteRecord function for testing purposes of indexers (see testHooks)
   function deleteRecord(uint256 table, bytes32[] calldata key) public override {
     StoreCore.deleteRecord(table, key);
@@ -539,6 +549,111 @@ contract StoreCoreTest is Test, StoreView {
     assertEq(keccak256(loadedData), keccak256(new bytes(schema.staticDataLength())));
   }
 
+  function testPushToField() public {
+    uint256 table = uint256(keccak256("some.table"));
+
+    {
+      // Register table's schema
+      Schema schema = SchemaLib.encode(SchemaType.UINT256, SchemaType.UINT32_ARRAY, SchemaType.UINT32_ARRAY);
+      StoreCore.registerSchema(table, schema);
+    }
+
+    // Create key
+    bytes32[] memory key = new bytes32[](1);
+    key[0] = bytes32("some.key");
+
+    // Create data
+    bytes32 firstDataBytes = keccak256("some data");
+    bytes memory secondDataBytes;
+    {
+      uint32[] memory secondData = new uint32[](2);
+      secondData[0] = 0x11121314;
+      secondData[1] = 0x15161718;
+      secondDataBytes = EncodeArray.encode(secondData);
+    }
+    bytes memory thirdDataBytes;
+    {
+      uint32[] memory thirdData = new uint32[](3);
+      thirdData[0] = 0x191a1b1c;
+      thirdData[1] = 0x1d1e1f20;
+      thirdData[2] = 0x21222324;
+      thirdDataBytes = EncodeArray.encode(thirdData);
+    }
+
+    // Set fields
+    StoreCore.setField(table, key, 0, abi.encodePacked(firstDataBytes));
+    StoreCore.setField(table, key, 1, secondDataBytes);
+    // Initialize a field with push
+    StoreCore.pushToField(table, key, 2, thirdDataBytes);
+
+    // Create data to push
+    bytes memory secondDataToPush;
+    {
+      uint32[] memory secondData = new uint32[](1);
+      secondData[0] = 0x25262728;
+      secondDataToPush = EncodeArray.encode(secondData);
+    }
+    bytes memory newSecondDataBytes = abi.encodePacked(secondDataBytes, secondDataToPush);
+
+    // Expect a StoreSetField event to be emitted
+    vm.expectEmit(true, true, true, true);
+    emit StoreSetField(table, key, 1, newSecondDataBytes);
+
+    // Push to second field
+    // !gasreport push to field (1 slot, 1 uint32 item)
+    StoreCore.pushToField(table, key, 1, secondDataToPush);
+
+    // Get second field
+    bytes memory loadedData = StoreCore.getField(table, key, 1);
+
+    // Verify loaded data is correct
+    assertEq(SliceLib.fromBytes(loadedData).decodeArray_uint32().length, 2 + 1);
+    assertEq(loadedData.length, newSecondDataBytes.length);
+    assertEq(loadedData, newSecondDataBytes);
+
+    // Verify none of the other fields were impacted
+    assertEq(bytes32(StoreCore.getField(table, key, 0)), firstDataBytes);
+    assertEq(StoreCore.getField(table, key, 2), thirdDataBytes);
+
+    // Create data to push
+    bytes memory thirdDataToPush;
+    {
+      uint32[] memory thirdData = new uint32[](10);
+      thirdData[0] = 0x12345678;
+      thirdData[1] = 0x9abcdef0;
+      thirdData[2] = 0x12345678;
+      thirdData[3] = 0x9abcdef0;
+      thirdData[4] = 0x12345678;
+      thirdData[5] = 0x9abcdef0;
+      thirdData[6] = 0x12345678;
+      thirdData[7] = 0x9abcdef0;
+      thirdData[8] = 0x12345678;
+      thirdData[9] = 0x9abcdef0;
+      thirdDataToPush = EncodeArray.encode(thirdData);
+    }
+    bytes memory newThirdDataBytes = abi.encodePacked(thirdDataBytes, thirdDataToPush);
+
+    // Expect a StoreSetField event to be emitted
+    vm.expectEmit(true, true, true, true);
+    emit StoreSetField(table, key, 2, newThirdDataBytes);
+
+    // Push to third field
+    // !gasreport push to field (2 slots, 10 uint32 items)
+    StoreCore.pushToField(table, key, 2, thirdDataToPush);
+
+    // Get third field
+    loadedData = StoreCore.getField(table, key, 2);
+
+    // Verify loaded data is correct
+    assertEq(SliceLib.fromBytes(loadedData).decodeArray_uint32().length, 3 + 10);
+    assertEq(loadedData.length, newThirdDataBytes.length);
+    assertEq(loadedData, newThirdDataBytes);
+
+    // Verify none of the other fields were impacted
+    assertEq(bytes32(StoreCore.getField(table, key, 0)), firstDataBytes);
+    assertEq(StoreCore.getField(table, key, 1), newSecondDataBytes);
+  }
+
   function testAccessEmptyData() public {
     uint256 table = uint256(keccak256("some.table"));
     Schema schema = SchemaLib.encode(SchemaType.UINT32, SchemaType.UINT32_ARRAY);