Add feature flag --receipt-compaction-enabled so this can be disabled

Signed-off-by: Jason Frame <[email protected]>

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -17,6 +17,7 @@
 package org.hyperledger.besu.cli.options.stable;
 
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
@@ -61,6 +62,12 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
       arity = "1")
   private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 
+  @Option(
+      names = "--receipt-compaction-enabled",
+      description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE}).",
+      arity = "1")
+  private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED;
+
   @CommandLine.ArgGroup(validate = false)
   private final DataStorageOptions.Unstable unstableOptions = new Unstable();
 
@@ -149,6 +156,7 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
     final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
     dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
     dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
+    dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled();
     dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled =
         domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
     dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
@@ -164,6 +172,7 @@ public DataStorageConfiguration toDomainObject() {
     return ImmutableDataStorageConfiguration.builder()
         .dataStorageFormat(dataStorageFormat)
         .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
+        .receiptCompactionEnabled(receiptCompactionEnabled)
         .unstable(
             ImmutableDataStorageConfiguration.Unstable.builder()
                 .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -592,7 +592,10 @@ public BesuController build() {
         storageProvider.createWorldStateStorage(dataStorageConfiguration);
 
     final BlockchainStorage blockchainStorage =
-        storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage);
+        storageProvider.createBlockchainStorage(
+            protocolSchedule,
+            variablesStorage,
+            dataStorageConfiguration.getReceiptCompactionEnabled());
 
     final MutableBlockchain blockchain =
         DefaultBlockchain.createMutable(

besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java

@@ -110,6 +110,24 @@ public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() {
         "false");
   }
 
+  @Test
+  public void receiptCompactionCanBeEnabled() {
+    internalTestSuccess(
+        dataStorageConfiguration ->
+            assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(true),
+        "--receipt-compaction-enabled",
+        "true");
+  }
+
+  @Test
+  public void receiptCompactionCanBeDisabled() {
+    internalTestSuccess(
+        dataStorageConfiguration ->
+            assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(false),
+        "--receipt-compaction-enabled",
+        "false");
+  }
+
   @Override
   protected DataStorageConfiguration createDefaultDomainObject() {
     return DataStorageConfiguration.DEFAULT_CONFIG;

besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.controller;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
@@ -118,12 +119,13 @@ public void setup() {
     when(ethashConfigOptions.getFixedDifficulty()).thenReturn(OptionalLong.empty());
     when(storageProvider.getStorageBySegmentIdentifier(any()))
         .thenReturn(new InMemoryKeyValueStorage());
-    when(storageProvider.createBlockchainStorage(any(), any()))
+    when(storageProvider.createBlockchainStorage(any(), any(), anyBoolean()))
         .thenReturn(
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 new InMemoryKeyValueStorage(),
                 new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-                new MainnetBlockHeaderFunctions()));
+                new MainnetBlockHeaderFunctions(),
+                false));
     when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1);
     when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1);
     when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1);

besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java

@@ -17,6 +17,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
@@ -127,12 +128,13 @@ public void setup() {
     when(genesisConfigOptions.getTerminalBlockHash()).thenReturn(Optional.of(Hash.ZERO));
     lenient().when(genesisConfigOptions.getTerminalBlockNumber()).thenReturn(OptionalLong.of(1L));
     lenient()
-        .when(storageProvider.createBlockchainStorage(any(), any()))
+        .when(storageProvider.createBlockchainStorage(any(), any(), anyBoolean()))
         .thenReturn(
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 new InMemoryKeyValueStorage(),
                 new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-                new MainnetBlockHeaderFunctions()));
+                new MainnetBlockHeaderFunctions(),
+                false));
     lenient()
         .when(storageProvider.getStorageBySegmentIdentifier(any()))
         .thenReturn(new InMemoryKeyValueStorage());

besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java

@@ -17,6 +17,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -107,12 +108,13 @@ public void setup() {
     lenient().when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions);
     lenient().when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions);
     lenient()
-        .when(storageProvider.createBlockchainStorage(any(), any()))
+        .when(storageProvider.createBlockchainStorage(any(), any(), anyBoolean()))
         .thenReturn(
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 new InMemoryKeyValueStorage(),
                 new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-                new MainnetBlockHeaderFunctions()));
+                new MainnetBlockHeaderFunctions(),
+                false));
     lenient()
         .when(
             storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_FOREST_CONFIG))

besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java

@@ -121,7 +121,8 @@ public void setUp() {
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 new InMemoryKeyValueStorage(),
                 new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-                new MainnetBlockHeaderFunctions()),
+                new MainnetBlockHeaderFunctions(),
+                false),
             new NoOpMetricsSystem(),
             0);
 

besu/src/test/resources/everything_config.toml

@@ -215,6 +215,7 @@ ethstats-cacert-file="./root.cert"
 # Data storage
 data-storage-format="BONSAI"
 bonsai-historical-block-limit=512
+receipt-compaction-enabled=true
 
 # feature flags
 Xsecp256k1-native-enabled=false

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java

@@ -311,7 +311,7 @@ private void backupChainData() throws IOException {
         bodyWriter.writeBytes(bodyOutput.encoded().toArrayUnsafe());
 
         final BytesValueRLPOutput receiptsOutput = new BytesValueRLPOutput();
-        receiptsOutput.writeList(receipts.get(), TransactionReceipt::writeToForStorage);
+        receiptsOutput.writeList(receipts.get(), (r, rlpOut) -> r.writeToForStorage(rlpOut, false));
         receiptsWriter.writeBytes(receiptsOutput.encoded().toArrayUnsafe());
 
         backupStatus.storedBlock = blockNumber;

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java

@@ -63,7 +63,8 @@ public class NewBlockHeadersSubscriptionServiceTest {
       new KeyValueStoragePrefixedKeyBlockchainStorage(
           new InMemoryKeyValueStorage(),
           new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-          new MainnetBlockHeaderFunctions());
+          new MainnetBlockHeaderFunctions(),
+          false);
   private final Block genesisBlock = gen.genesisBlock();
   private final MutableBlockchain blockchain =
       DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, new NoOpMetricsSystem(), 0);

ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java

@@ -162,7 +162,8 @@ public void setup() {
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 new InMemoryKeyValueStorage(),
                 new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-                new MainnetBlockHeaderFunctions()),
+                new MainnetBlockHeaderFunctions(),
+                false),
             new NoOpMetricsSystem(),
             0);
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java

@@ -174,8 +174,8 @@ public void writeToForNetwork(final RLPOutput out) {
     writeTo(out, false, false);
   }
 
-  public void writeToForStorage(final RLPOutput out) {
-    writeTo(out, true, true);
+  public void writeToForStorage(final RLPOutput out, final boolean compacted) {
+    writeTo(out, true, compacted);
   }
 
   @VisibleForTesting

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java

@@ -32,7 +32,9 @@ public interface StorageProvider extends Closeable {
   VariablesStorage createVariablesStorage();
 
   BlockchainStorage createBlockchainStorage(
-      ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage);
+      ProtocolSchedule protocolSchedule,
+      VariablesStorage variablesStorage,
+      final boolean receiptCompaction);
 
   WorldStateStorage createWorldStateStorage(DataStorageConfiguration dataStorageFormat);
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java

@@ -59,14 +59,17 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt
   final KeyValueStorage blockchainStorage;
   final VariablesStorage variablesStorage;
   final BlockHeaderFunctions blockHeaderFunctions;
+  final boolean receiptCompaction;
 
   public KeyValueStoragePrefixedKeyBlockchainStorage(
       final KeyValueStorage blockchainStorage,
       final VariablesStorage variablesStorage,
-      final BlockHeaderFunctions blockHeaderFunctions) {
+      final BlockHeaderFunctions blockHeaderFunctions,
+      final boolean receiptCompaction) {
     this.blockchainStorage = blockchainStorage;
     this.variablesStorage = variablesStorage;
     this.blockHeaderFunctions = blockHeaderFunctions;
+    this.receiptCompaction = receiptCompaction;
     migrateVariables();
   }
 
@@ -125,7 +128,8 @@ public Optional<TransactionLocation> getTransactionLocation(final Hash transacti
 
   @Override
   public Updater updater() {
-    return new Updater(blockchainStorage.startTransaction(), variablesStorage.updater());
+    return new Updater(
+        blockchainStorage.startTransaction(), variablesStorage.updater(), receiptCompaction);
   }
 
   private List<TransactionReceipt> rlpDecodeTransactionReceipts(final Bytes bytes) {
@@ -253,12 +257,15 @@ public static class Updater implements BlockchainStorage.Updater {
 
     private final KeyValueStorageTransaction blockchainTransaction;
     private final VariablesStorage.Updater variablesUpdater;
+    private final boolean receiptCompaction;
 
     Updater(
         final KeyValueStorageTransaction blockchainTransaction,
-        final VariablesStorage.Updater variablesUpdater) {
+        final VariablesStorage.Updater variablesUpdater,
+        final boolean receiptCompaction) {
       this.blockchainTransaction = blockchainTransaction;
       this.variablesUpdater = variablesUpdater;
+      this.receiptCompaction = receiptCompaction;
     }
 
     @Override
@@ -365,7 +372,10 @@ private void remove(final Bytes prefix, final Bytes key) {
     }
 
     private Bytes rlpEncode(final List<TransactionReceipt> receipts) {
-      return RLP.encode(o -> o.writeList(receipts, TransactionReceipt::writeToForStorage));
+      return RLP.encode(
+          o ->
+              o.writeList(
+                  receipts, (r, rlpOutput) -> r.writeToForStorage(rlpOutput, receiptCompaction)));
     }
 
     private void removeVariables() {

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java

@@ -68,11 +68,14 @@ public VariablesStorage createVariablesStorage() {
 
   @Override
   public BlockchainStorage createBlockchainStorage(
-      final ProtocolSchedule protocolSchedule, final VariablesStorage variablesStorage) {
+      final ProtocolSchedule protocolSchedule,
+      final VariablesStorage variablesStorage,
+      final boolean receiptCompaction) {
     return new KeyValueStoragePrefixedKeyBlockchainStorage(
         getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.BLOCKCHAIN),
         variablesStorage,
-        ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
+        ScheduleBasedBlockHeaderFunctions.create(protocolSchedule),
+        receiptCompaction);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java

@@ -23,6 +23,7 @@
 public interface DataStorageConfiguration {
 
   long DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD = 512;
+  boolean DEFAULT_RECEIPT_COMPACTION_ENABLED = true;
 
   DataStorageConfiguration DEFAULT_CONFIG =
       ImmutableDataStorageConfiguration.builder()
@@ -48,6 +49,11 @@ public interface DataStorageConfiguration {
 
   Long getBonsaiMaxLayersToLoad();
 
+  @Value.Default
+  default boolean getReceiptCompactionEnabled() {
+    return DEFAULT_RECEIPT_COMPACTION_ENABLED;
+  }
+
   @Value.Default
   default Unstable getUnstable() {
     return Unstable.DEFAULT;

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java

@@ -65,7 +65,8 @@ private ExecutionContextTestFixture(
             new KeyValueStoragePrefixedKeyBlockchainStorage(
                 blockchainKeyValueStorage,
                 new VariablesKeyValueStorage(variablesKeyValueStorage),
-                new MainnetBlockHeaderFunctions()),
+                new MainnetBlockHeaderFunctions(),
+                false),
             new NoOpMetricsSystem(),
             0);
     this.stateArchive = createInMemoryWorldStateArchive();

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java

@@ -72,7 +72,7 @@ public static MutableBlockchain createInMemoryBlockchain(
     return DefaultBlockchain.createMutable(
         genesisBlock,
         new KeyValueStoragePrefixedKeyBlockchainStorage(
-            keyValueStorage, variablesStorage, blockHeaderFunctions),
+            keyValueStorage, variablesStorage, blockHeaderFunctions, false),
         new NoOpMetricsSystem(),
         0);
   }

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java

@@ -41,7 +41,8 @@ public void singleChainPruning() {
         new KeyValueStoragePrefixedKeyBlockchainStorage(
             new InMemoryKeyValueStorage(),
             new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-            new MainnetBlockHeaderFunctions());
+            new MainnetBlockHeaderFunctions(),
+            false);
     final ChainDataPruner chainDataPruner =
         new ChainDataPruner(
             blockchainStorage,
@@ -79,7 +80,8 @@ public void forkPruning() {
         new KeyValueStoragePrefixedKeyBlockchainStorage(
             new InMemoryKeyValueStorage(),
             new VariablesKeyValueStorage(new InMemoryKeyValueStorage()),
-            new MainnetBlockHeaderFunctions());
+            new MainnetBlockHeaderFunctions(),
+            false);
     final ChainDataPruner chainDataPruner =
         new ChainDataPruner(
             blockchainStorage,

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java

@@ -1055,7 +1055,8 @@ private BlockchainStorage createStorage(
     return new KeyValueStoragePrefixedKeyBlockchainStorage(
         kvStoreChain,
         new VariablesKeyValueStorage(kvStorageVariables),
-        new MainnetBlockHeaderFunctions());
+        new MainnetBlockHeaderFunctions(),
+        false);
   }
 
   private DefaultBlockchain createMutableBlockchain(

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java

@@ -62,7 +62,7 @@ public void migrationToVariablesStorage() {
 
     final var blockchainStorage =
         new KeyValueStoragePrefixedKeyBlockchainStorage(
-            kvBlockchain, variablesStorage, blockHeaderFunctions);
+            kvBlockchain, variablesStorage, blockHeaderFunctions, false);
 
     assertNoVariablesInStorage(kvBlockchain);
     assertVariablesPresentInVariablesStorage(kvVariables, variableValues);
@@ -80,7 +80,7 @@ public void migrationToVariablesStorageWhenSomeVariablesDoNotExist() {
 
     final var blockchainStorage =
         new KeyValueStoragePrefixedKeyBlockchainStorage(
-            kvBlockchain, variablesStorage, blockHeaderFunctions);
+            kvBlockchain, variablesStorage, blockHeaderFunctions, false);
 
     assertNoVariablesInStorage(kvBlockchain);
     assertVariablesPresentInVariablesStorage(kvVariables, variableValues);
@@ -96,7 +96,7 @@ public void doesNothingIfVariablesAlreadyMigrated() {
 
     final var blockchainStorage =
         new KeyValueStoragePrefixedKeyBlockchainStorage(
-            kvBlockchain, variablesStorage, blockHeaderFunctions);
+            kvBlockchain, variablesStorage, blockHeaderFunctions, false);
 
     assertNoVariablesInStorage(kvBlockchain);
     assertVariablesPresentInVariablesStorage(kvVariables, variableValues);
@@ -114,6 +114,6 @@ public void failIfInconsistencyDetectedDuringVariablesMigration() {
         IllegalStateException.class,
         () ->
             new KeyValueStoragePrefixedKeyBlockchainStorage(
-                kvBlockchain, variablesStorage, blockHeaderFunctions));
+                kvBlockchain, variablesStorage, blockHeaderFunctions, false));
   }
 }