new(cli): Introduce eofwrap tool

pyproject.toml

@@ -90,6 +90,7 @@ checkfixtures = "cli.check_fixtures:check_fixtures"
 consume = "cli.pytest_commands.consume:consume"
 genindex = "cli.gen_index:generate_fixtures_index_cli"
 gentest = "cli.gentest:generate"
+eofwrap = "cli.eofwrap:eof_wrap"
 pyspelling_soft_fail = "cli.tox_helpers:pyspelling"
 markdownlintcli2_soft_fail = "cli.tox_helpers:markdownlint"
 order_fixtures = "cli.order_fixtures:order_fixtures"

src/cli/eofwrap.py

@@ -0,0 +1,346 @@
+"""
+Generate a JSON blockchain test from an existing JSON blockchain test by wrapping its prestate code
+in EOF wherever possible.
+
+Example Usage:
+
+1. Wrap tests
+
+    ```console
+    eofwrap <input_dir/file_path> <output_dir_path>
+    ```
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, no_type_check
+
+import click
+
+from cli.evm_bytes import OpcodeWithOperands, process_evm_bytes
+from ethereum_clis import CLINotFoundInPath
+from ethereum_clis.clis.evmone import EvmOneTransitionTool
+from ethereum_test_base_types.base_types import Bytes
+from ethereum_test_base_types.conversions import to_hex
+from ethereum_test_fixtures.blockchain import FixtureBlock, InvalidFixtureBlock
+from ethereum_test_fixtures.file import BaseFixturesRootModel, BlockchainFixtures
+from ethereum_test_forks.forks.forks import Osaka
+from ethereum_test_specs.blockchain import Block, BlockchainFixture, BlockchainTest
+from ethereum_test_specs.debugging import print_traces
+from ethereum_test_specs.eof import EOFParse
+from ethereum_test_tools import Opcodes as Op
+from ethereum_test_types import Transaction
+from ethereum_test_types.eof.v1 import Container
+from ethereum_test_types.types import Environment
+from ethereum_test_vm.bytecode import Bytecode
+
+
+@click.command()
+@click.argument("input", type=click.Path(exists=True, dir_okay=True, file_okay=True))
+@click.argument("output_dir", type=click.Path(dir_okay=True, file_okay=False))
+@click.option("--traces", is_flag=True, type=bool)
+def eof_wrap(input: str, output_dir: str, traces: bool):
+    """
+    Wraps JSON blockchain test file(s) found at `input` path and outputs them to the `output_dir`.
+    """
+    eof_wrapper = EofWrapper()
+
+    try:
+        EvmOneTransitionTool()
+    except CLINotFoundInPath:
+        print(f"Error: {EvmOneTransitionTool.default_binary} must be in the PATH.")
+        sys.exit(1)
+    except Exception as e:
+        raise Exception(f"Unexpected exception: {e}.")
+
+    if os.path.isfile(input):
+        file = os.path.basename(input)
+        out_file = "eof_wrapped_" + file
+        out_path = os.path.join(output_dir, out_file)
+
+        eof_wrapper.wrap_file(input, out_path, traces)
+    else:
+        for subdir, dirs, files in os.walk(input):
+            for file in files:
+                rel_dir = Path(subdir).relative_to(input)
+                out_file = "eof_wrapped_" + file
+                out_path = os.path.join(output_dir, rel_dir, out_file)
+                in_path = os.path.join(subdir, file)
+
+                eof_wrapper.wrap_file(in_path, out_path, traces)
+
+    os.makedirs(output_dir, exist_ok=True)
+    with open(os.path.join(output_dir, "metrics.json"), "w") as f:
+        json.dump(eof_wrapper.metrics, f, indent=4)
+
+
+class EofWrapper:
+    """
+    EOF wrapping of blockchain tests with some simple metrics tracking.
+    """
+
+    # JSON files had at least one fixture generated successfully with EOF
+    FILES_GENERATED = "files_generated"
+    # JSON files skipped explicitly or didn't have a fixture with EOF
+    FILES_SKIPPED = "files_skipped"
+    # Test fixtures with at least one EOF code and generated successfully
+    FIXTURES_GENERATED = "fixtures_generated"
+    # Test fixtures with no code able to be EOF-wrapped
+    FIXTURES_CANT_WRAP = "fixtures_cant_wrap"
+    # Test fixtures with EOF code but test doesn't pass and generation fails
+    FIXTURES_CANT_GENERATE = "fixtures_cant_generate"
+    # State accounts with code wrapped into valid EOF
+    ACCOUNTS_WRAPPED = "accounts_wrapped"
+    # State accounts with code wrapped into valid unique EOF
+    UNIQUE_ACCOUNTS_WRAPPED = "unique_accounts_wrapped"
+    # State accounts wrapped but the code is not valid EOF
+    ACCOUNTS_INVALID_EOF = "accounts_invalid_eof"
+    # State accounts wrapped into valid EOF but in a fixture of a failing test
+    ACCOUNTS_CANT_GENERATE = "accounts_cant_generate"
+    # Breakdown of EOF validation errors summing up to `accounts_invalid_eof`
+    VALIDATION_ERRORS = "validation_errors"
+    # Breakdown of runtime test failures summing up to `fixtures_cant_generate`
+    GENERATION_ERRORS = "generation_errors"
+
+    def __init__(self):
+        self.metrics = {
+            self.FILES_GENERATED: 0,
+            self.FILES_SKIPPED: 0,
+            self.FIXTURES_GENERATED: 0,
+            self.FIXTURES_CANT_WRAP: 0,
+            self.FIXTURES_CANT_GENERATE: 0,
+            self.ACCOUNTS_WRAPPED: 0,
+            self.UNIQUE_ACCOUNTS_WRAPPED: 0,
+            self.ACCOUNTS_INVALID_EOF: 0,
+            self.ACCOUNTS_CANT_GENERATE: 0,
+            self.VALIDATION_ERRORS: {},
+            self.GENERATION_ERRORS: {},
+        }
+        self.unique_eof = set()
+
+    file_skip_list = [
+        "Pyspecs",
+        # EXTCODE* opcodes return different results for EOF targets and that is tested elsewhere
+        "stExtCodeHash",
+        # bigint syntax
+        "ValueOverflowParis",
+        "bc4895-withdrawals",
+        # EOF opcodes at diff places - tests obsolete
+        "opcD0DiffPlaces",
+        "opcD1DiffPlaces",
+        "opcD2DiffPlaces",
+        "opcD3DiffPlaces",
+        "opcE0DiffPlaces",
+        "opcE1DiffPlaces",
+        "opcE2DiffPlaces",
+        "opcE3DiffPlaces",
+        "opcE4DiffPlaces",
+        "opcE5DiffPlaces",
+        "opcE6DiffPlaces",
+        "opcE7DiffPlaces",
+        "opcE8DiffPlaces",
+        "opcECDiffPlaces",
+        "opcEEDiffPlaces",
+        "opcF7DiffPlaces",
+        "opcF8DiffPlaces",
+        "opcF9DiffPlaces",
+        "opcFBDiffPlaces",
+        # stack overflow always (limit of `max_stack_height` is 1023!)
+        "push0_fill_stack",
+        "push0_stack_overflow",
+        "blobbasefee_stack_overflow",
+    ]
+
+    def wrap_file(self, in_path: str, out_path: str, traces: bool):
+        """
+        Wraps code from a blockchain test JSON file from `in_path` into EOF containers,
+        wherever possible. If not possible - skips and tracks that in metrics. Possible means
+        at least one account's code can be wrapped in a valid EOF container and the assertions
+        on post state are satisfied.
+        """
+        for skip in self.file_skip_list:
+            if skip in in_path:
+                self.metrics[self.FILES_SKIPPED] += 1
+                return
+
+        with open(in_path, "r") as input_file:
+            fixtures = BlockchainFixtures.from_json_data(json.load(input_file))
+
+        out_fixtures = BaseFixturesRootModel({})
+        fixture: BlockchainFixture
+        for id, fixture in fixtures.items():
+            fixture_eof_codes = []
+            wrapped_at_least_one_account = False
+
+            if fixture.pre:
+                for address, account in fixture.pre.root.items():
+                    if account is None or account.code is None or len(account.code) == 0:
+                        continue
+
+                    try:
+                        wrapped = wrap_code(account.code)
+                    except ValueError as e:
+                        self.metrics[self.ACCOUNTS_INVALID_EOF] += 1
+                        _inc_counter(
+                            self.metrics[self.VALIDATION_ERRORS], self._short_exception_msg(e)
+                        )
+                        continue
+
+                    if self._validate_eof(wrapped):
+                        account.code = Bytes(wrapped)
+                        wrapped_at_least_one_account = True
+                        self.metrics[self.ACCOUNTS_WRAPPED] += 1
+                        fixture_eof_codes.append(to_hex(account.code))
+
+                        # wrap the same account in post state the same way
+                        if fixture.post_state and fixture.post_state.root[address]:
+                            fixture.post_state.root[address].code = Bytes(wrapped)  # type: ignore
+                    else:
+                        self.metrics[self.ACCOUNTS_INVALID_EOF] += 1
+            if not wrapped_at_least_one_account:
+                self.metrics[self.FIXTURES_CANT_WRAP] += 1
+                continue
+
+            try:
+                out_fixture = self._wrap_fixture(fixture, traces)
+                out_fixtures[id] = out_fixture
+                self.metrics[self.FIXTURES_GENERATED] += 1
+                self.unique_eof.update(fixture_eof_codes)
+                self.metrics[self.UNIQUE_ACCOUNTS_WRAPPED] = len(self.unique_eof)
+            except Exception as e:
+                _inc_counter(self.metrics[self.GENERATION_ERRORS], self._short_exception_msg(e))
+
+                self.metrics[self.FIXTURES_CANT_GENERATE] += 1
+                self.metrics[self.ACCOUNTS_CANT_GENERATE] += len(fixture_eof_codes)
+
+        if len(out_fixtures) == 0:
+            self.metrics[self.FILES_SKIPPED] += 1
+            return
+
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+        out_fixtures.collect_into_file(Path(out_path))
+        self.metrics[self.FILES_GENERATED] += 1
+
+    def _short_exception_msg(self, e: Exception):
+        THRESHOLD = 30
+
+        short = str(e)
+        if len(short) > THRESHOLD:
+            short = short[:THRESHOLD] + "..."
+        return short
+
+    def _wrap_fixture(self, fixture: BlockchainFixture, traces: bool):
+        env = Environment()
+
+        pre = fixture.pre
+
+        t8n = EvmOneTransitionTool(trace=traces)
+
+        test = BlockchainTest(
+            genesis_environment=env,
+            pre=pre.root,
+            post=fixture.post_state.root if fixture.post_state else {},
+            blocks=[],
+            tag="wrapped test",
+        )
+
+        for fixture_block in fixture.blocks:
+            if isinstance(fixture_block, FixtureBlock):
+                header = fixture_block.header
+                block = Block(
+                    ommers_hash=header.ommers_hash,
+                    fee_recipient=header.fee_recipient,
+                    difficulty=header.difficulty,
+                    number=header.number,
+                    gas_limit=header.gas_limit,
+                    timestamp=header.timestamp,
+                    extra_data=header.extra_data,
+                    prev_randao=header.prev_randao,
+                    nonce=header.nonce,
+                    base_fee_per_gas=header.base_fee_per_gas,
+                    withdrawals_root=header.withdrawals_root,
+                    parent_beacon_block_root=header.parent_beacon_block_root,
+                    requests_root=header.requests_root,
+                )
+                assert not fixture_block.ommers
+                assert not fixture_block.withdrawals
+                assert not fixture_block.deposit_requests
+                assert not fixture_block.withdrawal_requests
+                assert not fixture_block.consolidation_requests
+
+                for fixture_tx in fixture_block.txs:
+                    fixture_tx_dump = fixture_tx.model_dump()
+                    fixture_tx_dump.pop("ty")
+                    fixture_tx_dump.pop("data")
+                    tx = Transaction(
+                        type=fixture_tx.ty,
+                        input=fixture_tx.data,
+                        **fixture_tx_dump,
+                    )
+                    block.txs.append(tx)
+            elif isinstance(fixture_block, InvalidFixtureBlock):
+                block = Block(
+                    rlp=fixture_block.rlp,
+                    exception=fixture_block.expect_exception,
+                )
+            else:
+                raise TypeError("not a FixtureBlock")
+
+            test.blocks.append(block)
+
+        result = test.generate(
+            request=None,  # type: ignore
+            t8n=t8n,
+            fork=Osaka,
+            fixture_format=BlockchainFixture,
+        )
+        if traces:
+            print_traces(t8n.get_traces())
+        return result
+
+    def _validate_eof(self, container: Container, metrics: bool = True) -> bool:
+        eof_parse = EOFParse()
+
+        result = eof_parse.run(input=to_hex(container))
+        actual_message = result.stdout.strip()
+        if "OK" not in actual_message:
+            if metrics:
+                _inc_counter(self.metrics[self.VALIDATION_ERRORS], actual_message)
+            return False
+
+        return True
+
+
+# `no_type_check` required because OpcodeWithOperand.opcode can be `None` when formatting as a
+# string, but here it can never be `None`.
+@no_type_check
+def wrap_code(account_code: Bytes) -> Container:
+    """
+    Wraps `account_code` into a simplest EOF container, applying some simple heuristics in
+    order to obtain a valid code section termination.
+    """
+    assert len(account_code) > 0
+
+    opcodes = process_evm_bytes(account_code)
+
+    if not opcodes[-1].terminating:
+        opcodes.append(OpcodeWithOperands(opcode=Op.STOP))
+
+    while len(opcodes) > 1 and opcodes[-2].terminating and opcodes[-1].terminating:
+        opcodes.pop()
+
+    bytecode = Bytecode()
+
+    for opcode in opcodes:
+        bytecode += opcode.bytecode
+
+    return Container.Code(bytecode)
+
+
+def _inc_counter(d: dict, key: Any) -> None:
+    if key in d:
+        d[key] += 1
+    else:
+        d[key] = 1