diff --git a/README.md b/README.md index ae2cdd3153..af8e2cbb0d 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The following are required to either generate or develop tests: 3. [`solc`](https://github.com/ethereum/solidity) >= `v0.8.17`; `solc` must be in accessible in the `PATH`. + ### Installation To generate tests from the test "fillers", it's necessary to install the Python packages provided by `execution-spec-tests` (it's recommended to use a virtual environment for the installation): @@ -41,7 +42,6 @@ tf --test-case yul head fixtures/example/example/yul.json ``` - ### Generating the Execution Spec Tests For Use With Clients To generate all the tests defined in the `./fillers` sub-directory, run the `tf` command: diff --git a/docs/getting_started/01_quick_start.md b/docs/getting_started/01_quick_start.md index 4662b51d9b..960ecc280c 100644 --- a/docs/getting_started/01_quick_start.md +++ b/docs/getting_started/01_quick_start.md @@ -36,7 +36,6 @@ tf --test-case yul head fixtures/example/example/yul.json ``` - ## Generating the Execution Spec Tests For Use With Clients To generate all the tests defined in the `./fillers` sub-directory, run the `tf` command: diff --git a/src/ethereum_test_filling_tool/__init__.py b/src/ethereum_test_filling_tool/__init__.py new file mode 100644 index 0000000000..d49c8b3f9e --- /dev/null +++ b/src/ethereum_test_filling_tool/__init__.py @@ -0,0 +1,11 @@ +""" +Filler related utilities and classes. +""" +from .filler import Filler +from .modules import find_modules, is_module_modified + +__all__ = ( + "Filler", + "find_modules", + "is_module_modified", +) diff --git a/src/ethereum_test_filling_tool/filler.py b/src/ethereum_test_filling_tool/filler.py new file mode 100644 index 0000000000..5c37b502ab --- /dev/null +++ b/src/ethereum_test_filling_tool/filler.py @@ -0,0 +1,133 @@ +""" +Provides the Filler Class: + +Fillers are python functions that, given an `EvmTransitionTool` and +`EvmBlockBuilder`, return a JSON object representing an Ethereum test case. + +This tool will traverse a package of filler python modules, fill each test +case within it, and write them to a file in a given output directory. +""" +import argparse +import concurrent.futures +import json +import logging +import os +import time + +from ethereum_test_tools import JSONEncoder +from evm_block_builder import EvmBlockBuilder +from evm_transition_tool import EvmTransitionTool + +from .modules import find_modules, is_module_modified + + +class Filler: + """ + A command line tool to process test fillers into full hydrated tests. + """ + + log: logging.Logger + + def __init__(self, options: argparse.Namespace) -> None: + self.log = logging.getLogger(__name__) + self.options = options + + def fill(self) -> None: + """ + Fill test fixtures. + """ + if self.options.benchmark: + start_time = time.time() + + fillers = self.get_fillers() + self.log.info(f"collected {len(fillers)} fillers") + + os.makedirs(self.options.output, exist_ok=True) + + t8n = EvmTransitionTool( + binary=self.options.evm_bin, trace=self.options.traces + ) + b11r = EvmBlockBuilder(binary=self.options.evm_bin) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=self.options.max_workers + ) as executor: + futures = [] + for filler in fillers: + future = executor.submit(self.fill_fixture, filler, t8n, b11r) + futures.append(future) + + for future in concurrent.futures.as_completed(futures): + future.result() + + if self.options.benchmark: + end_time = time.time() + elapsed_time = end_time - start_time + self.log.info( + f"Filled test fixtures in {elapsed_time:.2f} seconds." + ) + + def get_fillers(self): + """ + Returns a list of all fillers found in the specified package + and modules. + """ + fillers = [] + for package_name, module_name, module_loader in find_modules( + os.path.abspath(self.options.filler_path), + self.options.test_categories, + self.options.test_module, + ): + module_full_name = module_loader.name + self.log.debug(f"searching {module_full_name} for fillers") + module = module_loader.load_module() + for obj in module.__dict__.values(): + if callable(obj) and hasattr(obj, "__filler_metadata__"): + if ( + self.options.test_case + and self.options.test_case + not in obj.__filler_metadata__["name"] + ): + continue + obj.__filler_metadata__["module_path"] = [ + package_name, + module_name, + ] + fillers.append(obj) + return fillers + + def fill_fixture(self, filler, t8n, b11r): + """ + Fills the specified fixture using the given filler, + transaction tool, and block builder. + """ + name = filler.__filler_metadata__["name"] + module_path = filler.__filler_metadata__["module_path"] + output_dir = os.path.join( + self.options.output, + *(module_path if not self.options.no_output_structure else ""), + ) + os.makedirs(output_dir, exist_ok=True) + path = os.path.join(output_dir, f"{name}.json") + full_name = ".".join(module_path + [name]) + + # Only skip if the fixture file already exists, the module + # has not been modified since the last test filler run, and + # the user doesn't want to force a refill the + # fixtures (--force-refill). + if ( + os.path.exists(path) + and not is_module_modified( + path, self.options.filler_path, module_path + ) + and not self.options.force_refill + ): + self.log.debug(f"skipping - {full_name}") + return + + fixture = filler(t8n, b11r, "NoProof") + self.log.debug(f"filling - {full_name}") + with open(path, "w", encoding="utf-8") as f: + json.dump( + fixture, f, ensure_ascii=False, indent=4, cls=JSONEncoder + ) diff --git a/src/ethereum_test_filling_tool/main.py b/src/ethereum_test_filling_tool/main.py index cb299b8fdb..8b5477a8e0 100755 --- a/src/ethereum_test_filling_tool/main.py +++ b/src/ethereum_test_filling_tool/main.py @@ -2,200 +2,96 @@ Ethereum Test Filler ^^^^^^^^^^^^^^^^^^^^ -Execute test fillers to create "filled" tests that can be consumed by execution -clients. +Executes python test fillers to create "filled" tests (fixtures) +that can be consumed by ethereum execution clients. """ - import argparse -import json import logging -import os from pathlib import Path -from pkgutil import iter_modules - -from setuptools import find_packages -from ethereum_test_tools import JSONEncoder -from evm_block_builder import EvmBlockBuilder -from evm_transition_tool import EvmTransitionTool +from .filler import Filler -class Filler: - """ - A command line tool to process test fillers into full hydrated tests. - """ - - @staticmethod - def parse_arguments() -> argparse.Namespace: - """ - Parse command line arguments. - """ - parser = argparse.ArgumentParser() - - parser.add_argument( - "--evm-bin", - help="path to evm executable that provides `t8n` and `b11r` " - + "subcommands", - default=None, - type=Path, - ) - - parser.add_argument( - "--filler-path", - help="path to filler directives, default: ./fillers", - default="fillers", - type=Path, - ) - - parser.add_argument( - "--output", - help="directory to store filled test fixtures, \ - default: ./fixtures", - default="fixtures", - type=Path, - ) - - parser.add_argument( - "--test-categories", - type=str, - nargs="+", - help="limit to filling tests of specific categories", - ) - - parser.add_argument( - "--test-module", - help="limit to filling tests of a specific module", - ) - - parser.add_argument( - "--test-case", - help="limit to filling only tests with matching name", - ) - - parser.add_argument( - "--traces", - action="store_true", - help="collect traces of the execution information from the " - + "transition tool", - ) - - parser.add_argument( - "--no-output-structure", - action="store_true", - help="removes the folder structure from test fixture output", - ) - - return parser.parse_args() - - options: argparse.Namespace - log: logging.Logger - - def __init__(self) -> None: - self.log = logging.getLogger(__name__) - self.options = self.parse_arguments() - - def fill(self) -> None: - """ - Fill test fixtures. - """ - pkg_path = self.options.filler_path - - fillers = [] - - for package_name, module_name, module_loader in find_modules( - os.path.abspath(pkg_path), - self.options.test_categories, - self.options.test_module, - ): - module_full_name = module_loader.name - self.log.debug(f"searching {module_full_name} for fillers") - module = module_loader.load_module() - for obj in module.__dict__.values(): - if callable(obj): - if hasattr(obj, "__filler_metadata__"): - if ( - self.options.test_case - and self.options.test_case - not in obj.__filler_metadata__["name"] - ): - continue - obj.__filler_metadata__["module_path"] = [ - package_name, - module_name, - ] - fillers.append(obj) - - self.log.info(f"collected {len(fillers)} fillers") - - os.makedirs(self.options.output, exist_ok=True) - - t8n = EvmTransitionTool( - binary=self.options.evm_bin, trace=self.options.traces - ) - b11r = EvmBlockBuilder(binary=self.options.evm_bin) - - for filler in fillers: - name = filler.__filler_metadata__["name"] - output_dir = os.path.join( - self.options.output, - *(filler.__filler_metadata__["module_path"]) - if self.options.no_output_structure is None - else "", - ) - os.makedirs(output_dir, exist_ok=True) - path = os.path.join(output_dir, f"{name}.json") - - name = path[9 : len(path) - 5].replace("/", ".") - self.log.debug(f"filling - {name}") - fixture = filler(t8n, b11r, "NoProof") - - with open(path, "w", encoding="utf-8") as f: - json.dump( - fixture, f, ensure_ascii=False, indent=4, cls=JSONEncoder - ) - - -def find_modules(root, include_pkg, include_modules): - """ - Find modules recursively starting with the `root`. - Only modules in a package with name found in iterable `include_pkg` will be - yielded. - Only modules with name found in iterable `include_modules` will be yielded. - """ - modules = set() - for package in find_packages( - root, - include=include_pkg if include_pkg is not None else ("*",), - ): - package = package.replace( - ".", "/" - ) # sub_package tests i.e 'vm.vm_tests' - for info, package_path in recursive_iter_modules(root, package): - module_full_name = package_path + "." + info.name - if module_full_name not in modules: - if not include_modules or include_modules in info.name: - yield ( - package, - info.name, - info.module_finder.find_module(module_full_name), - ) - modules.add(module_full_name) - - -def recursive_iter_modules(root, package): +def parse_arguments() -> argparse.Namespace: """ - Helper function for find_packages. - Iterates through all sub-packages (packages within a package). - Recursively navigates down the package tree until a new module is found. + Parse command line arguments. """ - for info in iter_modules([os.path.join(root, package)]): - if info.ispkg: - yield from recursive_iter_modules( - root, os.path.join(package, info.name) - ) - else: - package_path = package.replace("/", ".") - yield info, package_path + parser = argparse.ArgumentParser() + + parser.add_argument( + "--evm-bin", + help="path to evm executable that provides `t8n` and `b11r` \ + subcommands", + default=None, + type=Path, + ) + + parser.add_argument( + "--filler-path", + help="path to filler directives, default: ./fillers", + default="fillers", + type=Path, + ) + + parser.add_argument( + "--output", + help="directory to store filled test fixtures, \ + default: ./fixtures", + default="fixtures", + type=Path, + ) + + parser.add_argument( + "--test-categories", + type=str, + nargs="+", + help="limit to filling tests of specific categories", + ) + + parser.add_argument( + "--test-module", + help="limit to filling tests of a specific module", + ) + + parser.add_argument( + "--test-case", + help="limit to filling only tests with matching name", + ) + + parser.add_argument( + "--traces", + action="store_true", + help="collect traces of the execution information from the \ + transition tool", + ) + + parser.add_argument( + "--no-output-structure", + action="store_true", + help="removes the folder structure from test fixture output", + ) + + parser.add_argument( + "--benchmark", + action="store_true", + help="logs the timing of the test filler for benchmarking", + ) + + parser.add_argument( + "--max-workers", + type=int, + help="specifies the max number of workers for the test filler \ + set to 1 for serial execution", + ) + + parser.add_argument( + "--force-refill", + action="store_true", + help="fill all test fillers and don't skip any tests \ + overwriting where necessary", + ) + + return parser.parse_args() def main() -> None: @@ -204,5 +100,5 @@ def main() -> None: """ logging.basicConfig(level=logging.DEBUG) - filler = Filler() + filler = Filler(parse_arguments()) filler.fill() diff --git a/src/ethereum_test_filling_tool/modules.py b/src/ethereum_test_filling_tool/modules.py new file mode 100644 index 0000000000..a237a7f387 --- /dev/null +++ b/src/ethereum_test_filling_tool/modules.py @@ -0,0 +1,70 @@ +""" +Utility functions for finding and working with modules. + +Functions: + is_module_modified: Checks if a module was modified more + recently than it was filled. + find_modules: Recursively finds modules in a directory, + filtered by package and module name. + recursive_iter_modules: Iterates through all sub-packages + of a package to find modules. +""" +import os +from pkgutil import iter_modules + +from setuptools import find_packages + + +def is_module_modified(path, pkg_path, module_path): + """ + Returns True if a module was modified more recently than + it was filled, False otherwise. + """ + modified_time = os.path.getmtime( + os.path.join(pkg_path, *module_path) + ".py" + ) + filled_time = os.path.getmtime(path) if os.path.exists(path) else 0 + return modified_time > filled_time + + +def find_modules(root, include_pkg, include_modules): + """ + Find modules recursively starting with the `root`. + Only modules in a package with name found in iterable `include_pkg` will be + yielded. + Only modules with name found in iterable `include_modules` will be yielded. + """ + modules = set() + for package in find_packages( + root, + include=include_pkg if include_pkg is not None else ("*",), + ): + package = package.replace( + ".", "/" + ) # sub_package tests i.e 'vm.vm_tests' + for info, package_path in recursive_iter_modules(root, package): + module_full_name = package_path + "." + info.name + if module_full_name not in modules: + if not include_modules or include_modules in info.name: + yield ( + package, + info.name, + info.module_finder.find_module(module_full_name), + ) + modules.add(module_full_name) + + +def recursive_iter_modules(root, package): + """ + Helper function for find_packages. + Iterates through all sub-packages (packages within a package). + Recursively navigates down the package tree until a new module is found. + """ + for info in iter_modules([os.path.join(root, package)]): + if info.ispkg: + yield from recursive_iter_modules( + root, os.path.join(package, info.name) + ) + else: + package_path = package.replace("/", ".") + yield info, package_path diff --git a/whitelist.txt b/whitelist.txt index 208b44eb65..dd7ecd56f4 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -41,6 +41,7 @@ vm gwei wei +getmtime byteorder delitem dirname