From 1e9922d9f76bed2083ce2187e5be72912cfe2082 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 25 Sep 2023 09:59:56 -0700 Subject: [PATCH 01/17] chore: add notes to selector table implementation (#3618) and a couple sanity checks --- vyper/codegen/function_definitions/common.py | 7 +++++++ vyper/codegen/module.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/function_definitions/common.py b/vyper/codegen/function_definitions/common.py index 3fd5ce0b29..1d24b6c6dd 100644 --- a/vyper/codegen/function_definitions/common.py +++ b/vyper/codegen/function_definitions/common.py @@ -73,6 +73,13 @@ class EntryPointInfo: min_calldatasize: int # the min calldata required for this entry point ir_node: IRnode # the ir for this entry point + def __post_init__(self): + # ABI v2 property guaranteed by the spec. + # https://docs.soliditylang.org/en/v0.8.21/abi-spec.html#formal-specification-of-the-encoding states: # noqa: E501 + # > Note that for any X, len(enc(X)) is a multiple of 32. + assert self.min_calldatasize >= 4 + assert (self.min_calldatasize - 4) % 32 == 0 + @dataclass class ExternalFuncIR(FuncIR): diff --git a/vyper/codegen/module.py b/vyper/codegen/module.py index 6445a5e1e0..bfdafa8ba9 100644 --- a/vyper/codegen/module.py +++ b/vyper/codegen/module.py @@ -93,9 +93,12 @@ def _generate_external_entry_points(external_functions, global_ctx): for code in external_functions: func_ir = generate_ir_for_function(code, global_ctx) for abi_sig, entry_point in func_ir.entry_points.items(): + method_id = method_id_int(abi_sig) assert abi_sig not in entry_points + assert method_id not in sig_of + entry_points[abi_sig] = entry_point - sig_of[method_id_int(abi_sig)] = abi_sig + sig_of[method_id] = abi_sig # stick function common body into final entry point to save a jump ir_node = IRnode.from_list(["seq", entry_point.ir_node, func_ir.common_ir]) From 7b9d159b84c0e568378d53e753350e7f691c413a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 25 Sep 2023 20:05:33 -0700 Subject: [PATCH 02/17] docs: mcopy is enabled with cancun target (#3620) --- docs/compiling-a-contract.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index 6d1cdf98d7..b529d1efb1 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -197,6 +197,7 @@ The following is a list of supported EVM versions, and changes in the compiler i - The ``transient`` keyword allows declaration of variables which live in transient storage - Functions marked with ``@nonreentrant`` are protected with TLOAD/TSTORE instead of SLOAD/SSTORE + - The ``MCOPY`` opcode will be generated automatically by the compiler for most memory operations. From e5c323afa4f61a2fc7a28bbfb824afd90ec86158 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 26 Sep 2023 11:24:32 -0700 Subject: [PATCH 03/17] chore: fix some documentation inconsistencies (#3624) * raw_args kwarg to create_from_blueprint * fix create_from_blueprint example * clarify "all but one 64th" behavior when forwarding gas left= * remove dead comment * update internal documentation for generate_ir_for_external_function() * update memory layout in create_from_blueprint comments * fix warning for BitwiseNot * fix error message about msg.data * docs: extract32 can output any bytesM type * remove dead variable in comment --- docs/built-in-functions.rst | 13 +++++++++---- vyper/builtins/functions.py | 6 ++---- .../function_definitions/external_function.py | 11 ++++------- vyper/semantics/analysis/local.py | 2 +- vyper/semantics/analysis/module.py | 1 - 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/built-in-functions.rst b/docs/built-in-functions.rst index bfaa8fdd5e..45cf9ec8c2 100644 --- a/docs/built-in-functions.rst +++ b/docs/built-in-functions.rst @@ -184,13 +184,14 @@ Vyper has three built-ins for contract creation; all three contract creation bui The implementation of ``create_copy_of`` assumes that the code at ``target`` is smaller than 16MB. While this is much larger than the EIP-170 constraint of 24KB, it is a conservative size limit intended to future-proof deployer contracts in case the EIP-170 constraint is lifted. If the code at ``target`` is larger than 16MB, the behavior of ``create_copy_of`` is undefined. -.. py:function:: create_from_blueprint(target: address, *args, value: uint256 = 0, code_offset=0, [, salt: bytes32]) -> address +.. py:function:: create_from_blueprint(target: address, *args, value: uint256 = 0, raw_args: bool = False, code_offset: int = 0, [, salt: bytes32]) -> address Copy the code of ``target`` into memory and execute it as initcode. In other words, this operation interprets the code at ``target`` not as regular runtime code, but directly as initcode. The ``*args`` are interpreted as constructor arguments, and are ABI-encoded and included when executing the initcode. * ``target``: Address of the blueprint to invoke * ``*args``: Constructor arguments to forward to the initcode. * ``value``: The wei value to send to the new contract address (Optional, default 0) + * ``raw_args``: If ``True``, ``*args`` must be a single ``Bytes[...]`` argument, which will be interpreted as a raw bytes buffer to forward to the create operation (which is useful for instance, if pre- ABI-encoded data is passed in from elsewhere). (Optional, default ``False``) * ``code_offset``: The offset to start the ``EXTCODECOPY`` from (Optional, default 0) * ``salt``: A ``bytes32`` value utilized by the deterministic ``CREATE2`` opcode (Optional, if not supplied, ``CREATE`` is used) @@ -201,7 +202,7 @@ Vyper has three built-ins for contract creation; all three contract creation bui @external def foo(blueprint: address) -> address: arg1: uint256 = 18 - arg2: String = "some string" + arg2: String[32] = "some string" return create_from_blueprint(blueprint, arg1, arg2, code_offset=1) .. note:: @@ -226,7 +227,7 @@ Vyper has three built-ins for contract creation; all three contract creation bui * ``to``: Destination address to call to * ``data``: Data to send to the destination address * ``max_outsize``: Maximum length of the bytes array returned from the call. If the returned call data exceeds this length, only this number of bytes is returned. (Optional, default ``0``) - * ``gas``: The amount of gas to attach to the call. If not set, all remaining gas is forwarded. + * ``gas``: The amount of gas to attach to the call. (Optional, defaults to ``msg.gas``). * ``value``: The wei value to send to the address (Optional, default ``0``) * ``is_delegate_call``: If ``True``, the call will be sent as ``DELEGATECALL`` (Optional, default ``False``) * ``is_static_call``: If ``True``, the call will be sent as ``STATICCALL`` (Optional, default ``False``) @@ -264,6 +265,10 @@ Vyper has three built-ins for contract creation; all three contract creation bui assert success return response + .. note:: + + Regarding "forwarding all gas", note that, while Vyper will provide ``msg.gas`` to the call, in practice, there are some subtleties around forwarding all remaining gas on the EVM which are out of scope of this documentation and could be subject to change. For instance, see the language in EIP-150 around "all but one 64th". + .. py:function:: raw_log(topics: bytes32[4], data: Union[Bytes, bytes32]) -> None Provides low level access to the ``LOG`` opcodes, emitting a log without having to specify an ABI type. @@ -500,7 +505,7 @@ Data Manipulation * ``b``: ``Bytes`` list to extract from * ``start``: Start point to extract from - * ``output_type``: Type of output (``bytes32``, ``integer``, or ``address``). Defaults to ``bytes32``. + * ``output_type``: Type of output (``bytesM``, ``integer``, or ``address``). Defaults to ``bytes32``. Returns a value of the type specified by ``output_type``. diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 95759372a6..a0936712b2 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -1418,7 +1418,7 @@ class BitwiseNot(BuiltinFunction): def evaluate(self, node): if not self.__class__._warned: - vyper_warn("`bitwise_not()` is deprecated! Please use the ^ operator instead.") + vyper_warn("`bitwise_not()` is deprecated! Please use the ~ operator instead.") self.__class__._warned = True validate_call_args(node, 1) @@ -1917,9 +1917,8 @@ def _build_create_IR(self, expr, args, context, value, salt, code_offset, raw_ar # copy the target code into memory. # layout starting from mem_ofst: - # 00...00 (22 0's) | preamble | bytecode + # | ir.append(["extcodecopy", target, mem_ofst, code_offset, codesize]) - ir.append(copy_bytes(add_ofst(mem_ofst, codesize), argbuf, encoded_args_len, bufsz)) # theoretically, dst = "msize", but just be safe. @@ -2586,7 +2585,6 @@ def evaluate(self, node): if isinstance(input_type, IntegerT): ret = vy_ast.Int.from_node(node, value=val) - # TODO: to change to known_type once #3213 is merged ret._metadata["type"] = input_type return ret diff --git a/vyper/codegen/function_definitions/external_function.py b/vyper/codegen/function_definitions/external_function.py index 32236e9aad..65276469e7 100644 --- a/vyper/codegen/function_definitions/external_function.py +++ b/vyper/codegen/function_definitions/external_function.py @@ -135,20 +135,17 @@ def handler_for(calldata_kwargs, default_kwargs): return ret -# TODO it would be nice if this returned a data structure which were -# amenable to generating a jump table instead of the linear search for -# method_id we have now. def generate_ir_for_external_function(code, func_t, context): # TODO type hints: # def generate_ir_for_external_function( # code: vy_ast.FunctionDef, # func_t: ContractFunctionT, # context: Context, - # check_nonpayable: bool, # ) -> IRnode: - """Return the IR for an external function. Includes code to inspect the method_id, - enter the function (nonpayable and reentrancy checks), handle kwargs and exit - the function (clean up reentrancy storage variables) + """ + Return the IR for an external function. Returns IR for the body + of the function, handle kwargs and exit the function. Also returns + metadata required for `module.py` to construct the selector table. """ nonreentrant_pre, nonreentrant_post = get_nonreentrant_lock(func_t) diff --git a/vyper/semantics/analysis/local.py b/vyper/semantics/analysis/local.py index c10df3b8fd..b391b33953 100644 --- a/vyper/semantics/analysis/local.py +++ b/vyper/semantics/analysis/local.py @@ -150,7 +150,7 @@ def _validate_msg_data_attribute(node: vy_ast.Attribute) -> None: allowed_builtins = ("slice", "len", "raw_call") if not isinstance(parent, vy_ast.Call) or parent.get("func.id") not in allowed_builtins: raise StructureException( - "msg.data is only allowed inside of the slice or len functions", node + "msg.data is only allowed inside of the slice, len or raw_call functions", node ) if parent.get("func.id") == "slice": ok_args = len(parent.args) == 3 and isinstance(parent.args[2], vy_ast.Int) diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index 02ae82faac..e59422294c 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -98,7 +98,6 @@ def __init__( _ns.update({k: namespace[k] for k in namespace._scopes[-1]}) # type: ignore module_node._metadata["namespace"] = _ns - # check for collisions between 4byte function selectors self_members = namespace["self"].typ.members # get list of internal function calls made by each function From d438d927bed3b850fe4768a490f3acde5f51b475 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 26 Sep 2023 15:24:46 -0700 Subject: [PATCH 04/17] fix: `_abi_decode()` validation (#3626) `_abi_decode()` does not validate input when it is nested in certain expressions. the following example gets correctly validated (bounds checked): ```vyper x: uint8 = _abi_decode(slice(msg.data, 4, 32), uint8) ``` however, the following example is not bounds checked: ```vyper @external def abi_decode(x: uint256) -> uint256: a: uint256 = convert( _abi_decode( slice(msg.data, 4, 32), (uint8) ), uint256 ) return a # abi_decode(256) returns: 256 ``` the issue is caused because the `ABIDecode()` builtin tags its output with `encoding=Encoding.ABI`, but this does not result in validation until that itself is passed to `make_setter` (which is called for instance when generating an assignment or return statement). the issue can be triggered by constructing an example where the output of `ABIDecode()` is not internally passed to `make_setter` or other input validating routine. this commit fixes the issue by calling `make_setter` in `ABIDecode()` before returning the output buffer, which causes validation to be performed. note that this causes a performance regression in the common (and majority of) cases where `make_setter` is immediately called on the result of `ABIDecode()` because a redundant memory copy ends up being generated (like in the aforementioned examples: in a plain assignment or return statement). however, fixing this performance regression is left to future work in the optimizer. --- tests/parser/functions/test_abi_decode.py | 28 ++++++++++++ vyper/builtins/functions.py | 52 +++++++++++------------ 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/tests/parser/functions/test_abi_decode.py b/tests/parser/functions/test_abi_decode.py index 2f9b93057d..2216a5bd76 100644 --- a/tests/parser/functions/test_abi_decode.py +++ b/tests/parser/functions/test_abi_decode.py @@ -344,6 +344,34 @@ def abi_decode(x: Bytes[96]) -> (uint256, uint256): assert_tx_failed(lambda: c.abi_decode(input_)) +def test_clamper_nested_uint8(get_contract, assert_tx_failed): + # check that _abi_decode clamps on word-types even when it is in a nested expression + # decode -> validate uint8 -> revert if input >= 256 -> cast back to uint256 + contract = """ +@external +def abi_decode(x: uint256) -> uint256: + a: uint256 = convert(_abi_decode(slice(msg.data, 4, 32), (uint8)), uint256) + return a + """ + c = get_contract(contract) + assert c.abi_decode(255) == 255 + assert_tx_failed(lambda: c.abi_decode(256)) + + +def test_clamper_nested_bytes(get_contract, assert_tx_failed): + # check that _abi_decode clamps dynamic even when it is in a nested expression + # decode -> validate Bytes[20] -> revert if len(input) > 20 -> convert back to -> add 1 + contract = """ +@external +def abi_decode(x: Bytes[96]) -> Bytes[21]: + a: Bytes[21] = concat(b"a", _abi_decode(x, Bytes[20])) + return a + """ + c = get_contract(contract) + assert c.abi_decode(abi.encode("(bytes)", (b"bc",))) == b"abc" + assert_tx_failed(lambda: c.abi_decode(abi.encode("(bytes)", (b"a" * 22,)))) + + @pytest.mark.parametrize( "output_typ,input_", [ diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index a0936712b2..8cdd2a4b8b 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -29,7 +29,6 @@ get_type_for_exact_size, ir_tuple_from_args, make_setter, - needs_external_call_wrap, promote_signed_int, sar, shl, @@ -2367,8 +2366,6 @@ def build_IR(self, expr, args, kwargs, context): class ABIEncode(BuiltinFunction): _id = "_abi_encode" # TODO prettier to rename this to abi.encode # signature: *, ensure_tuple= -> Bytes[] - # (check the signature manually since we have no utility methods - # to handle varargs.) # explanation of ensure_tuple: # default is to force even a single value into a tuple, # e.g. _abi_encode(bytes) -> _abi_encode((bytes,)) @@ -2529,24 +2526,11 @@ def build_IR(self, expr, args, kwargs, context): ) data = ensure_in_memory(data, context) + with data.cache_when_complex("to_decode") as (b1, data): data_ptr = bytes_data_ptr(data) data_len = get_bytearray_length(data) - # Normally, ABI-encoded data assumes the argument is a tuple - # (See comments for `wrap_value_for_external_return`) - # However, we do not want to use `wrap_value_for_external_return` - # technique as used in external call codegen because in order to be - # type-safe we would need an extra memory copy. To avoid a copy, - # we manually add the ABI-dynamic offset so that it is - # re-interpreted in-place. - if ( - unwrap_tuple is True - and needs_external_call_wrap(output_typ) - and output_typ.abi_type.is_dynamic() - ): - data_ptr = add_ofst(data_ptr, 32) - ret = ["seq"] if abi_min_size == abi_size_bound: @@ -2555,18 +2539,30 @@ def build_IR(self, expr, args, kwargs, context): # runtime assert: abi_min_size <= data_len <= abi_size_bound ret.append(clamp2(abi_min_size, data_len, abi_size_bound, signed=False)) - # return pointer to the buffer - ret.append(data_ptr) - - return b1.resolve( - IRnode.from_list( - ret, - typ=output_typ, - location=data.location, - encoding=Encoding.ABI, - annotation=f"abi_decode({output_typ})", - ) + to_decode = IRnode.from_list( + data_ptr, + typ=wrapped_typ, + location=data.location, + encoding=Encoding.ABI, + annotation=f"abi_decode({output_typ})", ) + to_decode.encoding = Encoding.ABI + + # TODO optimization: skip make_setter when we don't need + # input validation + + output_buf = context.new_internal_variable(wrapped_typ) + output = IRnode.from_list(output_buf, typ=wrapped_typ, location=MEMORY) + + # sanity check buffer size for wrapped output type will not buffer overflow + assert wrapped_typ.memory_bytes_required == output_typ.memory_bytes_required + ret.append(make_setter(output, to_decode)) + + ret.append(output) + # finalize. set the type and location for the return buffer. + # (note: unwraps the tuple type if necessary) + ret = IRnode.from_list(ret, typ=output_typ, location=MEMORY) + return b1.resolve(ret) class _MinMaxValue(TypenameFoldedFunction): From 950a97ea0d16db9884ec2f09bc71f1fc52c20bb5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 26 Sep 2023 16:19:34 -0700 Subject: [PATCH 05/17] fix: type check abi_decode arguments (#3623) currently, the following code will trigger a compiler panic: ```vyper @external def foo(j: uint256) -> bool: s: bool = _abi_decode(j, bool, unwrap_tuple= False) return s ``` the following code will compile, even though it should not typecheck: ```vyper @external def foo(s: String[32]) -> bool: t: bool = _abi_decode(s, bool, unwrap_tuple=False) return t ``` this commit fixes the issue by typechecking the input to `_abi_decode()`. it also adds syntax tests for `_abi_decode()`. --- tests/parser/syntax/test_abi_decode.py | 45 ++++++++++++++++++++++++++ vyper/builtins/functions.py | 2 ++ 2 files changed, 47 insertions(+) create mode 100644 tests/parser/syntax/test_abi_decode.py diff --git a/tests/parser/syntax/test_abi_decode.py b/tests/parser/syntax/test_abi_decode.py new file mode 100644 index 0000000000..f05ff429cd --- /dev/null +++ b/tests/parser/syntax/test_abi_decode.py @@ -0,0 +1,45 @@ +import pytest + +from vyper import compiler +from vyper.exceptions import TypeMismatch + +fail_list = [ + ( + """ +@external +def foo(j: uint256) -> bool: + s: bool = _abi_decode(j, bool, unwrap_tuple= False) + return s + """, + TypeMismatch, + ), + ( + """ +@external +def bar(j: String[32]) -> bool: + s: bool = _abi_decode(j, bool, unwrap_tuple= False) + return s + """, + TypeMismatch, + ), +] + + +@pytest.mark.parametrize("bad_code,exc", fail_list) +def test_abi_encode_fail(bad_code, exc): + with pytest.raises(exc): + compiler.compile_code(bad_code) + + +valid_list = [ + """ +@external +def foo(x: Bytes[32]) -> uint256: + return _abi_decode(x, uint256) + """ +] + + +@pytest.mark.parametrize("good_code", valid_list) +def test_abi_encode_success(good_code): + assert compiler.compile_code(good_code) is not None diff --git a/vyper/builtins/functions.py b/vyper/builtins/functions.py index 8cdd2a4b8b..f07202831d 100644 --- a/vyper/builtins/functions.py +++ b/vyper/builtins/functions.py @@ -2490,6 +2490,8 @@ def fetch_call_return(self, node): return output_type.typedef def infer_arg_types(self, node): + self._validate_arg_types(node) + validate_call_args(node, 2, ["unwrap_tuple"]) data_type = get_exact_type_from_node(node.args[0]) From 2bdbd846b09c94f05739e1274e00825912404fe3 Mon Sep 17 00:00:00 2001 From: tserg <8017125+tserg@users.noreply.github.com> Date: Wed, 27 Sep 2023 21:06:03 +0800 Subject: [PATCH 06/17] chore: add metadata to vyper-json (#3622) --- tests/cli/vyper_json/test_output_selection.py | 6 ++++++ vyper/cli/vyper_json.py | 2 +- vyper/compiler/output.py | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/cli/vyper_json/test_output_selection.py b/tests/cli/vyper_json/test_output_selection.py index c72f06f5a7..3b12e2b54a 100644 --- a/tests/cli/vyper_json/test_output_selection.py +++ b/tests/cli/vyper_json/test_output_selection.py @@ -52,3 +52,9 @@ def test_solc_style(): input_json = {"settings": {"outputSelection": {"foo.vy": {"": ["abi"], "foo.vy": ["ir"]}}}} sources = {"foo.vy": ""} assert get_input_dict_output_formats(input_json, sources) == {"foo.vy": ["abi", "ir_dict"]} + + +def test_metadata(): + input_json = {"settings": {"outputSelection": {"*": ["metadata"]}}} + sources = {"foo.vy": ""} + assert get_input_dict_output_formats(input_json, sources) == {"foo.vy": ["metadata"]} diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 4a1c91550e..f6d82c3fe0 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -29,7 +29,7 @@ "interface": "interface", "ir": "ir_dict", "ir_runtime": "ir_runtime_dict", - # "metadata": "metadata", # don't include in "*" output for now + "metadata": "metadata", "layout": "layout", "userdoc": "userdoc", } diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 334c5ba613..9ef492c3e2 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -104,7 +104,6 @@ def build_ir_runtime_dict_output(compiler_data: CompilerData) -> dict: def build_metadata_output(compiler_data: CompilerData) -> dict: - warnings.warn("metadata output format is unstable!") sigs = compiler_data.function_signatures def _var_rec_dict(variable_record): From aecd911347af5912a22540ff3dc513273e51c72d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 28 Sep 2023 14:42:12 -0700 Subject: [PATCH 07/17] fix: metadata output interaction with natspec (#3627) enabling the `-f metadata` output has an interaction with other outputs because the metadata output format mutates some internal data structures in-place. this is because `vars()` returns a reference to the object's `__dict__` as opposed to a copy of it. the behavior can be seen by trying to call the compiler with `-f metadata,devdoc,userdoc`. this issue was revealed in (but not introduced by) 2bdbd846b0, because that commit caused metadata and userdoc to be bundled together by default. this commit fixes the issue by constructing a copy of the object during metadata output formatting. it also modifies the test suite to include more output formats, to test the interactions between these different output formats. in doing so, it was also found that some examples have invalid natspec, which has also been fixed. --- examples/tokens/ERC1155ownable.vy | 2 -- tests/base_conftest.py | 4 ++-- vyper/compiler/output.py | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/tokens/ERC1155ownable.vy b/examples/tokens/ERC1155ownable.vy index 8094225f18..f1070b8f89 100644 --- a/examples/tokens/ERC1155ownable.vy +++ b/examples/tokens/ERC1155ownable.vy @@ -214,7 +214,6 @@ def mint(receiver: address, id: uint256, amount:uint256): @param receiver the account that will receive the minted token @param id the ID of the token @param amount of tokens for this ID - @param data the data associated with this mint. Usually stays empty """ assert not self.paused, "The contract has been paused" assert self.owner == msg.sender, "Only the contract owner can mint" @@ -232,7 +231,6 @@ def mintBatch(receiver: address, ids: DynArray[uint256, BATCH_SIZE], amounts: Dy @param receiver the account that will receive the minted token @param ids array of ids for the tokens @param amounts amounts of tokens for each ID in the ids array - @param data the data associated with this mint. Usually stays empty """ assert not self.paused, "The contract has been paused" assert self.owner == msg.sender, "Only the contract owner can mint" diff --git a/tests/base_conftest.py b/tests/base_conftest.py index 81e8dedc36..1c7c6f3aed 100644 --- a/tests/base_conftest.py +++ b/tests/base_conftest.py @@ -118,8 +118,8 @@ def _get_contract(w3, source_code, optimize, *args, override_opt_level=None, **k settings.optimize = override_opt_level or optimize out = compiler.compile_code( source_code, - # test that metadata gets generated - ["abi", "bytecode", "metadata"], + # test that metadata and natspecs get generated + ["abi", "bytecode", "metadata", "userdoc", "devdoc"], settings=settings, interface_codes=kwargs.pop("interface_codes", None), show_gas_estimates=True, # Enable gas estimates for testing diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 9ef492c3e2..1c38fcff9b 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -107,7 +107,7 @@ def build_metadata_output(compiler_data: CompilerData) -> dict: sigs = compiler_data.function_signatures def _var_rec_dict(variable_record): - ret = vars(variable_record) + ret = vars(variable_record).copy() ret["typ"] = str(ret["typ"]) if ret["data_offset"] is None: del ret["data_offset"] @@ -117,7 +117,7 @@ def _var_rec_dict(variable_record): return ret def _to_dict(func_t): - ret = vars(func_t) + ret = vars(func_t).copy() ret["return_type"] = str(ret["return_type"]) ret["_ir_identifier"] = func_t._ir_info.ir_identifier @@ -133,7 +133,7 @@ def _to_dict(func_t): args = ret[attr] ret[attr] = {arg.name: str(arg.typ) for arg in args} - ret["frame_info"] = vars(func_t._ir_info.frame_info) + ret["frame_info"] = vars(func_t._ir_info.frame_info).copy() del ret["frame_info"]["frame_vars"] # frame_var.pos might be IR, cannot serialize keep_keys = { From 42817806cadaffefed7bf9c8edd64abf439be4de Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 28 Sep 2023 14:55:05 -0700 Subject: [PATCH 08/17] fix: improve test case inputs in selector table fuzz (#3625) this commit improves the fuzz examples for the selector table. the nested `@given` tests too many "dumb" examples (ex. 0, 1, max_value) when `max_examples` is not large enough. the nested `@given` strategy can find falsifying inputs, but it requires the inner `max_examples` to be set much higher, and the shrinking takes much longer. this setting of `max_examples=125` with a single `@given` using the `@composite` strategy in this commit finds the selector table bug (that was fixed in 823675a8dc) after an average of 3 runs. --- tests/parser/test_selector_table.py | 98 +++++++++++++++-------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/tests/parser/test_selector_table.py b/tests/parser/test_selector_table.py index 3ac50707c2..180c0266bf 100644 --- a/tests/parser/test_selector_table.py +++ b/tests/parser/test_selector_table.py @@ -478,66 +478,72 @@ def test_dense_jumptable_bucket_size(n_methods, seed): assert n_buckets / n < 0.4 or n < 10 +@st.composite +def generate_methods(draw, max_calldata_bytes): + max_default_args = draw(st.integers(min_value=0, max_value=4)) + default_fn_mutability = draw(st.sampled_from(["", "@pure", "@view", "@nonpayable", "@payable"])) + + return ( + max_default_args, + default_fn_mutability, + draw( + st.lists( + st.tuples( + # function id: + st.integers(min_value=0), + # mutability: + st.sampled_from(["@pure", "@view", "@nonpayable", "@payable"]), + # n calldata words: + st.integers(min_value=0, max_value=max_calldata_bytes // 32), + # n bytes to strip from calldata + st.integers(min_value=1, max_value=4), + # n default args + st.integers(min_value=0, max_value=max_default_args), + ), + unique_by=lambda x: x[0], + min_size=1, + max_size=100, + ) + ), + ) + + @pytest.mark.parametrize("opt_level", list(OptimizationLevel)) # dense selector table packing boundaries at 256 and 65336 @pytest.mark.parametrize("max_calldata_bytes", [255, 256, 65336]) -@settings(max_examples=5, deadline=None) -@given( - seed=st.integers(min_value=0, max_value=2**64 - 1), - max_default_args=st.integers(min_value=0, max_value=4), - default_fn_mutability=st.sampled_from(["", "@pure", "@view", "@nonpayable", "@payable"]), -) @pytest.mark.fuzzing def test_selector_table_fuzz( - max_calldata_bytes, - seed, - max_default_args, - opt_level, - default_fn_mutability, - w3, - get_contract, - assert_tx_failed, - get_logs, + max_calldata_bytes, opt_level, w3, get_contract, assert_tx_failed, get_logs ): - def abi_sig(calldata_words, i, n_default_args): - args = [] if not calldata_words else [f"uint256[{calldata_words}]"] - args.extend(["uint256"] * n_default_args) - argstr = ",".join(args) - return f"foo{seed + i}({argstr})" + def abi_sig(func_id, calldata_words, n_default_args): + params = [] if not calldata_words else [f"uint256[{calldata_words}]"] + params.extend(["uint256"] * n_default_args) + paramstr = ",".join(params) + return f"foo{func_id}({paramstr})" - def generate_func_def(mutability, calldata_words, i, n_default_args): + def generate_func_def(func_id, mutability, calldata_words, n_default_args): arglist = [] if not calldata_words else [f"x: uint256[{calldata_words}]"] for j in range(n_default_args): arglist.append(f"x{j}: uint256 = 0") args = ", ".join(arglist) - _log_return = f"log _Return({i})" if mutability == "@payable" else "" + _log_return = f"log _Return({func_id})" if mutability == "@payable" else "" return f""" @external {mutability} -def foo{seed + i}({args}) -> uint256: +def foo{func_id}({args}) -> uint256: {_log_return} - return {i} + return {func_id} """ - @given( - methods=st.lists( - st.tuples( - st.sampled_from(["@pure", "@view", "@nonpayable", "@payable"]), - st.integers(min_value=0, max_value=max_calldata_bytes // 32), - # n bytes to strip from calldata - st.integers(min_value=1, max_value=4), - # n default args - st.integers(min_value=0, max_value=max_default_args), - ), - min_size=1, - max_size=100, - ) - ) - @settings(max_examples=25) - def _test(methods): + @given(_input=generate_methods(max_calldata_bytes)) + @settings(max_examples=125, deadline=None) + def _test(_input): + max_default_args, default_fn_mutability, methods = _input + func_defs = "\n".join( - generate_func_def(m, s, i, d) for i, (m, s, _, d) in enumerate(methods) + generate_func_def(func_id, mutability, calldata_words, n_default_args) + for (func_id, mutability, calldata_words, _, n_default_args) in (methods) ) if default_fn_mutability == "": @@ -571,8 +577,8 @@ def __default__(): c = get_contract(code, override_opt_level=opt_level) - for i, (mutability, n_calldata_words, n_strip_bytes, n_default_args) in enumerate(methods): - funcname = f"foo{seed + i}" + for func_id, mutability, n_calldata_words, n_strip_bytes, n_default_args in methods: + funcname = f"foo{func_id}" func = getattr(c, funcname) for j in range(n_default_args + 1): @@ -580,9 +586,9 @@ def __default__(): args.extend([1] * j) # check the function returns as expected - assert func(*args) == i + assert func(*args) == func_id - method_id = utils.method_id(abi_sig(n_calldata_words, i, j)) + method_id = utils.method_id(abi_sig(func_id, n_calldata_words, j)) argsdata = b"\x00" * (n_calldata_words * 32 + j * 32) @@ -590,7 +596,7 @@ def __default__(): if mutability == "@payable": tx = func(*args, transact={"value": 1}) (event,) = get_logs(tx, c, "_Return") - assert event.args.val == i + assert event.args.val == func_id else: hexstr = (method_id + argsdata).hex() txdata = {"to": c.address, "data": hexstr, "value": 1} From 917959e3993ab0592d28bb5326e89a7a3ae0eb58 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 28 Sep 2023 18:25:39 -0700 Subject: [PATCH 09/17] docs: new for loop range syntax: `bound=` (#3540) `for i in range(..., bound=...)` --------- Co-authored-by: El De-dog-lo <3859395+fubuloubu@users.noreply.github.com> --- docs/control-structures.rst | 13 +++++++++++-- tests/parser/features/iteration/test_for_range.py | 4 ++-- vyper/codegen/stmt.py | 4 +++- vyper/ir/compile_ir.py | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/control-structures.rst b/docs/control-structures.rst index fc8a472ff6..873135709a 100644 --- a/docs/control-structures.rst +++ b/docs/control-structures.rst @@ -271,16 +271,25 @@ Ranges are created using the ``range`` function. The following examples are vali ``STOP`` is a literal integer greater than zero. ``i`` begins as zero and increments by one until it is equal to ``STOP``. +.. code-block:: python + + for i in range(stop, bound=N): + ... + +Here, ``stop`` can be a variable with integer type, greater than zero. ``N`` must be a compile-time constant. ``i`` begins as zero and increments by one until it is equal to ``stop``. If ``stop`` is larger than ``N``, execution will revert at runtime. In certain cases, you may not have a guarantee that ``stop`` is less than ``N``, but still want to avoid the possibility of runtime reversion. To accomplish this, use the ``bound=`` keyword in combination with ``min(stop, N)`` as the argument to ``range``, like ``range(min(stop, N), bound=N)``. This is helpful for use cases like chunking up operations on larger arrays across multiple transactions. + +Another use of range can be with ``START`` and ``STOP`` bounds. + .. code-block:: python for i in range(START, STOP): ... -``START`` and ``STOP`` are literal integers, with ``STOP`` being a greater value than ``START``. ``i`` begins as ``START`` and increments by one until it is equal to ``STOP``. +Here, ``START`` and ``STOP`` are literal integers, with ``STOP`` being a greater value than ``START``. ``i`` begins as ``START`` and increments by one until it is equal to ``STOP``. .. code-block:: python for i in range(a, a + N): ... -``a`` is a variable with an integer type and ``N`` is a literal integer greater than zero. ``i`` begins as ``a`` and increments by one until it is equal to ``a + N``. +``a`` is a variable with an integer type and ``N`` is a literal integer greater than zero. ``i`` begins as ``a`` and increments by one until it is equal to ``a + N``. If ``a + N`` would overflow, execution will revert. diff --git a/tests/parser/features/iteration/test_for_range.py b/tests/parser/features/iteration/test_for_range.py index 395dd28231..ed6235d992 100644 --- a/tests/parser/features/iteration/test_for_range.py +++ b/tests/parser/features/iteration/test_for_range.py @@ -20,12 +20,12 @@ def test_range_bound(get_contract, assert_tx_failed): def repeat(n: uint256) -> uint256: x: uint256 = 0 for i in range(n, bound=6): - x += i + x += i + 1 return x """ c = get_contract(code) for n in range(7): - assert c.repeat(n) == sum(range(n)) + assert c.repeat(n) == sum(i + 1 for i in range(n)) # check codegen inserts assertion for n greater than bound assert_tx_failed(lambda: c.repeat(7)) diff --git a/vyper/codegen/stmt.py b/vyper/codegen/stmt.py index 3ecb0afdc3..c2951986c8 100644 --- a/vyper/codegen/stmt.py +++ b/vyper/codegen/stmt.py @@ -302,7 +302,9 @@ def _parse_For_range(self): loop_body.append(["mstore", iptr, i]) loop_body.append(parse_body(self.stmt.body, self.context)) - # NOTE: codegen for `repeat` inserts an assertion that rounds <= rounds_bound. + # NOTE: codegen for `repeat` inserts an assertion that + # (gt rounds_bound rounds). note this also covers the case where + # rounds < 0. # if we ever want to remove that, we need to manually add the assertion # where it makes sense. ir_node = IRnode.from_list( diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 7a3e97155b..1c4dc1ef7c 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -415,7 +415,7 @@ def _height_of(witharg): ) ) # stack: i, rounds, rounds_bound - # assert rounds <= rounds_bound + # assert 0 <= rounds <= rounds_bound (for rounds_bound < 2**255) # TODO this runtime assertion shouldn't fail for # internally generated repeats. o.extend(["DUP2", "GT"] + _assert_false()) From c913b2db0881a6f4e1c70b7929b713a6aab05c62 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 28 Sep 2023 18:32:04 -0700 Subject: [PATCH 10/17] chore: remove deadlines and reruns (#3630) for historical reasons, pytest ran with `--allow-reruns` when tests failed because tests would usually fail due to deadline errors. deadline errors have never indicated any meaningful issue with the compiler, they are just a somewhat unavoidable byproduct of the fact that we are running in a CI environment which has a lot of jitter. this commit changes the hypothesis deadline to `None` for the whole test suite, and removes the `--allow-reruns` parameter in the CI, which should make the test suite much more efficient when there are failures. --- .github/workflows/test.yml | 6 +++--- tests/ast/nodes/test_evaluate_binop_decimal.py | 4 ++-- tests/ast/nodes/test_evaluate_binop_int.py | 8 ++++---- tests/ast/nodes/test_evaluate_boolop.py | 4 ++-- tests/ast/nodes/test_evaluate_compare.py | 8 ++++---- tests/ast/nodes/test_evaluate_subscript.py | 2 +- tests/builtins/folding/test_abs.py | 4 ++-- tests/builtins/folding/test_addmod_mulmod.py | 2 +- tests/builtins/folding/test_bitwise.py | 8 ++++---- tests/builtins/folding/test_floor_ceil.py | 2 +- tests/builtins/folding/test_fold_as_wei_value.py | 4 ++-- tests/builtins/folding/test_keccak_sha.py | 6 +++--- tests/builtins/folding/test_min_max.py | 6 +++--- tests/builtins/folding/test_powmod.py | 2 +- tests/conftest.py | 6 ++++++ tests/fuzzing/test_exponents.py | 4 ++-- tests/grammar/test_grammar.py | 4 ++-- tests/parser/features/test_internal_call.py | 2 +- tests/parser/functions/test_slice.py | 4 ++-- tests/parser/test_call_graph_stability.py | 2 +- tests/parser/test_selector_table.py | 6 +++--- tests/parser/types/numbers/test_isqrt.py | 1 - tests/parser/types/numbers/test_sqrt.py | 2 -- tests/parser/types/test_bytes_zero_padding.py | 1 - 24 files changed, 50 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd78e2fff8..8d23368eb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,7 +104,7 @@ jobs: run: pip install tox - name: Run Tox - run: TOXENV=py${{ matrix.python-version[1] }} tox -r -- --optimize ${{ matrix.opt-mode }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} --reruns 10 --reruns-delay 1 -r aR tests/ + run: TOXENV=py${{ matrix.python-version[1] }} tox -r -- --optimize ${{ matrix.opt-mode }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} -r aR tests/ - name: Upload Coverage uses: codecov/codecov-action@v1 @@ -148,12 +148,12 @@ jobs: # fetch test durations # NOTE: if the tests get poorly distributed, run this and commit the resulting `.test_durations` file to the `vyper-test-durations` repo. - # `TOXENV=fuzzing tox -r -- --store-durations --reruns 10 --reruns-delay 1 -r aR tests/` + # `TOXENV=fuzzing tox -r -- --store-durations -r aR tests/` - name: Fetch test-durations run: curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/5982755ee8459f771f2e8622427c36494646e1dd/test_durations" -o .test_durations - name: Run Tox - run: TOXENV=fuzzing tox -r -- --splits 60 --group ${{ matrix.group }} --splitting-algorithm least_duration --reruns 10 --reruns-delay 1 -r aR tests/ + run: TOXENV=fuzzing tox -r -- --splits 60 --group ${{ matrix.group }} --splitting-algorithm least_duration -r aR tests/ - name: Upload Coverage uses: codecov/codecov-action@v1 diff --git a/tests/ast/nodes/test_evaluate_binop_decimal.py b/tests/ast/nodes/test_evaluate_binop_decimal.py index 3c8ba0888c..5c9956caba 100644 --- a/tests/ast/nodes/test_evaluate_binop_decimal.py +++ b/tests/ast/nodes/test_evaluate_binop_decimal.py @@ -13,7 +13,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=None) +@settings(max_examples=50) @given(left=st_decimals, right=st_decimals) @example(left=Decimal("0.9999999999"), right=Decimal("0.0000000001")) @example(left=Decimal("0.0000000001"), right=Decimal("0.9999999999")) @@ -52,7 +52,7 @@ def test_binop_pow(): @pytest.mark.fuzzing -@settings(max_examples=50, deadline=None) +@settings(max_examples=50) @given( values=st.lists(st_decimals, min_size=2, max_size=10), ops=st.lists(st.sampled_from("+-*/%"), min_size=11, max_size=11), diff --git a/tests/ast/nodes/test_evaluate_binop_int.py b/tests/ast/nodes/test_evaluate_binop_int.py index d632a95461..80c9381c0f 100644 --- a/tests/ast/nodes/test_evaluate_binop_int.py +++ b/tests/ast/nodes/test_evaluate_binop_int.py @@ -9,7 +9,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st_int32, right=st_int32) @example(left=1, right=1) @example(left=1, right=-1) @@ -42,7 +42,7 @@ def foo(a: int128, b: int128) -> int128: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st_uint64, right=st_uint64) @pytest.mark.parametrize("op", "+-*/%") def test_binop_uint256(get_contract, assert_tx_failed, op, left, right): @@ -69,7 +69,7 @@ def foo(a: uint256, b: uint256) -> uint256: @pytest.mark.xfail(reason="need to implement safe exponentiation logic") @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st.integers(min_value=2, max_value=245), right=st.integers(min_value=0, max_value=16)) @example(left=0, right=0) @example(left=0, right=1) @@ -89,7 +89,7 @@ def foo(a: uint256, b: uint256) -> uint256: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given( values=st.lists(st.integers(min_value=-256, max_value=256), min_size=2, max_size=10), ops=st.lists(st.sampled_from("+-*/%"), min_size=11, max_size=11), diff --git a/tests/ast/nodes/test_evaluate_boolop.py b/tests/ast/nodes/test_evaluate_boolop.py index 6bd9ecc6cb..8b70537c39 100644 --- a/tests/ast/nodes/test_evaluate_boolop.py +++ b/tests/ast/nodes/test_evaluate_boolop.py @@ -8,7 +8,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(values=st.lists(st.booleans(), min_size=2, max_size=10)) @pytest.mark.parametrize("comparator", ["and", "or"]) def test_boolop_simple(get_contract, values, comparator): @@ -32,7 +32,7 @@ def foo({input_value}) -> bool: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given( values=st.lists(st.booleans(), min_size=2, max_size=10), comparators=st.lists(st.sampled_from(["and", "or"]), min_size=11, max_size=11), diff --git a/tests/ast/nodes/test_evaluate_compare.py b/tests/ast/nodes/test_evaluate_compare.py index 9ff5cea338..07f8e70de6 100644 --- a/tests/ast/nodes/test_evaluate_compare.py +++ b/tests/ast/nodes/test_evaluate_compare.py @@ -8,7 +8,7 @@ # TODO expand to all signed types @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st.integers(), right=st.integers()) @pytest.mark.parametrize("op", ["==", "!=", "<", "<=", ">=", ">"]) def test_compare_eq_signed(get_contract, op, left, right): @@ -28,7 +28,7 @@ def foo(a: int128, b: int128) -> bool: # TODO expand to all unsigned types @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st.integers(min_value=0), right=st.integers(min_value=0)) @pytest.mark.parametrize("op", ["==", "!=", "<", "<=", ">=", ">"]) def test_compare_eq_unsigned(get_contract, op, left, right): @@ -47,7 +47,7 @@ def foo(a: uint128, b: uint128) -> bool: @pytest.mark.fuzzing -@settings(max_examples=20, deadline=1000) +@settings(max_examples=20) @given(left=st.integers(), right=st.lists(st.integers(), min_size=1, max_size=16)) def test_compare_in(left, right, get_contract): source = f""" @@ -76,7 +76,7 @@ def bar(a: int128) -> bool: @pytest.mark.fuzzing -@settings(max_examples=20, deadline=1000) +@settings(max_examples=20) @given(left=st.integers(), right=st.lists(st.integers(), min_size=1, max_size=16)) def test_compare_not_in(left, right, get_contract): source = f""" diff --git a/tests/ast/nodes/test_evaluate_subscript.py b/tests/ast/nodes/test_evaluate_subscript.py index 3c0fa5d16d..ca50a076a5 100644 --- a/tests/ast/nodes/test_evaluate_subscript.py +++ b/tests/ast/nodes/test_evaluate_subscript.py @@ -6,7 +6,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given( idx=st.integers(min_value=0, max_value=9), array=st.lists(st.integers(), min_size=10, max_size=10), diff --git a/tests/builtins/folding/test_abs.py b/tests/builtins/folding/test_abs.py index 58f861ed0c..1c919d7826 100644 --- a/tests/builtins/folding/test_abs.py +++ b/tests/builtins/folding/test_abs.py @@ -8,7 +8,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(a=st.integers(min_value=-(2**255) + 1, max_value=2**255 - 1)) @example(a=0) def test_abs(get_contract, a): @@ -27,7 +27,7 @@ def foo(a: int256) -> int256: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(a=st.integers(min_value=2**255, max_value=2**256 - 1)) def test_abs_upper_bound_folding(get_contract, a): source = f""" diff --git a/tests/builtins/folding/test_addmod_mulmod.py b/tests/builtins/folding/test_addmod_mulmod.py index 0514dea18a..33dcc62984 100644 --- a/tests/builtins/folding/test_addmod_mulmod.py +++ b/tests/builtins/folding/test_addmod_mulmod.py @@ -9,7 +9,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(a=st_uint256, b=st_uint256, c=st_uint256) @pytest.mark.parametrize("fn_name", ["uint256_addmod", "uint256_mulmod"]) def test_modmath(get_contract, a, b, c, fn_name): diff --git a/tests/builtins/folding/test_bitwise.py b/tests/builtins/folding/test_bitwise.py index d28e482589..63e733644f 100644 --- a/tests/builtins/folding/test_bitwise.py +++ b/tests/builtins/folding/test_bitwise.py @@ -14,7 +14,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("op", ["&", "|", "^"]) @given(a=st_uint256, b=st_uint256) def test_bitwise_ops(get_contract, a, b, op): @@ -34,7 +34,7 @@ def foo(a: uint256, b: uint256) -> uint256: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("op", ["<<", ">>"]) @given(a=st_uint256, b=st.integers(min_value=0, max_value=256)) def test_bitwise_shift_unsigned(get_contract, a, b, op): @@ -64,7 +64,7 @@ def foo(a: uint256, b: uint256) -> uint256: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("op", ["<<", ">>"]) @given(a=st_sint256, b=st.integers(min_value=0, max_value=256)) def test_bitwise_shift_signed(get_contract, a, b, op): @@ -92,7 +92,7 @@ def foo(a: int256, b: uint256) -> int256: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(value=st_uint256) def test_bitwise_not(get_contract, value): source = """ diff --git a/tests/builtins/folding/test_floor_ceil.py b/tests/builtins/folding/test_floor_ceil.py index 763f8fec63..87db23889a 100644 --- a/tests/builtins/folding/test_floor_ceil.py +++ b/tests/builtins/folding/test_floor_ceil.py @@ -13,7 +13,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(value=st_decimals) @example(value=Decimal("0.9999999999")) @example(value=Decimal("0.0000000001")) diff --git a/tests/builtins/folding/test_fold_as_wei_value.py b/tests/builtins/folding/test_fold_as_wei_value.py index 11d23bd3bf..210ab51f0d 100644 --- a/tests/builtins/folding/test_fold_as_wei_value.py +++ b/tests/builtins/folding/test_fold_as_wei_value.py @@ -19,7 +19,7 @@ @pytest.mark.fuzzing -@settings(max_examples=10, deadline=1000) +@settings(max_examples=10) @given(value=st_decimals) @pytest.mark.parametrize("denom", denoms) def test_decimal(get_contract, value, denom): @@ -38,7 +38,7 @@ def foo(a: decimal) -> uint256: @pytest.mark.fuzzing -@settings(max_examples=10, deadline=1000) +@settings(max_examples=10) @given(value=st.integers(min_value=0, max_value=2**128)) @pytest.mark.parametrize("denom", denoms) def test_integer(get_contract, value, denom): diff --git a/tests/builtins/folding/test_keccak_sha.py b/tests/builtins/folding/test_keccak_sha.py index 8e283566de..a2fe460dd1 100644 --- a/tests/builtins/folding/test_keccak_sha.py +++ b/tests/builtins/folding/test_keccak_sha.py @@ -10,7 +10,7 @@ @pytest.mark.fuzzing @given(value=st.text(alphabet=alphabet, min_size=0, max_size=100)) -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"]) def test_string(get_contract, value, fn_name): source = f""" @@ -29,7 +29,7 @@ def foo(a: String[100]) -> bytes32: @pytest.mark.fuzzing @given(value=st.binary(min_size=0, max_size=100)) -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"]) def test_bytes(get_contract, value, fn_name): source = f""" @@ -48,7 +48,7 @@ def foo(a: Bytes[100]) -> bytes32: @pytest.mark.fuzzing @given(value=st.binary(min_size=1, max_size=100)) -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"]) def test_hex(get_contract, value, fn_name): source = f""" diff --git a/tests/builtins/folding/test_min_max.py b/tests/builtins/folding/test_min_max.py index e2d33237ca..309f7519c0 100644 --- a/tests/builtins/folding/test_min_max.py +++ b/tests/builtins/folding/test_min_max.py @@ -18,7 +18,7 @@ @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st_decimals, right=st_decimals) @pytest.mark.parametrize("fn_name", ["min", "max"]) def test_decimal(get_contract, left, right, fn_name): @@ -37,7 +37,7 @@ def foo(a: decimal, b: decimal) -> decimal: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st_int128, right=st_int128) @pytest.mark.parametrize("fn_name", ["min", "max"]) def test_int128(get_contract, left, right, fn_name): @@ -56,7 +56,7 @@ def foo(a: int128, b: int128) -> int128: @pytest.mark.fuzzing -@settings(max_examples=50, deadline=1000) +@settings(max_examples=50) @given(left=st_uint256, right=st_uint256) @pytest.mark.parametrize("fn_name", ["min", "max"]) def test_min_uint256(get_contract, left, right, fn_name): diff --git a/tests/builtins/folding/test_powmod.py b/tests/builtins/folding/test_powmod.py index fdc0e300ab..8667ec93fd 100644 --- a/tests/builtins/folding/test_powmod.py +++ b/tests/builtins/folding/test_powmod.py @@ -9,7 +9,7 @@ @pytest.mark.fuzzing -@settings(max_examples=100, deadline=1000) +@settings(max_examples=100) @given(a=st_uint256, b=st_uint256) def test_powmod_uint256(get_contract, a, b): source = """ diff --git a/tests/conftest.py b/tests/conftest.py index d519ca3100..c9d3f794a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import logging from functools import wraps +import hypothesis import pytest from eth_tester import EthereumTester, PyEVMBackend from eth_utils import setup_DEBUG2_logging @@ -23,6 +24,11 @@ ############ +# disable hypothesis deadline globally +hypothesis.settings.register_profile("ci", deadline=None) +hypothesis.settings.load_profile("ci") + + def set_evm_verbose_logging(): logger = logging.getLogger("eth.vm.computation.Computation") setup_DEBUG2_logging() diff --git a/tests/fuzzing/test_exponents.py b/tests/fuzzing/test_exponents.py index 29c1f198ed..5726e4c1ca 100644 --- a/tests/fuzzing/test_exponents.py +++ b/tests/fuzzing/test_exponents.py @@ -92,7 +92,7 @@ def foo(a: int16) -> int16: @example(a=2**127 - 1) # 256 bits @example(a=2**256 - 1) -@settings(max_examples=200, deadline=1000) +@settings(max_examples=200) def test_max_exp(get_contract, assert_tx_failed, a): code = f""" @external @@ -127,7 +127,7 @@ def foo(b: uint256) -> uint256: @example(a=2**63 - 1) # 128 bits @example(a=2**127 - 1) -@settings(max_examples=200, deadline=1000) +@settings(max_examples=200) def test_max_exp_int128(get_contract, assert_tx_failed, a): code = f""" @external diff --git a/tests/grammar/test_grammar.py b/tests/grammar/test_grammar.py index d665ca2544..aa0286cfa5 100644 --- a/tests/grammar/test_grammar.py +++ b/tests/grammar/test_grammar.py @@ -4,7 +4,7 @@ import hypothesis import hypothesis.strategies as st import pytest -from hypothesis import HealthCheck, assume, given +from hypothesis import assume, given from hypothesis.extra.lark import LarkStrategy from vyper.ast import Module, parse_to_ast @@ -103,7 +103,7 @@ def has_no_docstrings(c): @pytest.mark.fuzzing @given(code=from_grammar().filter(lambda c: utf8_encodable(c))) -@hypothesis.settings(deadline=400, max_examples=500, suppress_health_check=(HealthCheck.too_slow,)) +@hypothesis.settings(max_examples=500) def test_grammar_bruteforce(code): if utf8_encodable(code): _, _, reformatted_code = pre_parse(code + "\n") diff --git a/tests/parser/features/test_internal_call.py b/tests/parser/features/test_internal_call.py index d7a41acbc0..f10d22ec99 100644 --- a/tests/parser/features/test_internal_call.py +++ b/tests/parser/features/test_internal_call.py @@ -669,7 +669,7 @@ def test_internal_call_kwargs(get_contract, typ1, strategy1, typ2, strategy2): # GHSA-ph9x-4vc9-m39g @given(kwarg1=strategy1, default1=strategy1, kwarg2=strategy2, default2=strategy2) - @settings(deadline=None, max_examples=5) # len(cases) * len(cases) * 5 * 5 + @settings(max_examples=5) # len(cases) * len(cases) * 5 * 5 def fuzz(kwarg1, kwarg2, default1, default2): code = f""" @internal diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py index 6229b47921..3090dafda0 100644 --- a/tests/parser/functions/test_slice.py +++ b/tests/parser/functions/test_slice.py @@ -36,7 +36,7 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]: @pytest.mark.parametrize("literal_length", (True, False)) @pytest.mark.parametrize("opt_level", list(OptimizationLevel)) @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) -@settings(max_examples=100, deadline=None) +@settings(max_examples=100) @pytest.mark.fuzzing def test_slice_immutable( get_contract, @@ -90,7 +90,7 @@ def _get_contract(): @pytest.mark.parametrize("literal_length", (True, False)) @pytest.mark.parametrize("opt_level", list(OptimizationLevel)) @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) -@settings(max_examples=100, deadline=None) +@settings(max_examples=100) @pytest.mark.fuzzing def test_slice_bytes( get_contract, diff --git a/tests/parser/test_call_graph_stability.py b/tests/parser/test_call_graph_stability.py index a6193610e2..4c85c330f3 100644 --- a/tests/parser/test_call_graph_stability.py +++ b/tests/parser/test_call_graph_stability.py @@ -15,7 +15,7 @@ def _valid_identifier(attr): # random names for functions -@settings(max_examples=20, deadline=None) +@settings(max_examples=20) @given( st.lists( st.tuples( diff --git a/tests/parser/test_selector_table.py b/tests/parser/test_selector_table.py index 180c0266bf..161cd480fd 100644 --- a/tests/parser/test_selector_table.py +++ b/tests/parser/test_selector_table.py @@ -446,7 +446,7 @@ def aILR4U1Z()->uint256: seed=st.integers(min_value=0, max_value=2**64 - 1), ) @pytest.mark.fuzzing -@settings(max_examples=10, deadline=None) +@settings(max_examples=10) def test_sparse_jumptable_probe_depth(n_methods, seed): sigs = [f"foo{i + seed}()" for i in range(n_methods)] _, buckets = generate_sparse_jumptable_buckets(sigs) @@ -466,7 +466,7 @@ def test_sparse_jumptable_probe_depth(n_methods, seed): seed=st.integers(min_value=0, max_value=2**64 - 1), ) @pytest.mark.fuzzing -@settings(max_examples=10, deadline=None) +@settings(max_examples=10) def test_dense_jumptable_bucket_size(n_methods, seed): sigs = [f"foo{i + seed}()" for i in range(n_methods)] n = len(sigs) @@ -537,7 +537,7 @@ def foo{func_id}({args}) -> uint256: """ @given(_input=generate_methods(max_calldata_bytes)) - @settings(max_examples=125, deadline=None) + @settings(max_examples=125) def _test(_input): max_default_args, default_fn_mutability, methods = _input diff --git a/tests/parser/types/numbers/test_isqrt.py b/tests/parser/types/numbers/test_isqrt.py index ce26d24d06..b734323a6e 100644 --- a/tests/parser/types/numbers/test_isqrt.py +++ b/tests/parser/types/numbers/test_isqrt.py @@ -119,7 +119,6 @@ def test(a: uint256) -> (uint256, uint256, uint256, uint256, uint256, String[100 @hypothesis.example(2704) @hypothesis.example(110889) @hypothesis.example(32239684) -@hypothesis.settings(deadline=1000) def test_isqrt_valid_range(isqrt_contract, value): vyper_isqrt = isqrt_contract.test(value) actual_isqrt = math.isqrt(value) diff --git a/tests/parser/types/numbers/test_sqrt.py b/tests/parser/types/numbers/test_sqrt.py index df1ed0539c..020a79e7ef 100644 --- a/tests/parser/types/numbers/test_sqrt.py +++ b/tests/parser/types/numbers/test_sqrt.py @@ -145,7 +145,6 @@ def test_sqrt_bounds(sqrt_contract, value): ) @hypothesis.example(value=Decimal(SizeLimits.MAX_INT128)) @hypothesis.example(value=Decimal(0)) -@hypothesis.settings(deadline=1000) def test_sqrt_valid_range(sqrt_contract, value): vyper_sqrt = sqrt_contract.test(value) actual_sqrt = decimal_sqrt(value) @@ -158,7 +157,6 @@ def test_sqrt_valid_range(sqrt_contract, value): min_value=Decimal(SizeLimits.MIN_INT128), max_value=Decimal("-1E10"), places=DECIMAL_PLACES ) ) -@hypothesis.settings(deadline=400) @hypothesis.example(value=Decimal(SizeLimits.MIN_INT128)) @hypothesis.example(value=Decimal("-1E10")) def test_sqrt_invalid_range(sqrt_contract, value): diff --git a/tests/parser/types/test_bytes_zero_padding.py b/tests/parser/types/test_bytes_zero_padding.py index ee938fdffb..f9fcf37b25 100644 --- a/tests/parser/types/test_bytes_zero_padding.py +++ b/tests/parser/types/test_bytes_zero_padding.py @@ -26,7 +26,6 @@ def get_count(counter: uint256) -> Bytes[24]: @pytest.mark.fuzzing @hypothesis.given(value=hypothesis.strategies.integers(min_value=0, max_value=2**64)) -@hypothesis.settings(deadline=400) def test_zero_pad_range(little_endian_contract, value): actual_bytes = value.to_bytes(8, byteorder="little") contract_bytes = little_endian_contract.get_count(value) From 8aae7cd6b86c15978bdfa16d5a6e3ca273121107 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 2 Oct 2023 19:20:19 -0700 Subject: [PATCH 11/17] feat: disallow invalid pragmas (#3634) this commit prevents typos (ex. `# pragma evm-versionn ...`) from getting unexpectedly ignored. it also fixes an issue with the `evm_version` pragma getting properly propagated into `CompilerData.settings`. it also fixes a misnamed test function, and adds some unit tests to check that `evm_version` gets properly propagated into `CompilerData.settings`. commit summary: * fix `evm_version` passing into compiler data * fix `compile_code()` * add tests for `CompilerData.settings` * add tests for invalid pragmas --- tests/ast/test_pre_parser.py | 70 ++++++++++++++++++++++++++++++++---- vyper/ast/pre_parser.py | 7 ++-- vyper/compiler/README.md | 6 ++-- vyper/compiler/__init__.py | 22 ++++++------ vyper/compiler/phases.py | 5 +++ 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/tests/ast/test_pre_parser.py b/tests/ast/test_pre_parser.py index 5427532c16..3d072674f6 100644 --- a/tests/ast/test_pre_parser.py +++ b/tests/ast/test_pre_parser.py @@ -1,8 +1,9 @@ import pytest from vyper.ast.pre_parser import pre_parse, validate_version_pragma +from vyper.compiler.phases import CompilerData from vyper.compiler.settings import OptimizationLevel, Settings -from vyper.exceptions import VersionException +from vyper.exceptions import StructureException, VersionException SRC_LINE = (1, 0) # Dummy source line COMPILER_VERSION = "0.1.1" @@ -96,43 +97,50 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): """ """, Settings(), + Settings(optimize=OptimizationLevel.GAS), ), ( """ #pragma optimize codesize """, Settings(optimize=OptimizationLevel.CODESIZE), + None, ), ( """ #pragma optimize none """, Settings(optimize=OptimizationLevel.NONE), + None, ), ( """ #pragma optimize gas """, Settings(optimize=OptimizationLevel.GAS), + None, ), ( """ #pragma version 0.3.10 """, Settings(compiler_version="0.3.10"), + Settings(optimize=OptimizationLevel.GAS), ), ( """ #pragma evm-version shanghai """, Settings(evm_version="shanghai"), + Settings(evm_version="shanghai", optimize=OptimizationLevel.GAS), ), ( """ #pragma optimize codesize #pragma evm-version shanghai """, - Settings(evm_version="shanghai", optimize=OptimizationLevel.GAS), + Settings(evm_version="shanghai", optimize=OptimizationLevel.CODESIZE), + None, ), ( """ @@ -140,6 +148,7 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): #pragma evm-version shanghai """, Settings(evm_version="shanghai", compiler_version="0.3.10"), + Settings(evm_version="shanghai", optimize=OptimizationLevel.GAS), ), ( """ @@ -147,6 +156,7 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): #pragma optimize gas """, Settings(compiler_version="0.3.10", optimize=OptimizationLevel.GAS), + Settings(optimize=OptimizationLevel.GAS), ), ( """ @@ -155,11 +165,59 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): #pragma optimize gas """, Settings(compiler_version="0.3.10", optimize=OptimizationLevel.GAS, evm_version="shanghai"), + Settings(optimize=OptimizationLevel.GAS, evm_version="shanghai"), ), ] -@pytest.mark.parametrize("code, expected_pragmas", pragma_examples) -def parse_pragmas(code, expected_pragmas): - pragmas, _, _ = pre_parse(code) - assert pragmas == expected_pragmas +@pytest.mark.parametrize("code, pre_parse_settings, compiler_data_settings", pragma_examples) +def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_version): + mock_version("0.3.10") + settings, _, _ = pre_parse(code) + + assert settings == pre_parse_settings + + compiler_data = CompilerData(code) + + # check what happens after CompilerData constructor + if compiler_data_settings is None: + # None is sentinel here meaning that nothing changed + compiler_data_settings = pre_parse_settings + + assert compiler_data.settings == compiler_data_settings + + +invalid_pragmas = [ + # evm-versionnn + """ +# pragma evm-versionnn cancun + """, + # bad fork name + """ +# pragma evm-version cancunn + """, + # oppptimize + """ +# pragma oppptimize codesize + """, + # ggas + """ +# pragma optimize ggas + """, + # double specified + """ +# pragma optimize gas +# pragma optimize codesize + """, + # double specified + """ +# pragma evm-version cancun +# pragma evm-version shanghai + """, +] + + +@pytest.mark.parametrize("code", invalid_pragmas) +def test_invalid_pragma(code): + with pytest.raises(StructureException): + pre_parse(code) diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 0ead889787..9d96efea5e 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -111,7 +111,7 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, str]: validate_version_pragma(compiler_version, start) settings.compiler_version = compiler_version - if pragma.startswith("optimize "): + elif pragma.startswith("optimize "): if settings.optimize is not None: raise StructureException("pragma optimize specified twice!", start) try: @@ -119,7 +119,7 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, str]: settings.optimize = OptimizationLevel.from_string(mode) except ValueError: raise StructureException(f"Invalid optimization mode `{mode}`", start) - if pragma.startswith("evm-version "): + elif pragma.startswith("evm-version "): if settings.evm_version is not None: raise StructureException("pragma evm-version specified twice!", start) evm_version = pragma.removeprefix("evm-version").strip() @@ -127,6 +127,9 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, str]: raise StructureException("Invalid evm version: `{evm_version}`", start) settings.evm_version = evm_version + else: + raise StructureException(f"Unknown pragma `{pragma.split()[0]}`") + if typ == NAME and string in ("class", "yield"): raise SyntaxException( f"The `{string}` keyword is not allowed. ", code, start[0], start[1] diff --git a/vyper/compiler/README.md b/vyper/compiler/README.md index d6b55fdd82..eb70750a2b 100644 --- a/vyper/compiler/README.md +++ b/vyper/compiler/README.md @@ -51,11 +51,9 @@ for specific implementation details. [`vyper.compiler.compile_codes`](__init__.py) is the main user-facing function for generating compiler output from Vyper source. The process is as follows: -1. The `@evm_wrapper` decorator sets the target EVM version in -[`opcodes.py`](../evm/opcodes.py). -2. A [`CompilerData`](phases.py) object is created for each contract to be compiled. +1. A [`CompilerData`](phases.py) object is created for each contract to be compiled. This object uses `@property` methods to trigger phases of the compiler as required. -3. Functions in [`output.py`](output.py) generate the requested outputs from the +2. Functions in [`output.py`](output.py) generate the requested outputs from the compiler data. ## Design diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 0b3c0d8191..b1c4201361 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -120,17 +120,17 @@ def compile_codes( # make IR output the same between runs codegen.reset_names() - with anchor_evm_version(settings.evm_version): - compiler_data = CompilerData( - source_code, - contract_name, - interfaces, - source_id, - settings, - storage_layout_override, - show_gas_estimates, - no_bytecode_metadata, - ) + compiler_data = CompilerData( + source_code, + contract_name, + interfaces, + source_id, + settings, + storage_layout_override, + show_gas_estimates, + no_bytecode_metadata, + ) + with anchor_evm_version(compiler_data.settings.evm_version): for output_format in output_formats[contract_name]: if output_format not in OUTPUT_FORMATS: raise ValueError(f"Unsupported format type {repr(output_format)}") diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index a1c7342320..5ddf071caf 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -88,9 +88,12 @@ def __init__( self.no_bytecode_metadata = no_bytecode_metadata self.settings = settings or Settings() + _ = self._generate_ast # force settings to be calculated + @cached_property def _generate_ast(self): settings, ast = generate_ast(self.source_code, self.source_id, self.contract_name) + # validate the compiler settings # XXX: this is a bit ugly, clean up later if settings.evm_version is not None: @@ -117,6 +120,8 @@ def _generate_ast(self): if self.settings.optimize is None: self.settings.optimize = OptimizationLevel.default() + # note self.settings.compiler_version is erased here as it is + # not used after pre-parsing return ast @cached_property From e9c16e40dd11ba21ba817ff0da78eaeab744fd39 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 3 Oct 2023 16:36:58 -0700 Subject: [PATCH 12/17] fix: block `mload` merging when src and dst overlap (#3635) this commit fixes an optimization bug when the target architecture has the `mcopy` instruction (i.e. `cancun` or later). the bug was introduced in 5dc3ac7. specifically, the `merge_mload` step can incorrectly merge `mload`/`mstore` sequences (into `mcopy`) when the source and destination buffers overlap, and the destination buffer is "ahead of" (i.e. greater than) the source buffer. this commit fixes the issue by blocking the optimization in these cases, and adds unit and functional tests demonstrating the correct behavior. --------- Co-authored-by: Robert Chen --- tests/compiler/ir/test_optimize_ir.py | 107 +++++++++++++++++++++++ tests/parser/features/test_assignment.py | 60 +++++++++++++ vyper/ir/optimizer.py | 9 +- 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/tests/compiler/ir/test_optimize_ir.py b/tests/compiler/ir/test_optimize_ir.py index 1466166501..cb46ba238d 100644 --- a/tests/compiler/ir/test_optimize_ir.py +++ b/tests/compiler/ir/test_optimize_ir.py @@ -1,9 +1,13 @@ import pytest from vyper.codegen.ir_node import IRnode +from vyper.evm.opcodes import EVM_VERSIONS, anchor_evm_version from vyper.exceptions import StaticAssertionException from vyper.ir import optimizer +POST_CANCUN = {k: v for k, v in EVM_VERSIONS.items() if v >= EVM_VERSIONS["cancun"]} + + optimize_list = [ (["eq", 1, 2], [0]), (["lt", 1, 2], [1]), @@ -272,3 +276,106 @@ def test_operator_set_values(): assert optimizer.COMPARISON_OPS == {"lt", "gt", "le", "ge", "slt", "sgt", "sle", "sge"} assert optimizer.STRICT_COMPARISON_OPS == {"lt", "gt", "slt", "sgt"} assert optimizer.UNSTRICT_COMPARISON_OPS == {"le", "ge", "sle", "sge"} + + +mload_merge_list = [ + # copy "backward" with no overlap between src and dst buffers, + # OK to become mcopy + ( + ["seq", ["mstore", 32, ["mload", 128]], ["mstore", 64, ["mload", 160]]], + ["mcopy", 32, 128, 64], + ), + # copy with overlap "backwards", OK to become mcopy + (["seq", ["mstore", 32, ["mload", 64]], ["mstore", 64, ["mload", 96]]], ["mcopy", 32, 64, 64]), + # "stationary" overlap (i.e. a no-op mcopy), OK to become mcopy + (["seq", ["mstore", 32, ["mload", 32]], ["mstore", 64, ["mload", 64]]], ["mcopy", 32, 32, 64]), + # copy "forward" with no overlap, OK to become mcopy + (["seq", ["mstore", 64, ["mload", 0]], ["mstore", 96, ["mload", 32]]], ["mcopy", 64, 0, 64]), + # copy "forwards" with overlap by one word, must NOT become mcopy + (["seq", ["mstore", 64, ["mload", 32]], ["mstore", 96, ["mload", 64]]], None), + # check "forward" overlap by one byte, must NOT become mcopy + (["seq", ["mstore", 64, ["mload", 1]], ["mstore", 96, ["mload", 33]]], None), + # check "forward" overlap by one byte again, must NOT become mcopy + (["seq", ["mstore", 63, ["mload", 0]], ["mstore", 95, ["mload", 32]]], None), + # copy 3 words with partial overlap "forwards", partially becomes mcopy + # (2 words are mcopied and 1 word is mload/mstored + ( + [ + "seq", + ["mstore", 96, ["mload", 32]], + ["mstore", 128, ["mload", 64]], + ["mstore", 160, ["mload", 96]], + ], + ["seq", ["mcopy", 96, 32, 64], ["mstore", 160, ["mload", 96]]], + ), + # copy 4 words with partial overlap "forwards", becomes 2 mcopies of 2 words each + ( + [ + "seq", + ["mstore", 96, ["mload", 32]], + ["mstore", 128, ["mload", 64]], + ["mstore", 160, ["mload", 96]], + ["mstore", 192, ["mload", 128]], + ], + ["seq", ["mcopy", 96, 32, 64], ["mcopy", 160, 96, 64]], + ), + # copy 4 words with 1 byte of overlap, must NOT become mcopy + ( + [ + "seq", + ["mstore", 96, ["mload", 33]], + ["mstore", 128, ["mload", 65]], + ["mstore", 160, ["mload", 97]], + ["mstore", 192, ["mload", 129]], + ], + None, + ), + # Ensure only sequential mstore + mload sequences are optimized + ( + [ + "seq", + ["mstore", 0, ["mload", 32]], + ["sstore", 0, ["calldataload", 4]], + ["mstore", 32, ["mload", 64]], + ], + None, + ), + # not-word aligned optimizations (not overlap) + (["seq", ["mstore", 0, ["mload", 1]], ["mstore", 32, ["mload", 33]]], ["mcopy", 0, 1, 64]), + # not-word aligned optimizations (overlap) + (["seq", ["mstore", 1, ["mload", 0]], ["mstore", 33, ["mload", 32]]], None), + # not-word aligned optimizations (overlap and not-overlap) + ( + [ + "seq", + ["mstore", 0, ["mload", 1]], + ["mstore", 32, ["mload", 33]], + ["mstore", 1, ["mload", 0]], + ["mstore", 33, ["mload", 32]], + ], + ["seq", ["mcopy", 0, 1, 64], ["mstore", 1, ["mload", 0]], ["mstore", 33, ["mload", 32]]], + ), + # overflow test + ( + [ + "seq", + ["mstore", 2**256 - 1 - 31 - 32, ["mload", 0]], + ["mstore", 2**256 - 1 - 31, ["mload", 32]], + ], + ["mcopy", 2**256 - 1 - 31 - 32, 0, 64], + ), +] + + +@pytest.mark.parametrize("ir", mload_merge_list) +@pytest.mark.parametrize("evm_version", list(POST_CANCUN.keys())) +def test_mload_merge(ir, evm_version): + with anchor_evm_version(evm_version): + optimized = optimizer.optimize(IRnode.from_list(ir[0])) + if ir[1] is None: + # no-op, assert optimizer does nothing + expected = IRnode.from_list(ir[0]) + else: + expected = IRnode.from_list(ir[1]) + + assert optimized == expected diff --git a/tests/parser/features/test_assignment.py b/tests/parser/features/test_assignment.py index e550f60541..35b008a8ba 100644 --- a/tests/parser/features/test_assignment.py +++ b/tests/parser/features/test_assignment.py @@ -442,3 +442,63 @@ def bug(p: Point) -> Point: """ c = get_contract(code) assert c.bug((1, 2)) == (2, 1) + + +mload_merge_codes = [ + ( + """ +@external +def foo() -> uint256[4]: + # copy "backwards" + xs: uint256[4] = [1, 2, 3, 4] + +# dst < src + xs[0] = xs[1] + xs[1] = xs[2] + xs[2] = xs[3] + + return xs + """, + [2, 3, 4, 4], + ), + ( + """ +@external +def foo() -> uint256[4]: + # copy "forwards" + xs: uint256[4] = [1, 2, 3, 4] + +# src < dst + xs[1] = xs[0] + xs[2] = xs[1] + xs[3] = xs[2] + + return xs + """, + [1, 1, 1, 1], + ), + ( + """ +@external +def foo() -> uint256[5]: + # partial "forward" copy + xs: uint256[5] = [1, 2, 3, 4, 5] + +# src < dst + xs[2] = xs[0] + xs[3] = xs[1] + xs[4] = xs[2] + + return xs + """, + [1, 2, 1, 2, 1], + ), +] + + +# functional test that mload merging does not occur when source and dest +# buffers overlap. (note: mload merging only applies after cancun) +@pytest.mark.parametrize("code,expected_result", mload_merge_codes) +def test_mcopy_overlap(get_contract, code, expected_result): + c = get_contract(code) + assert c.foo() == expected_result diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index 08c2168381..8df4bbac2d 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -662,10 +662,10 @@ def _rewrite_mstore_dload(argz): def _merge_mload(argz): if not version_check(begin="cancun"): return False - return _merge_load(argz, "mload", "mcopy") + return _merge_load(argz, "mload", "mcopy", allow_overlap=False) -def _merge_load(argz, _LOAD, _COPY): +def _merge_load(argz, _LOAD, _COPY, allow_overlap=True): # look for sequential operations copying from X to Y # and merge them into a single copy operation changed = False @@ -689,9 +689,14 @@ def _merge_load(argz, _LOAD, _COPY): initial_dst_offset = dst_offset initial_src_offset = src_offset idx = i + + # dst and src overlap, discontinue the optimization + has_overlap = initial_src_offset < initial_dst_offset < src_offset + 32 + if ( initial_dst_offset + total_length == dst_offset and initial_src_offset + total_length == src_offset + and (allow_overlap or not has_overlap) ): mstore_nodes.append(ir_node) total_length += 32 From 9136169468f317a53b4e7448389aa315f90b95ba Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 4 Oct 2023 06:16:44 -0700 Subject: [PATCH 13/17] docs: v0.3.10 release (#3629) --- docs/release-notes.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index da86c5c0ce..64199bc860 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -14,14 +14,10 @@ Release Notes for advisory links: :'<,'>s/\v(https:\/\/github.com\/vyperlang\/vyper\/security\/advisories\/)([-A-Za-z0-9]+)/(`\2 <\1\2>`_)/g -.. - v0.3.10 ("Black Adder") - *********************** - -v0.3.10rc1 -********** +v0.3.10 ("Black Adder") +*********************** -Date released: 2023-09-06 +Date released: 2023-10-04 ========================= v0.3.10 is a performance focused release. It adds a ``codesize`` optimization mode (`#3493 `_), adds new vyper-specific ``#pragma`` directives (`#3493 `_), uses Cancun's ``MCOPY`` opcode for some compiler generated code (`#3483 `_), and generates selector tables which now feature O(1) performance (`#3496 `_). @@ -32,6 +28,7 @@ Breaking changes: - add runtime code layout to initcode (`#3584 `_) - drop evm versions through istanbul (`#3470 `_) - remove vyper signature from runtime (`#3471 `_) +- only allow valid identifiers to be nonreentrant keys (`#3605 `_) Non-breaking changes and improvements: -------------------------------------- @@ -46,12 +43,15 @@ Notable fixes: - fix ``ecrecover()`` behavior when signature is invalid (`GHSA-f5x6-7qgp-jhf3 `_, `#3586 `_) - fix: order of evaluation for some builtins (`#3583 `_, `#3587 `_) +- fix: memory allocation in certain builtins using ``msize`` (`#3610 `_) +- fix: ``_abi_decode()`` input validation in certain complex expressions (`#3626 `_) - fix: pycryptodome for arm builds (`#3485 `_) - let params of internal functions be mutable (`#3473 `_) - typechecking of folded builtins in (`#3490 `_) - update tload/tstore opcodes per latest 1153 EIP spec (`#3484 `_) - fix: raw_call type when max_outsize=0 is set (`#3572 `_) - fix: implements check for indexed event arguments (`#3570 `_) +- fix: type-checking for ``_abi_decode()`` arguments (`#3626 `_) Other docs updates, chores and fixes: ------------------------------------- From b8b4610a46379558367ba60f94ac2813eec056d4 Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Wed, 4 Oct 2023 20:25:09 +0200 Subject: [PATCH 14/17] chore: update `FUNDING.yml` to point directly at wallet (#3636) Add link to the Vyper multisig address. --- FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FUNDING.yml b/FUNDING.yml index 81e82160d0..efb9eb01b7 100644 --- a/FUNDING.yml +++ b/FUNDING.yml @@ -1 +1 @@ -custom: https://gitcoin.co/grants/200/vyper-smart-contract-language-2 +custom: https://etherscan.io/address/0x70CCBE10F980d80b7eBaab7D2E3A73e87D67B775 From 5ca0cbfa4064a5577347bca8377bc9bea1cebca5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Wed, 4 Oct 2023 12:17:36 -0700 Subject: [PATCH 15/17] docs: fix nit in v0.3.10 release notes (#3638) --- docs/release-notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 64199bc860..3db11dc451 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -20,7 +20,7 @@ v0.3.10 ("Black Adder") Date released: 2023-10-04 ========================= -v0.3.10 is a performance focused release. It adds a ``codesize`` optimization mode (`#3493 `_), adds new vyper-specific ``#pragma`` directives (`#3493 `_), uses Cancun's ``MCOPY`` opcode for some compiler generated code (`#3483 `_), and generates selector tables which now feature O(1) performance (`#3496 `_). +v0.3.10 is a performance focused release that additionally ships numerous bugfixes. It adds a ``codesize`` optimization mode (`#3493 `_), adds new vyper-specific ``#pragma`` directives (`#3493 `_), uses Cancun's ``MCOPY`` opcode for some compiler generated code (`#3483 `_), and generates selector tables which now feature O(1) performance (`#3496 `_). Breaking changes: ----------------- From 435754dd1db3e1b0b4608569e003f060e6e9eb40 Mon Sep 17 00:00:00 2001 From: sudo rm -rf --no-preserve-root / Date: Thu, 5 Oct 2023 16:54:29 +0200 Subject: [PATCH 16/17] docs: add note on `pragma` parsing (#3640) * Fix broken link in `structure-of-a-contract.rst` * Add note on `pragma` version parsing --- docs/structure-of-a-contract.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/structure-of-a-contract.rst b/docs/structure-of-a-contract.rst index d2c5d48d96..3861bf4380 100644 --- a/docs/structure-of-a-contract.rst +++ b/docs/structure-of-a-contract.rst @@ -17,7 +17,7 @@ Vyper supports several source code directives to control compiler modes and help Version Pragma -------------- -The version pragma ensures that a contract is only compiled by the intended compiler version, or range of versions. Version strings use `NPM `_ style syntax. Starting from v0.4.0 and up, version strings will use `PEP440 version specifiers _`. +The version pragma ensures that a contract is only compiled by the intended compiler version, or range of versions. Version strings use `NPM `_ style syntax. Starting from v0.4.0 and up, version strings will use `PEP440 version specifiers `_. As of 0.3.10, the recommended way to specify the version pragma is as follows: @@ -25,6 +25,10 @@ As of 0.3.10, the recommended way to specify the version pragma is as follows: #pragma version ^0.3.0 +.. note:: + + Both pragma directive versions ``#pragma`` and ``# pragma`` are supported. + The following declaration is equivalent, and, prior to 0.3.10, was the only supported method to specify the compiler version: .. code-block:: python From 68da04b2e9e010c2e4da288a80eeeb9c8e076025 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 5 Oct 2023 09:04:18 -0700 Subject: [PATCH 17/17] fix: block memory allocation overflow (#3639) this fixes potential overflow bugs in pointer calculation by blocking memory allocation above a certain size. the size limit is set at `2**64`, which is the size of addressable memory on physical machines. practically, for EVM use cases, we could limit at a much smaller number (like `2**24`), but we want to allow for "exotic" targets which may allow much more addressable memory. --- vyper/codegen/memory_allocator.py | 12 +++++++++++- vyper/exceptions.py | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/memory_allocator.py b/vyper/codegen/memory_allocator.py index 582d4b9c54..b5e1212917 100644 --- a/vyper/codegen/memory_allocator.py +++ b/vyper/codegen/memory_allocator.py @@ -1,6 +1,6 @@ from typing import List -from vyper.exceptions import CompilerPanic +from vyper.exceptions import CompilerPanic, MemoryAllocationException from vyper.utils import MemoryPositions @@ -46,6 +46,8 @@ class MemoryAllocator: next_mem: int + _ALLOCATION_LIMIT: int = 2**64 + def __init__(self, start_position: int = MemoryPositions.RESERVED_MEMORY): """ Initializer. @@ -110,6 +112,14 @@ def _expand_memory(self, size: int) -> int: before_value = self.next_mem self.next_mem += size self.size_of_mem = max(self.size_of_mem, self.next_mem) + + if self.size_of_mem >= self._ALLOCATION_LIMIT: + # this should not be caught + raise MemoryAllocationException( + f"Tried to allocate {self.size_of_mem} bytes! " + f"(limit is {self._ALLOCATION_LIMIT} (2**64) bytes)" + ) + return before_value def deallocate_memory(self, pos: int, size: int) -> None: diff --git a/vyper/exceptions.py b/vyper/exceptions.py index defca7cc53..8b2020285a 100644 --- a/vyper/exceptions.py +++ b/vyper/exceptions.py @@ -269,6 +269,10 @@ class StorageLayoutException(VyperException): """Invalid slot for the storage layout overrides""" +class MemoryAllocationException(VyperException): + """Tried to allocate too much memory""" + + class JSONError(Exception): """Invalid compiler input JSON."""