From 9428196a0c48ec7cc02938ee4ae1a0e2fee133c8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 9 Mar 2024 07:11:14 -0800 Subject: [PATCH] feat[tool]: improvements to AST annotation (#3829) this commit enriches the annotated AST output so that links between nodes are explicit and consumers of the AST do not need to "guess" about relationships. - link to the type declaration node for user types - add `sha256sum` field for `Module` nodes - add fields on `Import*` nodes so that they can be linked directly to `Module` nodes (including `sha256sum`, so that changes in dependencies can be detected) - add improved type information (including parsed out metadata, like array length) consistently across all types. misc/refactors: - add `to_dict()` for type objects - removed some dead code - `compare_nodes` was only used in a couple of tests, and node equality could be used there instead. (node equality is not super well-defined, but we could revisit that later). --- tests/unit/ast/nodes/test_binary.py | 2 +- tests/unit/ast/nodes/test_compare_nodes.py | 14 +- tests/unit/ast/nodes/test_from_node.py | 7 - tests/unit/ast/test_ast_dict.py | 1410 +++++++++++++++++--- vyper/ast/__init__.py | 2 +- vyper/ast/nodes.py | 72 +- vyper/ast/nodes.pyi | 1 - vyper/ast/parse.py | 5 +- vyper/builtins/_signatures.py | 2 + vyper/compiler/input_bundle.py | 6 + vyper/semantics/analysis/base.py | 22 +- vyper/semantics/analysis/module.py | 24 +- vyper/semantics/types/base.py | 40 +- vyper/semantics/types/bytestrings.py | 7 + vyper/semantics/types/function.py | 11 + vyper/semantics/types/module.py | 28 +- vyper/semantics/types/primitives.py | 6 + vyper/semantics/types/subscriptable.py | 15 +- vyper/semantics/types/user.py | 9 + vyper/utils.py | 7 + 20 files changed, 1448 insertions(+), 242 deletions(-) diff --git a/tests/unit/ast/nodes/test_binary.py b/tests/unit/ast/nodes/test_binary.py index 069101d7ff..d7662bc4bb 100644 --- a/tests/unit/ast/nodes/test_binary.py +++ b/tests/unit/ast/nodes/test_binary.py @@ -18,7 +18,7 @@ def x(): """ ) - assert vy_ast.compare_nodes(expected, mutated) + assert expected == mutated def test_binary_length(): diff --git a/tests/unit/ast/nodes/test_compare_nodes.py b/tests/unit/ast/nodes/test_compare_nodes.py index 73dc319203..164cd3d371 100644 --- a/tests/unit/ast/nodes/test_compare_nodes.py +++ b/tests/unit/ast/nodes/test_compare_nodes.py @@ -7,7 +7,6 @@ def test_compare_different_node_clases(): right = vyper_ast.body[0].value assert left != right - assert not vy_ast.compare_nodes(left, right) def test_compare_different_nodes_same_class(): @@ -15,7 +14,6 @@ def test_compare_different_nodes_same_class(): left, right = vyper_ast.body[0].value.elements assert left != right - assert not vy_ast.compare_nodes(left, right) def test_compare_different_nodes_same_value(): @@ -23,15 +21,14 @@ def test_compare_different_nodes_same_value(): left, right = vyper_ast.body[0].value.elements assert left != right - assert vy_ast.compare_nodes(left, right) -def test_compare_complex_nodes_same_value(): - vyper_ast = vy_ast.parse_to_ast("[{'foo':'bar', 43:[1,2,3]}, {'foo':'bar', 43:[1,2,3]}]") - left, right = vyper_ast.body[0].value.elements +def test_compare_similar_node(): + # test equality without node_ids + left = vy_ast.Int(value=1) + right = vy_ast.Int(value=1) - assert left != right - assert vy_ast.compare_nodes(left, right) + assert left == right def test_compare_same_node(): @@ -39,4 +36,3 @@ def test_compare_same_node(): node = vyper_ast.body[0].value assert node == node - assert vy_ast.compare_nodes(node, node) diff --git a/tests/unit/ast/nodes/test_from_node.py b/tests/unit/ast/nodes/test_from_node.py index 8a7922d582..8f4a50e729 100644 --- a/tests/unit/ast/nodes/test_from_node.py +++ b/tests/unit/ast/nodes/test_from_node.py @@ -24,13 +24,6 @@ def test_kwargs(): assert new_node.value == 666 -def test_compare_nodes(): - old_node = vy_ast.parse_to_ast("foo = 42") - new_node = vy_ast.Int.from_node(old_node, value=666) - - assert not vy_ast.compare_nodes(old_node, new_node) - - def test_new_node_has_no_parent(): old_node = vy_ast.parse_to_ast("foo = 42") new_node = vy_ast.Int.from_node(old_node, value=666) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 3f14e3d2f7..81c3dc46fa 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -58,6 +58,10 @@ def test_basic_ast(): "col_offset": 0, "end_col_offset": 9, "end_lineno": 2, + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, "lineno": 2, "node_id": 1, "src": "1:9:0", @@ -70,14 +74,10 @@ def test_basic_ast(): "lineno": 2, "node_id": 2, "src": "1:1:0", - "type": "int128", + "type": {"bits": 128, "is_signed": True, "name": "int128", "typeclass": "integer"}, }, + "type": {"bits": 128, "is_signed": True, "name": "int128", "typeclass": "integer"}, "value": None, - "is_constant": False, - "is_immutable": False, - "is_public": False, - "is_transient": False, - "type": "int128", } @@ -142,21 +142,1047 @@ def test() -> int128: # strip source annotations like lineno, we don't care for inspecting # the analysis result -def _strip_source_annotations(dict_node): - to_strip = NODE_SRC_ATTRIBUTES + ("node_id",) +def _strip_source_annotations(dict_node, to_strip): if isinstance(dict_node, dict): for k in list(dict_node.keys()): if k in to_strip: del dict_node[k] continue - _strip_source_annotations(dict_node[k]) + if "decl_node" not in k: + _strip_source_annotations(dict_node[k], to_strip) elif isinstance(dict_node, list): for child in dict_node: - _strip_source_annotations(child) + _strip_source_annotations(child, to_strip) + + +def test_output_type_info(make_input_bundle, chdir_tmp_path): + # test type info is output in the ast dict + # test different, complex types, and test import info is also output + lib1 = """ +struct Foo: + x: uint256 + +event Bar: + pass + +struct Baz: + x: decimal + y: Bytes[20] + z: String[32] + w: uint256 + u: address + +interface Qux: + def return_tuple() -> (Foo[1], uint256): nonpayable + +foo_var: Foo +sarray_var: Foo[1] +darray_var: DynArray[Foo, 5] +interface_var: Qux + +hashmap_var: HashMap[address, Foo] + +sarray_var2: uint256[2] +darray_var2: DynArray[uint256, 5] + +@internal +def foo(): + t: uint256 = max_value(uint256) + u: int24 = empty(int24) + + self.foo_var = empty(Foo) + self.sarray_var[0] = empty(Foo) + self.darray_var[1] = empty(Foo) + + self.sarray_var, t = extcall self.interface_var.return_tuple() + +@external +def bar(): + s: bytes24 = empty(bytes24) + """ + + main = """ +import lib1 + +initializes: lib1 + +@internal +def foo(): + lib1.foo() + log lib1.Bar() + s: lib1.Foo = empty(lib1.Foo) + """ + + input_bundle = make_input_bundle({"lib1.vy": lib1, "main.vy": main}) + + lib1_file = input_bundle.load_file("lib1.vy") + out = compiler.compile_from_file_input( + lib1_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + lib1_ast = out["annotated_ast_dict"]["ast"] + lib1_sha256sum = lib1_ast.pop("source_sha256sum") + assert lib1_sha256sum == lib1_file.sha256sum + to_strip = NODE_SRC_ATTRIBUTES + ("resolved_path", "variable_reads", "variable_writes") + _strip_source_annotations(lib1_ast, to_strip=to_strip) + + main_file = input_bundle.load_file("main.vy") + out = compiler.compile_from_file_input( + main_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + main_ast = out["annotated_ast_dict"]["ast"] + main_sha256sum = main_ast.pop("source_sha256sum") + assert main_sha256sum == main_file.sha256sum + _strip_source_annotations(main_ast, to_strip=to_strip) + + # TODO: would be nice to refactor this into bunch of small test cases + assert main_ast == { + "ast_type": "Module", + "body": [ + { + "alias": None, + "ast_type": "Import", + "import_info": { + "alias": "lib1", + "file_sha256sum": lib1_file.sha256sum, + "path": "lib1.vy", + "qualified_module_name": "lib1", + "source_id": 0, + }, + "name": "lib1", + "node_id": 1, + }, + { + "annotation": {"ast_type": "Name", "id": "lib1", "node_id": 6}, + "ast_type": "InitializesDecl", + "node_id": 3, + }, + { + "args": { + "args": [], + "ast_type": "arguments", + "default": None, + "defaults": [], + "node_id": 9, + }, + "ast_type": "FunctionDef", + "body": [ + { + "ast_type": "Expr", + "node_id": 10, + "value": { + "args": [], + "ast_type": "Call", + "func": { + "ast_type": "Attribute", + "attr": "foo", + "node_id": 12, + "type": { + "name": "foo", + "type_decl_node": {"node_id": 119, "source_id": 0}, + "typeclass": "contract_function", + }, + "value": { + "ast_type": "Name", + "id": "lib1", + "node_id": 13, + "type": { + "name": "lib1.vy", + "type_decl_node": {"node_id": 0, "source_id": 0}, + "typeclass": "module", + }, + }, + }, + "keywords": [], + "node_id": 11, + "type": {"name": "(void)"}, + }, + }, + { + "ast_type": "Log", + "node_id": 17, + "type": { + "name": "Bar", + "type_decl_node": {"node_id": 7, "source_id": 0}, + "typeclass": "event", + }, + "value": { + "args": [], + "ast_type": "Call", + "func": { + "ast_type": "Attribute", + "attr": "Bar", + "node_id": 19, + "type": { + "type_t": { + "name": "Bar", + "type_decl_node": {"node_id": 7, "source_id": 0}, + "typeclass": "event", + } + }, + "value": { + "ast_type": "Name", + "id": "lib1", + "node_id": 20, + "type": { + "name": "lib1.vy", + "type_decl_node": {"node_id": 0, "source_id": 0}, + "typeclass": "module", + }, + }, + }, + "keywords": [], + "node_id": 18, + "type": {"name": "(void)"}, + }, + }, + { + "annotation": { + "ast_type": "Attribute", + "attr": "Foo", + "node_id": 26, + "value": {"ast_type": "Name", "id": "lib1", "node_id": 27}, + }, + "ast_type": "AnnAssign", + "node_id": 23, + "target": { + "ast_type": "Name", + "id": "s", + "node_id": 24, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": { + "args": [ + { + "ast_type": "Attribute", + "attr": "Foo", + "node_id": 33, + "type": {"type_t": {"name": "Foo", "typeclass": "struct"}}, + "value": { + "ast_type": "Name", + "id": "lib1", + "node_id": 34, + "type": { + "name": "lib1.vy", + "type_decl_node": {"node_id": 0, "source_id": 0}, + "typeclass": "module", + }, + }, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 31, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 30, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + ], + "decorator_list": [{"ast_type": "Name", "id": "internal", "node_id": 37}], + "doc_string": None, + "name": "foo", + "node_id": 8, + "pos": None, + "returns": None, + }, + ], + "doc_string": None, + "name": None, + "node_id": 0, + "path": "main.vy", + "source_id": 1, + "type": { + "name": "main.vy", + "type_decl_node": {"node_id": 0, "source_id": 1}, + "typeclass": "module", + }, + } + + # TODO: would be nice to refactor this into bunch of small test cases + # TODO: write the test in a way which makes the links between nodes + # clearer + assert lib1_ast == { + "ast_type": "Module", + "body": [ + { + "ast_type": "StructDef", + "body": [ + { + "annotation": {"ast_type": "Name", "id": "uint256", "node_id": 5}, + "ast_type": "AnnAssign", + "node_id": 2, + "target": {"ast_type": "Name", "id": "x", "node_id": 3}, + "value": None, + } + ], + "doc_string": None, + "name": "Foo", + "node_id": 1, + }, + { + "ast_type": "EventDef", + "body": [{"ast_type": "Pass", "node_id": 8}], + "doc_string": None, + "name": "Bar", + "node_id": 7, + }, + { + "ast_type": "StructDef", + "body": [ + { + "annotation": {"ast_type": "Name", "id": "decimal", "node_id": 13}, + "ast_type": "AnnAssign", + "node_id": 10, + "target": {"ast_type": "Name", "id": "x", "node_id": 11}, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 18, + "slice": {"ast_type": "Int", "node_id": 21, "value": 20}, + "value": {"ast_type": "Name", "id": "Bytes", "node_id": 19}, + }, + "ast_type": "AnnAssign", + "node_id": 15, + "target": {"ast_type": "Name", "id": "y", "node_id": 16}, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 26, + "slice": {"ast_type": "Int", "node_id": 29, "value": 32}, + "value": {"ast_type": "Name", "id": "String", "node_id": 27}, + }, + "ast_type": "AnnAssign", + "node_id": 23, + "target": {"ast_type": "Name", "id": "z", "node_id": 24}, + "value": None, + }, + { + "annotation": {"ast_type": "Name", "id": "uint256", "node_id": 34}, + "ast_type": "AnnAssign", + "node_id": 31, + "target": {"ast_type": "Name", "id": "w", "node_id": 32}, + "value": None, + }, + { + "annotation": {"ast_type": "Name", "id": "address", "node_id": 39}, + "ast_type": "AnnAssign", + "node_id": 36, + "target": {"ast_type": "Name", "id": "u", "node_id": 37}, + "value": None, + }, + ], + "doc_string": None, + "name": "Baz", + "node_id": 9, + }, + { + "ast_type": "InterfaceDef", + "body": [ + { + "args": { + "args": [], + "ast_type": "arguments", + "default": None, + "defaults": [], + "node_id": 43, + }, + "ast_type": "FunctionDef", + "body": [ + { + "ast_type": "Expr", + "node_id": 44, + "value": {"ast_type": "Name", "id": "nonpayable", "node_id": 45}, + } + ], + "decorator_list": [], + "doc_string": None, + "name": "return_tuple", + "node_id": 42, + "pos": None, + "returns": { + "ast_type": "Tuple", + "elements": [ + { + "ast_type": "Subscript", + "node_id": 48, + "slice": {"ast_type": "Int", "node_id": 51, "value": 1}, + "value": {"ast_type": "Name", "id": "Foo", "node_id": 49}, + }, + {"ast_type": "Name", "id": "uint256", "node_id": 53}, + ], + "node_id": 47, + }, + } + ], + "doc_string": None, + "name": "Qux", + "node_id": 41, + }, + { + "annotation": {"ast_type": "Name", "id": "Foo", "node_id": 59}, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 56, + "target": { + "ast_type": "Name", + "id": "foo_var", + "node_id": 57, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + "type": {"name": "Foo", "typeclass": "struct"}, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 64, + "slice": {"ast_type": "Int", "node_id": 67, "value": 1}, + "value": {"ast_type": "Name", "id": "Foo", "node_id": 65}, + }, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 61, + "target": { + "ast_type": "Name", + "id": "sarray_var", + "node_id": 62, + "type": { + "length": 1, + "name": "$SArray", + "typeclass": "static_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + "type": { + "length": 1, + "name": "$SArray", + "typeclass": "static_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 72, + "slice": { + "ast_type": "Tuple", + "elements": [ + {"ast_type": "Name", "id": "Foo", "node_id": 76}, + {"ast_type": "Int", "node_id": 78, "value": 5}, + ], + "node_id": 75, + }, + "value": {"ast_type": "Name", "id": "DynArray", "node_id": 73}, + }, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 69, + "target": { + "ast_type": "Name", + "id": "darray_var", + "node_id": 70, + "type": { + "length": 5, + "name": "DynArray", + "typeclass": "dynamic_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + "type": { + "length": 5, + "name": "DynArray", + "typeclass": "dynamic_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": None, + }, + { + "annotation": {"ast_type": "Name", "id": "Qux", "node_id": 84}, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 81, + "target": { + "ast_type": "Name", + "id": "interface_var", + "node_id": 82, + "type": { + "name": "Qux", + "type_decl_node": {"node_id": 41, "source_id": 0}, + "typeclass": "interface", + }, + }, + "type": { + "name": "Qux", + "type_decl_node": {"node_id": 41, "source_id": 0}, + "typeclass": "interface", + }, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 89, + "slice": { + "ast_type": "Tuple", + "elements": [ + {"ast_type": "Name", "id": "address", "node_id": 93}, + {"ast_type": "Name", "id": "Foo", "node_id": 95}, + ], + "node_id": 92, + }, + "value": {"ast_type": "Name", "id": "HashMap", "node_id": 90}, + }, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 86, + "target": { + "ast_type": "Name", + "id": "hashmap_var", + "node_id": 87, + "type": { + "key_type": {"name": "address"}, + "name": "HashMap", + "typeclass": "hashmap", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + "type": { + "key_type": {"name": "address"}, + "name": "HashMap", + "typeclass": "hashmap", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 102, + "slice": {"ast_type": "Int", "node_id": 105, "value": 2}, + "value": {"ast_type": "Name", "id": "uint256", "node_id": 103}, + }, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 99, + "target": { + "ast_type": "Name", + "id": "sarray_var2", + "node_id": 100, + "type": { + "length": 2, + "name": "$SArray", + "typeclass": "static_array", + "value_type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + }, + "type": { + "length": 2, + "name": "$SArray", + "typeclass": "static_array", + "value_type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + "value": None, + }, + { + "annotation": { + "ast_type": "Subscript", + "node_id": 110, + "slice": { + "ast_type": "Tuple", + "elements": [ + {"ast_type": "Name", "id": "uint256", "node_id": 114}, + {"ast_type": "Int", "node_id": 116, "value": 5}, + ], + "node_id": 113, + }, + "value": {"ast_type": "Name", "id": "DynArray", "node_id": 111}, + }, + "ast_type": "VariableDecl", + "is_constant": False, + "is_immutable": False, + "is_public": False, + "is_transient": False, + "node_id": 107, + "target": { + "ast_type": "Name", + "id": "darray_var2", + "node_id": 108, + "type": { + "length": 5, + "name": "DynArray", + "typeclass": "dynamic_array", + "value_type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + }, + "type": { + "length": 5, + "name": "DynArray", + "typeclass": "dynamic_array", + "value_type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + "value": None, + }, + { + "args": { + "args": [], + "ast_type": "arguments", + "default": None, + "defaults": [], + "node_id": 120, + }, + "ast_type": "FunctionDef", + "body": [ + { + "annotation": {"ast_type": "Name", "id": "uint256", "node_id": 124}, + "ast_type": "AnnAssign", + "node_id": 121, + "target": { + "ast_type": "Name", + "id": "t", + "node_id": 122, + "type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "uint256", + "node_id": 129, + "type": { + "type_t": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + } + }, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "max_value", + "node_id": 127, + "type": {"name": "max_value", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 126, + "type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + }, + { + "annotation": {"ast_type": "Name", "id": "int24", "node_id": 134}, + "ast_type": "AnnAssign", + "node_id": 131, + "target": { + "ast_type": "Name", + "id": "u", + "node_id": 132, + "type": { + "bits": 24, + "is_signed": True, + "name": "int24", + "typeclass": "integer", + }, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "int24", + "node_id": 139, + "type": { + "type_t": { + "bits": 24, + "is_signed": True, + "name": "int24", + "typeclass": "integer", + } + }, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 137, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 136, + "type": { + "bits": 24, + "is_signed": True, + "name": "int24", + "typeclass": "integer", + }, + }, + }, + { + "ast_type": "Assign", + "node_id": 141, + "target": { + "ast_type": "Attribute", + "attr": "foo_var", + "node_id": 142, + "type": {"name": "Foo", "typeclass": "struct"}, + "value": { + "ast_type": "Name", + "id": "self", + "node_id": 143, + "type": {"name": "self"}, + }, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "Foo", + "node_id": 149, + "type": {"type_t": {"name": "Foo", "typeclass": "struct"}}, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 147, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 146, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + { + "ast_type": "Assign", + "node_id": 151, + "target": { + "ast_type": "Subscript", + "node_id": 152, + "slice": { + "ast_type": "Int", + "node_id": 157, + "type": { + "bits": 8, + "is_signed": True, + "name": "int8", + "typeclass": "integer", + }, + "value": 0, + }, + "type": {"name": "Foo", "typeclass": "struct"}, + "value": { + "ast_type": "Attribute", + "attr": "sarray_var", + "node_id": 153, + "type": { + "length": 1, + "name": "$SArray", + "typeclass": "static_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": { + "ast_type": "Name", + "id": "self", + "node_id": 154, + "type": {"name": "self"}, + }, + }, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "Foo", + "node_id": 162, + "type": {"type_t": {"name": "Foo", "typeclass": "struct"}}, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 160, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 159, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + { + "ast_type": "Assign", + "node_id": 164, + "target": { + "ast_type": "Subscript", + "node_id": 165, + "slice": { + "ast_type": "Int", + "node_id": 170, + "type": { + "bits": 8, + "is_signed": True, + "name": "int8", + "typeclass": "integer", + }, + "value": 1, + }, + "type": {"name": "Foo", "typeclass": "struct"}, + "value": { + "ast_type": "Attribute", + "attr": "darray_var", + "node_id": 166, + "type": { + "length": 5, + "name": "DynArray", + "typeclass": "dynamic_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": { + "ast_type": "Name", + "id": "self", + "node_id": 167, + "type": {"name": "self"}, + }, + }, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "Foo", + "node_id": 175, + "type": {"type_t": {"name": "Foo", "typeclass": "struct"}}, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 173, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 172, + "type": {"name": "Foo", "typeclass": "struct"}, + }, + }, + { + "ast_type": "Assign", + "node_id": 177, + "target": { + "ast_type": "Tuple", + "elements": [ + { + "ast_type": "Attribute", + "attr": "sarray_var", + "node_id": 179, + "type": { + "length": 1, + "name": "$SArray", + "typeclass": "static_array", + "value_type": {"name": "Foo", "typeclass": "struct"}, + }, + "value": { + "ast_type": "Name", + "id": "self", + "node_id": 180, + "type": {"name": "self"}, + }, + }, + { + "ast_type": "Name", + "id": "t", + "node_id": 183, + "type": { + "bits": 256, + "is_signed": False, + "name": "uint256", + "typeclass": "integer", + }, + }, + ], + "node_id": 178, + "type": {"members": {}, "name": "$Tuple", "typeclass": "tuple"}, + }, + "value": { + "ast_type": "ExtCall", + "node_id": 186, + "type": {"members": {}, "name": "$Tuple", "typeclass": "tuple"}, + "value": { + "args": [], + "ast_type": "Call", + "func": { + "ast_type": "Attribute", + "attr": "return_tuple", + "node_id": 188, + "type": { + "name": "return_tuple", + "type_decl_node": {"node_id": 42, "source_id": 0}, + "typeclass": "contract_function", + }, + "value": { + "ast_type": "Attribute", + "attr": "interface_var", + "node_id": 189, + "type": { + "name": "Qux", + "type_decl_node": {"node_id": 41, "source_id": 0}, + "typeclass": "interface", + }, + "value": { + "ast_type": "Name", + "id": "self", + "node_id": 190, + "type": {"name": "self"}, + }, + }, + }, + "keywords": [], + "node_id": 187, + "type": {"members": {}, "name": "$Tuple", "typeclass": "tuple"}, + }, + }, + }, + ], + "decorator_list": [{"ast_type": "Name", "id": "internal", "node_id": 194}], + "doc_string": None, + "name": "foo", + "node_id": 119, + "pos": None, + "returns": None, + }, + { + "args": { + "args": [], + "ast_type": "arguments", + "default": None, + "defaults": [], + "node_id": 197, + }, + "ast_type": "FunctionDef", + "body": [ + { + "annotation": {"ast_type": "Name", "id": "bytes24", "node_id": 201}, + "ast_type": "AnnAssign", + "node_id": 198, + "target": { + "ast_type": "Name", + "id": "s", + "node_id": 199, + "type": {"m": 24, "name": "bytes24", "typeclass": "bytes_m"}, + }, + "value": { + "args": [ + { + "ast_type": "Name", + "id": "bytes24", + "node_id": 206, + "type": { + "type_t": { + "m": 24, + "name": "bytes24", + "typeclass": "bytes_m", + } + }, + } + ], + "ast_type": "Call", + "func": { + "ast_type": "Name", + "id": "empty", + "node_id": 204, + "type": {"name": "empty", "typeclass": "builtin_function"}, + }, + "keywords": [], + "node_id": 203, + "type": {"m": 24, "name": "bytes24", "typeclass": "bytes_m"}, + }, + } + ], + "decorator_list": [{"ast_type": "Name", "id": "external", "node_id": 208}], + "doc_string": None, + "name": "bar", + "node_id": 196, + "pos": None, + "returns": None, + }, + ], + "doc_string": None, + "name": None, + "node_id": 0, + "path": "lib1.vy", + "source_id": 0, + "type": { + "name": "lib1.vy", + "type_decl_node": {"node_id": 0, "source_id": 0}, + "typeclass": "module", + }, + } def test_output_variable_read_write_analysis(make_input_bundle, chdir_tmp_path): # test we output the result of variable read/write correctly + # note: also tests serialization of structs, strings, static arrays, + # and type_decl_nodes across modules. lib1 = """ struct Foo: a: uint256 @@ -170,7 +1196,7 @@ def test_output_variable_read_write_analysis(make_input_bundle, chdir_tmp_path): bars: DynArray[Bar, 10] """ - code = """ + main = """ import lib1 initializes: lib1 @@ -206,30 +1232,41 @@ def qux(): def qux2(): self.qux() """ - input_bundle = make_input_bundle({"lib1.vy": lib1}) - - out = compiler.compile_code( - code, - contract_path="main.vy", - input_bundle=input_bundle, - output_formats=["annotated_ast_dict"], - source_id=0, - )["annotated_ast_dict"]["ast"] - _strip_source_annotations(out) - - foo, bar, baz, qux, qux2 = out["body"][3:] + input_bundle = make_input_bundle({"lib1.vy": lib1, "main.vy": main}) + + # preliminaries: main.vy has source_id==0, lib1.vy has source_id==1. + file = input_bundle.load_file("main.vy") + assert file.source_id == 0 + assert input_bundle.load_file("lib1.vy").source_id == 1 + + out = compiler.compile_from_file_input( + file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + ast = out["annotated_ast_dict"]["ast"] + + assert ast["path"] == "main.vy" + assert ast["source_id"] == 0 + + _strip_source_annotations(ast, to_strip=NODE_SRC_ATTRIBUTES + ("node_id", "type")) + + foo, bar, baz, qux, qux2 = ast["body"][3:] assert foo["name"] == "foo" assert foo["body"] == [ { "annotation": {"ast_type": "Name", "id": "uint256"}, "ast_type": "AnnAssign", - "target": {"ast_type": "Name", "id": "x", "type": "uint256"}, + "target": {"ast_type": "Name", "id": "x"}, "value": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, - "variable_reads": [{"access_path": [], "module": "lib1.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "lib1"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } + ], }, }, { @@ -238,14 +1275,23 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, - "variable_reads": [{"access_path": [], "module": "lib1.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "lib1"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } + ], "variable_writes": [ - {"access_path": [], "module": "lib1.vy", "variable": "counter"} + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } ], }, - "value": {"ast_type": "Int", "type": "uint256", "value": 1}, + "value": {"ast_type": "Int", "value": 1}, }, ] @@ -254,25 +1300,35 @@ def qux2(): { "annotation": {"ast_type": "Name", "id": "uint256"}, "ast_type": "AnnAssign", - "target": {"ast_type": "Name", "id": "x", "type": "uint256"}, + "target": {"ast_type": "Name", "id": "x"}, "value": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, - "variable_reads": [{"access_path": [], "module": "lib1.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "lib1"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } + ], }, }, { "annotation": {"ast_type": "Name", "id": "uint256"}, "ast_type": "AnnAssign", - "target": {"ast_type": "Name", "id": "y", "type": "uint256"}, + "target": {"ast_type": "Name", "id": "y"}, "value": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "self", "type": "self"}, - "variable_reads": [{"access_path": [], "module": "main.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "self"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 8, "source_id": 0}, + "name": "counter", + } + ], }, }, { @@ -281,14 +1337,23 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, - "variable_reads": [{"access_path": [], "module": "lib1.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "lib1"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } + ], "variable_writes": [ - {"access_path": [], "module": "lib1.vy", "variable": "counter"} + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } ], }, - "value": {"ast_type": "Int", "type": "uint256", "value": 1}, + "value": {"ast_type": "Int", "value": 1}, }, ] @@ -302,18 +1367,28 @@ def qux2(): "func": { "ast_type": "Attribute", "attr": "bar", - "type": "def bar():", - "value": {"ast_type": "Name", "id": "self", "type": "self"}, + "value": {"ast_type": "Name", "id": "self"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "counter"}, - {"access_path": [], "module": "main.vy", "variable": "counter"}, + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + }, + { + "access_path": [], + "decl_node": {"node_id": 8, "source_id": 0}, + "name": "counter", + }, ], "variable_writes": [ - {"access_path": [], "module": "lib1.vy", "variable": "counter"} + { + "access_path": [], + "decl_node": {"node_id": 29, "source_id": 1}, + "name": "counter", + } ], }, "keywords": [], - "type": "(void)", }, }, { @@ -322,14 +1397,23 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "counter", - "type": "uint256", - "value": {"ast_type": "Name", "id": "self", "type": "self"}, - "variable_reads": [{"access_path": [], "module": "main.vy", "variable": "counter"}], + "value": {"ast_type": "Name", "id": "self"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 8, "source_id": 0}, + "name": "counter", + } + ], "variable_writes": [ - {"access_path": [], "module": "main.vy", "variable": "counter"} + { + "access_path": [], + "decl_node": {"node_id": 8, "source_id": 0}, + "name": "counter", + } ], }, - "value": {"ast_type": "Int", "type": "uint256", "value": 1}, + "value": {"ast_type": "Int", "value": 1}, }, ] @@ -340,37 +1424,54 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "bars", - "type": "DynArray[Bar declaration object, 10]", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, - "variable_reads": [{"access_path": [], "module": "lib1.vy", "variable": "bars"}], - "variable_writes": [{"access_path": [], "module": "lib1.vy", "variable": "bars"}], - }, - "value": { - "ast_type": "List", - "elements": [], - "type": "DynArray[Bar declaration object, 10]", + "value": {"ast_type": "Name", "id": "lib1"}, + "variable_reads": [ + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } + ], + "variable_writes": [ + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } + ], }, + "value": {"ast_type": "List", "elements": []}, }, { "ast_type": "Assign", "target": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 0}, - "type": "Bar declaration object", + "slice": {"ast_type": "Int", "value": 0}, "value": { "ast_type": "Attribute", "attr": "bars", - "type": "DynArray[Bar declaration object, 10]", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, + "value": {"ast_type": "Name", "id": "lib1"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"} + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], }, "variable_reads": [ - {"access_path": ["$subscript_access"], "module": "lib1.vy", "variable": "bars"} + { + "access_path": ["$subscript_access"], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], "variable_writes": [ - {"access_path": ["$subscript_access"], "module": "lib1.vy", "variable": "bars"} + { + "access_path": ["$subscript_access"], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], }, "value": { @@ -378,14 +1479,12 @@ def qux2(): { "ast_type": "Attribute", "attr": "Bar", - "type": "type(Bar declaration object)", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, + "value": {"ast_type": "Name", "id": "lib1"}, } ], "ast_type": "Call", - "func": {"ast_type": "Name", "id": "empty", "type": "(builtin) empty"}, + "func": {"ast_type": "Name", "id": "empty"}, "keywords": [], - "type": "Bar declaration object", }, }, { @@ -393,40 +1492,41 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "items", - "type": "Foo declaration object[2]", "value": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 1}, - "type": "Bar declaration object", + "slice": {"ast_type": "Int", "value": 1}, "value": { "ast_type": "Attribute", "attr": "bars", - "type": "DynArray[Bar declaration object, 10]", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, + "value": {"ast_type": "Name", "id": "lib1"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"} + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], }, "variable_reads": [ { "access_path": ["$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], "variable_writes": [ { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, @@ -435,7 +1535,6 @@ def qux2(): { "ast_type": "Subscript", "slice": {"ast_type": "Int", "value": 2}, - "type": "type(Foo declaration object[2])", "value": { "ast_type": "Attribute", "attr": "Foo", @@ -444,9 +1543,8 @@ def qux2(): } ], "ast_type": "Call", - "func": {"ast_type": "Name", "id": "empty", "type": "(builtin) empty"}, + "func": {"ast_type": "Name", "id": "empty"}, "keywords": [], - "type": "Foo declaration object[2]", }, }, { @@ -454,136 +1552,134 @@ def qux2(): "target": { "ast_type": "Attribute", "attr": "a", - "type": "uint256", "value": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 0}, - "type": "Foo declaration object", + "slice": {"ast_type": "Int", "value": 0}, "value": { "ast_type": "Attribute", "attr": "items", - "type": "Foo declaration object[2]", "value": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 1}, - "type": "Bar declaration object", + "slice": {"ast_type": "Int", "value": 1}, "value": { "ast_type": "Attribute", "attr": "bars", - "type": "DynArray[Bar " "declaration " "object, 10]", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, + "value": {"ast_type": "Name", "id": "lib1"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"} + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], }, "variable_reads": [ { "access_path": ["$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items", "$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items", "$subscript_access", "a"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], "variable_writes": [ { "access_path": ["$subscript_access", "items", "$subscript_access", "a"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, - "value": {"ast_type": "Int", "type": "uint256", "value": 1}, + "value": {"ast_type": "Int", "value": 1}, }, { "ast_type": "Assign", "target": { "ast_type": "Attribute", "attr": "c", - "type": "decimal", "value": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 1}, - "type": "Foo declaration object", + "slice": {"ast_type": "Int", "value": 1}, "value": { "ast_type": "Attribute", "attr": "items", - "type": "Foo declaration object[2]", "value": { "ast_type": "Subscript", - "slice": {"ast_type": "Int", "type": "int8", "value": 0}, - "type": "Bar declaration object", + "slice": {"ast_type": "Int", "value": 0}, "value": { "ast_type": "Attribute", "attr": "bars", - "type": "DynArray[Bar " "declaration " "object, 10]", - "value": {"ast_type": "Name", "id": "lib1", "type": "lib1.vy"}, + "value": {"ast_type": "Name", "id": "lib1"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"} + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + } ], }, "variable_reads": [ { "access_path": ["$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items", "$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, "variable_reads": [ { "access_path": ["$subscript_access", "items", "$subscript_access", "c"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], "variable_writes": [ { "access_path": ["$subscript_access", "items", "$subscript_access", "c"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", } ], }, - "value": {"ast_type": "Decimal", "type": "decimal", "value": "10.0"}, + "value": {"ast_type": "Decimal", "value": "10.0"}, }, ] @@ -597,62 +1693,68 @@ def qux2(): "func": { "ast_type": "Attribute", "attr": "qux", - "type": "def qux():", - "value": {"ast_type": "Name", "id": "self", "type": "self"}, + "value": {"ast_type": "Name", "id": "self"}, "variable_reads": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"}, + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + }, { "access_path": ["$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items", "$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items", "$subscript_access", "a"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items", "$subscript_access", "c"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, ], "variable_writes": [ - {"access_path": [], "module": "lib1.vy", "variable": "bars"}, + { + "access_path": [], + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", + }, { "access_path": ["$subscript_access"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items", "$subscript_access", "a"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, { "access_path": ["$subscript_access", "items", "$subscript_access", "c"], - "module": "lib1.vy", - "variable": "bars", + "decl_node": {"node_id": 34, "source_id": 1}, + "name": "bars", }, ], }, "keywords": [], - "type": "(void)", }, } ] diff --git a/vyper/ast/__init__.py b/vyper/ast/__init__.py index 39530d0c3e..67734ea7ab 100644 --- a/vyper/ast/__init__.py +++ b/vyper/ast/__init__.py @@ -5,7 +5,7 @@ from . import nodes, validation from .natspec import parse_natspec -from .nodes import compare_nodes, as_tuple +from .nodes import as_tuple from .utils import ast_to_dict from .parse import parse_to_ast, parse_to_ast_with_settings diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index bd95b68e09..2ca199bd7e 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -25,7 +25,7 @@ VyperException, ZeroDivisionException, ) -from vyper.utils import MAX_DECIMAL_PLACES, SizeLimits, annotate_source_code, evm_div +from vyper.utils import MAX_DECIMAL_PLACES, SizeLimits, annotate_source_code, evm_div, sha256sum NODE_BASE_ATTRIBUTES = ( "_children", @@ -139,49 +139,6 @@ def get_node( return node -def compare_nodes(left_node: "VyperNode", right_node: "VyperNode") -> bool: - """ - Compare the represented value(s) of two vyper nodes. - - This method evaluates a sort of "loose equality". It recursively compares the - values of each field within two different nodes but does not compare the - node_id or any members related to source offsets. - - Arguments - --------- - left_node : VyperNode - First node object to compare. - right_node : VyperNode - Second node object to compare. - - Returns - ------- - bool - True if the given nodes represent the same value(s), False otherwise. - """ - if not isinstance(left_node, type(right_node)): - return False - - for field_name in (i for i in left_node.get_fields() if i not in VyperNode.__slots__): - left_value = getattr(left_node, field_name, None) - right_value = getattr(right_node, field_name, None) - - # compare types instead of isinstance() in case one node class inherits the other - if type(left_value) is not type(right_value): - return False - - if isinstance(left_value, list): - if next((i for i in zip(left_value, right_value) if not compare_nodes(*i)), None): - return False - elif isinstance(left_value, VyperNode): - if not compare_nodes(left_value, right_value): - return False - elif left_value != right_value: - return False - - return True - - def _to_node(obj, parent): # if object is a Python node or dict representing a node, convert to a Vyper node if isinstance(obj, (dict, python_ast.AST)): @@ -375,6 +332,8 @@ def __deepcopy__(self, memo): return pickle.loads(pickle.dumps(self)) def __eq__(self, other): + # CMC 2024-03-03 I'm not sure it makes much sense to compare AST + # nodes, especially if they come from other modules if not isinstance(other, type(self)): return False if getattr(other, "node_id", None) != getattr(self, "node_id", None): @@ -413,8 +372,16 @@ def description(self): @property def module_node(self): + if isinstance(self, Module): + return self return self.get_ancestor(Module) + def get_id_dict(self): + source_id = None + if self.module_node is not None: + source_id = self.module_node.source_id + return {"node_id": self.node_id, "source_id": source_id} + @property def is_literal_value(self): """ @@ -487,8 +454,9 @@ def to_dict(self) -> dict: else: ast_dict[key] = _to_dict(value) + # TODO: add full analysis result, e.g. expr_info if "type" in self._metadata: - ast_dict["type"] = str(self._metadata["type"]) + ast_dict["type"] = self._metadata["type"].to_dict() return ast_dict @@ -659,6 +627,13 @@ class Module(TopLevel): # metadata __slots__ = ("path", "resolved_path", "source_id") + def to_dict(self): + return dict(source_sha256sum=self.source_sha256sum, **super().to_dict()) + + @property + def source_sha256sum(self): + return sha256sum(self.full_source_code) + @contextlib.contextmanager def namespace(self): from vyper.semantics.namespace import get_namespace, override_global_namespace @@ -1454,6 +1429,13 @@ class Pass(Stmt): class _ImportStmt(Stmt): __slots__ = ("name", "alias") + def to_dict(self): + ret = super().to_dict() + if (import_info := self._metadata.get("import_info")) is not None: + ret["import_info"] = import_info.to_dict() + + return ret + def __init__(self, *args, **kwargs): if len(kwargs["names"]) > 1: _raise_syntax_exc("Assignment statement must have one target", kwargs) diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index a49ac43bdf..4ebb61e76e 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -13,7 +13,6 @@ DICT_AST_SKIPLIST: Any def get_node( ast_struct: Union[dict, python_ast.AST], parent: Optional[VyperNode] = ... ) -> VyperNode: ... -def compare_nodes(left_node: VyperNode, right_node: VyperNode) -> bool: ... class VyperNode: full_source_code: str = ... diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index d14d0a33be..787b1404e6 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -10,7 +10,7 @@ from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic, ParserException, SyntaxException from vyper.typing import ModificationOffsets -from vyper.utils import vyper_warn +from vyper.utils import sha256sum, vyper_warn def parse_to_ast(*args: Any, **kwargs: Any) -> vy_ast.Module: @@ -244,8 +244,11 @@ def _visit_docstring(self, node): return node def visit_Module(self, node): + # TODO: is this the best place for these? maybe they can be on + # CompilerData instead. node.path = self._module_path node.resolved_path = self._resolved_path + node.source_sha256sum = sha256sum(self._source_code) node.source_id = self._source_id return self._visit_docstring(node) diff --git a/vyper/builtins/_signatures.py b/vyper/builtins/_signatures.py index ab5854d68f..d012e4a1cf 100644 --- a/vyper/builtins/_signatures.py +++ b/vyper/builtins/_signatures.py @@ -80,6 +80,8 @@ def decorator_fn(self, node, context): class BuiltinFunctionT(VyperType): + typeclass = "builtin_function" + _has_varargs = False _inputs: list[tuple[str, Any]] = [] _kwargs: dict[str, KwargSettings] = {} diff --git a/vyper/compiler/input_bundle.py b/vyper/compiler/input_bundle.py index d4132cad50..4fe16a4bf1 100644 --- a/vyper/compiler/input_bundle.py +++ b/vyper/compiler/input_bundle.py @@ -2,10 +2,12 @@ import json import os from dataclasses import dataclass +from functools import cached_property from pathlib import Path, PurePath from typing import Any, Iterator, Optional from vyper.exceptions import JSONError +from vyper.utils import sha256sum # a type to make mypy happy PathLike = Path | PurePath @@ -26,6 +28,10 @@ class CompilerInput: class FileInput(CompilerInput): source_code: str + @cached_property + def sha256sum(self): + return sha256sum(self.source_code) + @dataclass class ABIInput(CompilerInput): diff --git a/vyper/semantics/analysis/base.py b/vyper/semantics/analysis/base.py index 762345a726..e424f94e19 100644 --- a/vyper/semantics/analysis/base.py +++ b/vyper/semantics/analysis/base.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union from vyper import ast as vy_ast -from vyper.compiler.input_bundle import InputBundle +from vyper.compiler.input_bundle import CompilerInput, FileInput from vyper.exceptions import CompilerPanic, StructureException from vyper.semantics.data_locations import DataLocation from vyper.semantics.types.base import VyperType @@ -123,10 +123,21 @@ class ImportInfo(AnalysisResult): typ: Union[ModuleInfo, "InterfaceT"] alias: str # the name in the namespace qualified_module_name: str # for error messages - # source_id: int - input_bundle: InputBundle + compiler_input: CompilerInput # to recover file info for ast export node: vy_ast.VyperNode + def to_dict(self): + ret = {"alias": self.alias, "qualified_module_name": self.qualified_module_name} + + ret["source_id"] = self.compiler_input.source_id + ret["path"] = str(self.compiler_input.path) + ret["resolved_path"] = str(self.compiler_input.resolved_path) + + if isinstance(self.compiler_input, FileInput): + ret["file_sha256sum"] = self.compiler_input.sha256sum + + return ret + # analysis result of InitializesDecl @dataclass @@ -242,9 +253,8 @@ def to_dict(self): path = ["$subscript_access" if s is self.SUBSCRIPT_ACCESS else s for s in self.path] varname = var.decl_node.target.id - module_node = var.decl_node.get_ancestor(vy_ast.Module) - module_path = module_node.path - ret = {"variable": varname, "module": module_path, "access_path": path} + decl_node = var.decl_node.get_id_dict() + ret = {"name": varname, "decl_node": decl_node, "access_path": path} return ret diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index b8b4bf48f2..90493d643b 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -4,7 +4,13 @@ import vyper.builtins.interfaces from vyper import ast as vy_ast -from vyper.compiler.input_bundle import ABIInput, FileInput, FilesystemInputBundle, InputBundle +from vyper.compiler.input_bundle import ( + ABIInput, + CompilerInput, + FileInput, + FilesystemInputBundle, + InputBundle, +) from vyper.evm.opcodes import version_check from vyper.exceptions import ( BorrowException, @@ -715,9 +721,9 @@ def visit_StructDef(self, node): def _add_import( self, node: vy_ast.VyperNode, level: int, qualified_module_name: str, alias: str ) -> None: - module_info = self._load_import(node, level, qualified_module_name, alias) + compiler_input, module_info = self._load_import(node, level, qualified_module_name, alias) node._metadata["import_info"] = ImportInfo( - module_info, alias, qualified_module_name, self.input_bundle, node + module_info, alias, qualified_module_name, compiler_input, node ) self.namespace[alias] = module_info @@ -732,7 +738,7 @@ def _load_import(self, node: vy_ast.VyperNode, level: int, module_str: str, alia def _load_import_helper( self, node: vy_ast.VyperNode, level: int, module_str: str, alias: str - ) -> Any: + ) -> tuple[CompilerInput, Any]: if _is_builtin(module_str): return _load_builtin_import(level, module_str) @@ -762,7 +768,7 @@ def _load_import_helper( is_interface=False, ) - return ModuleInfo(module_t, alias) + return file, ModuleInfo(module_t, alias) except FileNotFoundError as e: # escape `e` from the block scope, it can make things @@ -783,7 +789,7 @@ def _load_import_helper( ) module_t = module_ast._metadata["type"] - return module_t.interface + return file, module_t.interface except FileNotFoundError: pass @@ -791,7 +797,7 @@ def _load_import_helper( try: file = self.input_bundle.load_file(path.with_suffix(".json")) assert isinstance(file, ABIInput) # mypy hint - return InterfaceT.from_json_abi(str(file.path), file.abi) + return file, InterfaceT.from_json_abi(str(file.path), file.abi) except FileNotFoundError: pass @@ -844,7 +850,7 @@ def _is_builtin(module_str): return any(module_str.startswith(prefix) for prefix in BUILTIN_PREFIXES) -def _load_builtin_import(level: int, module_str: str) -> InterfaceT: +def _load_builtin_import(level: int, module_str: str) -> tuple[CompilerInput, InterfaceT]: if not _is_builtin(module_str): raise ModuleNotFound(module_str) @@ -885,4 +891,4 @@ def _load_builtin_import(level: int, module_str: str) -> InterfaceT: with override_global_namespace(Namespace()): module_t = _analyze_module_r(interface_ast, input_bundle, ImportGraph(), is_interface=True) - return module_t.interface + return file, module_t.interface diff --git a/vyper/semantics/types/base.py b/vyper/semantics/types/base.py index 94d9c1e371..46edb522ca 100644 --- a/vyper/semantics/types/base.py +++ b/vyper/semantics/types/base.py @@ -32,6 +32,13 @@ def compare_type(self, other): # type is the same return isinstance(other, self.__class__) and other.type_ == self.type_ + def to_dict(self): + # this shouldn't really appear in the AST type annotations, but it's + # there for certain string literals which don't have a known type. this + # should be fixed soon by improving type inference. for now just put + # *something* in the AST. + return {"generic": self.type_.typeclass} + class VyperType: """ @@ -58,7 +65,9 @@ class VyperType: `InterfaceT`s. """ - _id: str + typeclass: str = None # type: ignore + + _id: str # rename to `_name` _type_members: Optional[Dict] = None _valid_literal: Tuple = () _invalid_locations: Tuple = () @@ -74,6 +83,7 @@ class VyperType: _attribute_in_annotation: bool = False size_in_bytes = 32 # default; override for larger types + decl_node: Optional[vy_ast.VyperNode] = None def __init__(self, members: Optional[Dict] = None) -> None: @@ -106,6 +116,31 @@ def __eq__(self, other): def __lt__(self, other): return self.abi_type.selector_name() < other.abi_type.selector_name() + # return a dict suitable for serializing in the AST + def to_dict(self): + ret = {"name": self._id} + if self.decl_node is not None: + ret["type_decl_node"] = self.decl_node.get_id_dict() + if self.typeclass is not None: + ret["typeclass"] = self.typeclass + + # use dict ctor to block duplicates + return dict(**self._addl_dict_fields(), **ret) + + # for most types, this is a reasonable implementation, but it can + # be overridden as needed. + def _addl_dict_fields(self): + keys = self._equality_attrs or () + ret = {} + for k in keys: + if k.startswith("_"): + continue + v = getattr(self, k) + if hasattr(v, "to_dict"): + v = v.to_dict() + ret[k] = v + return ret + @cached_property def _as_darray(self): return self._as_array @@ -369,6 +404,9 @@ def __init__(self, typedef): self.typedef = typedef + def to_dict(self): + return {"type_t": self.typedef.to_dict()} + def __repr__(self): return f"type({self.typedef})" diff --git a/vyper/semantics/types/bytestrings.py b/vyper/semantics/types/bytestrings.py index 96bb1bbf74..cd330681cf 100644 --- a/vyper/semantics/types/bytestrings.py +++ b/vyper/semantics/types/bytestrings.py @@ -42,6 +42,9 @@ def __init__(self, length: int = 0) -> None: def __repr__(self): return f"{self._id}[{self.length}]" + def _addl_dict_fields(self): + return {"length": self.length} + @property def length(self): """ @@ -153,6 +156,8 @@ def from_literal(cls, node: vy_ast.Constant) -> "_BytestringT": class BytesT(_BytestringT): + typeclass = "bytes" + _id = "Bytes" _valid_literal = (vy_ast.Bytes,) @@ -162,6 +167,8 @@ def abi_type(self) -> ABIType: class StringT(_BytestringT): + typeclass = "string" + _id = "String" _valid_literal = (vy_ast.Str,) diff --git a/vyper/semantics/types/function.py b/vyper/semantics/types/function.py index 2cbb972ac7..fbeb3e37cd 100644 --- a/vyper/semantics/types/function.py +++ b/vyper/semantics/types/function.py @@ -82,6 +82,8 @@ class ContractFunctionT(VyperType): Whether this function is marked `@nonreentrant` or not """ + typeclass = "contract_function" + _is_callable = True def __init__( @@ -140,6 +142,10 @@ def __init__( def decl_node(self): return self.ast_def + @property + def _id(self): + return self.name + def mark_analysed(self): assert not self._analysed self._analysed = True @@ -810,6 +816,7 @@ class MemberFunctionT(VyperType): return_type: the return type of this method. ex. None """ + typeclass = "member_function" _is_callable = True # keep LGTM linter happy @@ -836,6 +843,10 @@ def __init__( def modifiability(self): return Modifiability.MODIFIABLE if self.is_modifying else Modifiability.RUNTIME_CONSTANT + @property + def _id(self): + return self.name + def __repr__(self): return f"{self.underlying_type._id} member function '{self.name}'" diff --git a/vyper/semantics/types/module.py b/vyper/semantics/types/module.py index 5faefaf404..a242bfa1fe 100644 --- a/vyper/semantics/types/module.py +++ b/vyper/semantics/types/module.py @@ -29,6 +29,8 @@ class InterfaceT(_UserType): + typeclass = "interface" + _type_members = {"address": AddressT()} _is_prim_word = True _as_array = True @@ -36,7 +38,14 @@ class InterfaceT(_UserType): _supports_external_calls = True _attribute_in_annotation = True - def __init__(self, _id: str, functions: dict, events: dict, structs: dict) -> None: + def __init__( + self, + _id: str, + decl_node: Optional[vy_ast.VyperNode], + functions: dict, + events: dict, + structs: dict, + ) -> None: validate_unique_method_ids(list(functions.values())) members = functions | events | structs @@ -53,6 +62,8 @@ def __init__(self, _id: str, functions: dict, events: dict, structs: dict) -> No self.events = events self.structs = structs + self.decl_node = decl_node + def get_type_member(self, attr, node): # get an event or struct from this interface return TYPE_T(self._helper.get_member(attr, node)) @@ -140,6 +151,7 @@ def to_toplevel_abi_dict(self) -> list[dict]: def _from_lists( cls, interface_name: str, + decl_node: Optional[vy_ast.VyperNode], function_list: list[tuple[str, ContractFunctionT]], event_list: list[tuple[str, EventT]], struct_list: list[tuple[str, StructT]], @@ -169,7 +181,7 @@ def _mark_seen(name, item): _mark_seen(name, struct) structs[name] = struct - return cls(interface_name, functions, events, structs) + return cls(interface_name, decl_node, functions, events, structs) @classmethod def from_json_abi(cls, name: str, abi: dict) -> "InterfaceT": @@ -197,7 +209,7 @@ def from_json_abi(cls, name: str, abi: dict) -> "InterfaceT": events.append((item["name"], EventT.from_abi(item))) structs: list = [] # no structs in json ABI (as of yet) - return cls._from_lists(name, functions, events, structs) + return cls._from_lists(name, None, functions, events, structs) @classmethod def from_ModuleT(cls, module_t: "ModuleT") -> "InterfaceT": @@ -230,7 +242,7 @@ def from_ModuleT(cls, module_t: "ModuleT") -> "InterfaceT": # in the ABI json structs = [(node.name, node._metadata["struct_type"]) for node in module_t.struct_defs] - return cls._from_lists(module_t._id, funcs, events, structs) + return cls._from_lists(module_t._id, module_t.decl_node, funcs, events, structs) @classmethod def from_InterfaceDef(cls, node: vy_ast.InterfaceDef) -> "InterfaceT": @@ -251,11 +263,13 @@ def from_InterfaceDef(cls, node: vy_ast.InterfaceDef) -> "InterfaceT": events: list = [] structs: list = [] - return cls._from_lists(node.name, functions, events, structs) + return cls._from_lists(node.name, node, functions, events, structs) # Datatype to store all module information. class ModuleT(VyperType): + typeclass = "module" + _attribute_in_annotation = True _invalid_locations = ( DataLocation.CALLDATA, @@ -318,6 +332,10 @@ def __eq__(self, other): def __hash__(self): return hash(id(self)) + @property + def decl_node(self) -> Optional[vy_ast.VyperNode]: # type: ignore[override] + return self._module + def get_type_member(self, key: str, node: vy_ast.VyperNode) -> "VyperType": return self._helper.get_member(key, node) diff --git a/vyper/semantics/types/primitives.py b/vyper/semantics/types/primitives.py index 66efabd1db..e3a5d7f834 100644 --- a/vyper/semantics/types/primitives.py +++ b/vyper/semantics/types/primitives.py @@ -55,6 +55,8 @@ def validate_literal(self, node: vy_ast.Constant) -> None: # one-word bytesM with m possible bytes set, e.g. bytes1..bytes32 class BytesM_T(_PrimT): + typeclass = "bytes_m" + _valid_literal = (vy_ast.Hex,) _equality_attrs = ("m",) @@ -231,6 +233,8 @@ class IntegerT(NumericT): Is the value signed? """ + typeclass = "integer" + _valid_literal = (vy_ast.Int,) _equality_attrs = ("is_signed", "bits") @@ -307,6 +311,8 @@ def SINT(bits): class DecimalT(NumericT): + typeclass = "decimal" + _bits = 168 # TODO generalize _decimal_places = 10 # TODO generalize _id = "decimal" diff --git a/vyper/semantics/types/subscriptable.py b/vyper/semantics/types/subscriptable.py index 635a1631a2..e6e8971087 100644 --- a/vyper/semantics/types/subscriptable.py +++ b/vyper/semantics/types/subscriptable.py @@ -41,7 +41,8 @@ def validate_index_type(self, node): class HashMapT(_SubscriptableT): - _id = "HashMap" + typeclass = "hashmap" + _id = "HashMap" # CMC 2024-03-03 maybe this would be better as repr(self) _equality_attrs = ("key_type", "value_type") @@ -152,6 +153,10 @@ class SArrayT(_SequenceT): Static array type """ + typeclass = "static_array" + + _id = "$SArray" + def __init__(self, value_type: VyperType, length: int) -> None: super().__init__(value_type, length) @@ -217,9 +222,12 @@ class DArrayT(_SequenceT): Dynamic array type """ + typeclass = "dynamic_array" + _valid_literal = (vy_ast.List,) _as_array = True - _id = "DynArray" + + _id = "DynArray" # CMC 2024-03-03 maybe this would be better as repr(self) def __init__(self, value_type: VyperType, length: int) -> None: super().__init__(value_type, length) @@ -306,7 +314,10 @@ class TupleT(VyperType): This class is used to represent multiple return values from functions. """ + typeclass = "tuple" + _equality_attrs = ("members",) + _id = "$Tuple" # note: docs say that tuples are not instantiable but they # are in fact instantiable and the codegen works. if we diff --git a/vyper/semantics/types/user.py b/vyper/semantics/types/user.py index 8af229337b..a6ee646e62 100644 --- a/vyper/semantics/types/user.py +++ b/vyper/semantics/types/user.py @@ -46,6 +46,8 @@ def __hash__(self): # note: flag behaves a lot like uint256, or uints in general. class FlagT(_UserType): + typeclass = "flag" + # this is a carveout because currently we allow dynamic arrays of # flags, but not static arrays of flags _as_darray = True @@ -163,6 +165,8 @@ class EventT(_UserType): Name of the event. """ + typeclass = "event" + _invalid_locations = tuple(iter(DataLocation)) # not instantiable in any location def __init__( @@ -180,6 +184,10 @@ def __init__( self.decl_node = decl_node + @property + def _id(self): + return self.name + # backward compatible @property def arguments(self): @@ -292,6 +300,7 @@ def to_toplevel_abi_dict(self) -> list[dict]: class StructT(_UserType): + typeclass = "struct" _as_array = True def __init__(self, _id, members, ast_def=None): diff --git a/vyper/utils.py b/vyper/utils.py index f1e4352d57..ba615e58d7 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -2,6 +2,8 @@ import contextlib import decimal import enum +import functools +import hashlib import sys import time import traceback @@ -159,6 +161,11 @@ def __setattr__(self, name, value): keccak256 = lambda x: _sha3.sha3_256(x).digest() # noqa: E731 +@functools.lru_cache(maxsize=512) +def sha256sum(s: str) -> str: + return hashlib.sha256(s.encode("utf-8")).digest().hex() + + # Converts four bytes to an integer def fourbytes_to_int(inp): return (inp[0] << 24) + (inp[1] << 16) + (inp[2] << 8) + inp[3]