From 24aafda7a3a57de47f92bf2f3b95da0d4ef39b80 Mon Sep 17 00:00:00 2001 From: Josselin Date: Wed, 7 Apr 2021 17:43:22 +0200 Subject: [PATCH 1/6] Add support for multiple compilation units (fix #164) This is a breaking change (see the PR notes) Please enter the commit message for your changes. Lines starting --- crytic_compile/__main__.py | 32 +- crytic_compile/compilation_unit.py | 657 ++++++++++++++++++ crytic_compile/crytic_compile.py | 639 +---------------- crytic_compile/platform/abstract_platform.py | 3 +- crytic_compile/platform/archive.py | 4 +- crytic_compile/platform/brownie.py | 26 +- crytic_compile/platform/buidler.py | 32 +- crytic_compile/platform/dapp.py | 29 +- crytic_compile/platform/embark.py | 22 +- crytic_compile/platform/etherlime.py | 23 +- crytic_compile/platform/etherscan.py | 49 +- crytic_compile/platform/hardhat.py | 141 ++-- crytic_compile/platform/solc.py | 126 ++-- crytic_compile/platform/solc_standard_json.py | 35 +- crytic_compile/platform/standard.py | 172 +++-- crytic_compile/platform/truffle.py | 64 +- crytic_compile/platform/vyper.py | 25 +- crytic_compile/platform/waffle.py | 25 +- 18 files changed, 1147 insertions(+), 957 deletions(-) create mode 100644 crytic_compile/compilation_unit.py diff --git a/crytic_compile/__main__.py b/crytic_compile/__main__.py index 36176377..e439623b 100644 --- a/crytic_compile/__main__.py +++ b/crytic_compile/__main__.py @@ -6,6 +6,7 @@ import logging import os import sys +from typing import TYPE_CHECKING from pkg_resources import require @@ -14,6 +15,10 @@ from crytic_compile.platform import InvalidCompilation from crytic_compile.utils.zip import ZIP_TYPES_ACCEPTED, save_to_zip +if TYPE_CHECKING: + from crytic_compile import CryticCompile + + logging.basicConfig() LOGGER = logging.getLogger("CryticCompile") LOGGER.setLevel(logging.INFO) @@ -145,6 +150,21 @@ def __call__(self, parser, args, values, option_string=None): parser.exit() +def _print_filenames(compilation: "CryticCompile"): + printed_filenames = set() + for compilation_id, compilation_unit in compilation.compilation_units.items(): + print(f"Compilation unit: {compilation_id} ({len(compilation_unit.contracts_names)} files)") + for contract in compilation_unit.contracts_names: + filename = compilation_unit.filename_of_contract(contract) + unique_id = f"{contract} - {filename} - {compilation_id}" + if unique_id not in printed_filenames: + print(f"\t{contract} -> \n\tAbsolute: {filename.absolute}") + print(f"\t\tRelative: {filename.relative}") + print(f"\t\tShort: {filename.short}") + print(f"\t\tUsed: {filename.used}") + printed_filenames.add(unique_id) + + def main(): """ Main function run from the cli @@ -157,19 +177,11 @@ def main(): compilations = compile_all(**vars(args)) # Perform relevant tasks for each compilation - printed_filenames = set() + # print(compilations[0].compilation_units) for compilation in compilations: # Print the filename of each contract (no duplicates). if args.print_filename: - for contract in compilation.contracts_names: - filename = compilation.filename_of_contract(contract) - unique_id = f"{contract} - {filename}" - if unique_id not in printed_filenames: - print(f"{contract} -> \n\tAbsolute: {filename.absolute}") - print(f"\tRelative: {filename.relative}") - print(f"\tShort: {filename.short}") - print(f"\tUsed: {filename.used}") - printed_filenames.add(unique_id) + _print_filenames(compilation) if args.export_format: compilation.export(**vars(args)) diff --git a/crytic_compile/compilation_unit.py b/crytic_compile/compilation_unit.py new file mode 100644 index 00000000..7066963c --- /dev/null +++ b/crytic_compile/compilation_unit.py @@ -0,0 +1,657 @@ +import re +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union + +import sha3 + +from crytic_compile.utils.naming import Filename +from crytic_compile.utils.natspec import Natspec +from crytic_compile.compiler.compiler import CompilerVersion + +# Cycle dependency +if TYPE_CHECKING: + from crytic_compile import CryticCompile + +# pylint: disable=too-many-instance-attributes,too-many-public-methods +class CompilationUnit: + def __init__(self, crytic_compile: "CryticCompile", unique_id: str): + # ASTS are indexed by absolute path + self._asts: Dict = {} + + # ABI, bytecode and srcmap are indexed by contract_name + self._abis: Dict = {} + self._runtime_bytecodes: Dict = {} + self._init_bytecodes: Dict = {} + self._hashes: Dict = {} + self._events: Dict = {} + self._srcmaps: Dict[str, List[str]] = {} + self._srcmaps_runtime: Dict[str, List[str]] = {} + + # set containing all the contract names + self._contracts_name: Set[str] = set() + # set containing all the contract name without the libraries + self._contracts_name_without_libraries: Optional[Set[str]] = None + + # mapping from contract name to filename (naming.Filename) + self._contracts_filenames: Dict[str, Filename] = {} + + # Libraries used by the contract + # contract_name -> (library, pattern) + self._libraries: Dict[str, List[Tuple[str, str]]] = {} + + # Natspec + self._natspec: Dict[str, Natspec] = {} + + # compiler.compiler + self._compiler_version: CompilerVersion = CompilerVersion( + compiler="N/A", version="N/A", optimized=False + ) + + self._crytic_compile: "CryticCompile" = crytic_compile + crytic_compile.compilation_units[unique_id] = self + + self._unique_id = unique_id + + @property + def unique_id(self): + return self._unique_id + + @property + def crytic_compile(self) -> "CryticCompile": + return self._crytic_compile + + ################################################################################### + ################################################################################### + # region Natspec + ################################################################################### + ################################################################################### + + @property + def natspec(self): + """ + Return the natspec of the contractse + + :return: Dict[str, Natspec] + """ + return self._natspec + + ################################################################################### + ################################################################################### + # region Filenames + ################################################################################### + ################################################################################### + + @property + def contracts_filenames(self) -> Dict[str, Filename]: + """ + Return a dict contract_name -> Filename namedtuple (absolute, used) + + :return: dict(name -> utils.namings.Filename) + """ + return self._contracts_filenames + + @property + def contracts_absolute_filenames(self) -> Dict[str, str]: + """ + Return a dict (contract_name -> absolute filename) + + :return: + """ + return {k: f.absolute for (k, f) in self._contracts_filenames.items()} + + def filename_of_contract(self, name: str) -> Filename: + """ + :return: utils.namings.Filename + """ + return self._contracts_filenames[name] + + def absolute_filename_of_contract(self, name: str) -> str: + """ + :return: Absolute filename + """ + return self._contracts_filenames[name].absolute + + def used_filename_of_contract(self, name: str) -> str: + """ + :return: Used filename + """ + return self._contracts_filenames[name].used + + def find_absolute_filename_from_used_filename(self, used_filename: str) -> str: + """ + Return the absolute filename based on the used one + + :param used_filename: + :return: absolute filename + """ + # Note: we could memoize this function if the third party end up using it heavily + # If used_filename is already an absolute pathn no need to lookup + if used_filename in self._crytic_compile.filenames: + return used_filename + d_file = {f.used: f.absolute for _, f in self._contracts_filenames.items()} + if used_filename not in d_file: + raise ValueError("f{filename} does not exist in {d}") + return d_file[used_filename] + + def relative_filename_from_absolute_filename(self, absolute_filename: str) -> str: + """ + Return the relative file based on the absolute name + + :param absolute_filename: + :return: + """ + d_file = {f.absolute: f.relative for _, f in self._contracts_filenames.items()} + if absolute_filename not in d_file: + raise ValueError("f{absolute_filename} does not exist in {d}") + return d_file[absolute_filename] + + # endregion + ################################################################################### + ################################################################################### + # region Contract Names + ################################################################################### + ################################################################################### + + @property + def contracts_names(self) -> Set[str]: + """ + Return the contracts names + + :return: + """ + return self._contracts_name + + @contracts_names.setter + def contracts_names(self, names: Set[str]): + """ + Return the contracts names + + :return: + """ + self._contracts_name = names + + @property + def contracts_names_without_libraries(self) -> Set[str]: + """ + Return the contracts names (without the librairies) + + :return: + """ + if self._contracts_name_without_libraries is None: + libraries: List[str] = [] + for contract_name in self._contracts_name: + libraries += self.libraries_names(contract_name) + self._contracts_name_without_libraries = { + l for l in self._contracts_name if l not in set(libraries) + } + return self._contracts_name_without_libraries + + # endregion + ################################################################################### + ################################################################################### + # region ABI + ################################################################################### + ################################################################################### + + @property + def abis(self) -> Dict: + """ + Return the ABIs + + :return: + """ + return self._abis + + def abi(self, name: str) -> Dict: + """ + Get the ABI from a contract + + :param name: + :return: + """ + return self._abis.get(name, None) + + # endregion + ################################################################################### + ################################################################################### + # region AST + ################################################################################### + ################################################################################### + + @property + def asts(self) -> Dict: + """ + Return the ASTs + + :return: dict (absolute filename -> AST) + """ + return self._asts + + @asts.setter + def asts(self, value: Dict): + self._asts = value + + def ast(self, path: str) -> Union[Dict, None]: + """ + Return of the file + + :param path: + :return: + """ + if path not in self._asts: + try: + path = self.find_absolute_filename_from_used_filename(path) + except ValueError: + pass + return self._asts.get(path, None) + + # endregion + ################################################################################### + ################################################################################### + # region Bytecode + ################################################################################### + ################################################################################### + + @property + def bytecodes_runtime(self) -> Dict[str, str]: + """ + Return the runtime bytecodes + + :return: + """ + return self._runtime_bytecodes + + @bytecodes_runtime.setter + def bytecodes_runtime(self, bytecodes: Dict[str, str]): + """ + Return the init bytecodes + + :return: + """ + self._runtime_bytecodes = bytecodes + + @property + def bytecodes_init(self) -> Dict[str, str]: + """ + Return the init bytecodes + + :return: + """ + return self._init_bytecodes + + @bytecodes_init.setter + def bytecodes_init(self, bytecodes: Dict[str, str]): + """ + Return the init bytecodes + + :return: + """ + self._init_bytecodes = bytecodes + + def bytecode_runtime(self, name: str, libraries: Union[None, Dict[str, str]] = None) -> str: + """ + Return the runtime bytecode of the contract. If library is provided, patch the bytecode + + :param name: + :param libraries: + :return: + """ + runtime = self._runtime_bytecodes.get(name, None) + return self._update_bytecode_with_libraries(runtime, libraries) + + def bytecode_init(self, name: str, libraries: Union[None, Dict[str, str]] = None) -> str: + """ + Return the init bytecode of the contract. If library is provided, patch the bytecode + + :param name: + :param libraries: + :return: + """ + init = self._init_bytecodes.get(name, None) + return self._update_bytecode_with_libraries(init, libraries) + + # endregion + ################################################################################### + ################################################################################### + # region Source mapping + ################################################################################### + ################################################################################### + + @property + def srcmaps_init(self) -> Dict[str, List[str]]: + """ + Return the init srcmap + + :return: + """ + return self._srcmaps + + @property + def srcmaps_runtime(self) -> Dict[str, List[str]]: + """ + Return the runtime srcmap + + :return: + """ + return self._srcmaps_runtime + + def srcmap_init(self, name: str) -> List[str]: + """ + Return the init srcmap + + :param name: + :return: + """ + return self._srcmaps.get(name, []) + + def srcmap_runtime(self, name: str) -> List[str]: + """ + Return the runtime srcmap + + :param name: + :return: + """ + return self._srcmaps_runtime.get(name, []) + + # endregion + ################################################################################### + ################################################################################### + # region Libraries + ################################################################################### + ################################################################################### + + @property + def libraries(self) -> Dict[str, List[Tuple[str, str]]]: + """ + Return the libraries used (contract_name -> [(library, pattern))]) + + :return: + """ + return self._libraries + + def _convert_libraries_names(self, libraries: Dict[str, str]) -> Dict[str, str]: + """ + :param libraries: list(name, addr). Name can be the library name, or filename:library_name + :return: + """ + new_names = {} + for (lib, addr) in libraries.items(): + # Prior solidity 0.5 + # libraries were on the format __filename:contract_name_____ + # From solidity 0.5, + # libraries are on the format __$kecckack(filename:contract_name)[34]$__ + # https://solidity.readthedocs.io/en/v0.5.7/050-breaking-changes.html#command-line-and-json-interfaces + + lib_4 = "__" + lib + "_" * (38 - len(lib)) + + sha3_result = sha3.keccak_256() + sha3_result.update(lib.encode("utf-8")) + lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" + + new_names[lib] = addr + new_names[lib_4] = addr + new_names[lib_5] = addr + + if lib in self.contracts_names: + lib_filename = self.contracts_filenames[lib] + + lib_with_abs_filename = lib_filename.absolute + ":" + lib + lib_with_abs_filename = lib_with_abs_filename[0:36] + + lib_4 = "__" + lib_with_abs_filename + "_" * (38 - len(lib_with_abs_filename)) + new_names[lib_4] = addr + + lib_with_used_filename = lib_filename.used + ":" + lib + lib_with_used_filename = lib_with_used_filename[0:36] + + lib_4 = "__" + lib_with_used_filename + "_" * (38 - len(lib_with_used_filename)) + new_names[lib_4] = addr + + sha3_result = sha3.keccak_256() + sha3_result.update(lib_with_abs_filename.encode("utf-8")) + lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" + new_names[lib_5] = addr + + sha3_result = sha3.keccak_256() + sha3_result.update(lib_with_used_filename.encode("utf-8")) + lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" + new_names[lib_5] = addr + + return new_names + + def _library_name_lookup( + self, lib_name: str, original_contract: str + ) -> Optional[Tuple[str, str]]: + """ + Convert a library name to the contract + The library can be: + - the original contract name + - __X__ following Solidity 0.4 format + - __$..$__ following Solidity 0.5 format + + :param lib_name: + :return: (contract name, pattern) (None if not found) + """ + + for name in self.contracts_names: + if name == lib_name: + return name, name + + # Some platform use only the contract name + # Some use fimename:contract_name + name_with_absolute_filename = self.contracts_filenames[name].absolute + ":" + name + name_with_absolute_filename = name_with_absolute_filename[0:36] + + name_with_used_filename = self.contracts_filenames[name].used + ":" + name + name_with_used_filename = name_with_used_filename[0:36] + + # Solidity 0.4 + solidity_0_4 = "__" + name + "_" * (38 - len(name)) + if solidity_0_4 == lib_name: + return name, solidity_0_4 + + # Solidity 0.4 with filename + solidity_0_4_filename = ( + "__" + name_with_absolute_filename + "_" * (38 - len(name_with_absolute_filename)) + ) + if solidity_0_4_filename == lib_name: + return name, solidity_0_4_filename + + # Solidity 0.4 with filename + solidity_0_4_filename = ( + "__" + name_with_used_filename + "_" * (38 - len(name_with_used_filename)) + ) + if solidity_0_4_filename == lib_name: + return name, solidity_0_4_filename + + # Solidity 0.5 + sha3_result = sha3.keccak_256() + sha3_result.update(name.encode("utf-8")) + v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" + + if v5_name == lib_name: + return name, v5_name + + # Solidity 0.5 with filename + sha3_result = sha3.keccak_256() + sha3_result.update(name_with_absolute_filename.encode("utf-8")) + v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" + + if v5_name == lib_name: + return name, v5_name + + sha3_result = sha3.keccak_256() + sha3_result.update(name_with_used_filename.encode("utf-8")) + v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" + + if v5_name == lib_name: + return name, v5_name + + # handle specific case of collision for Solidity <0.4 + # We can only detect that the second contract is meant to be the library + # if there is only two contracts in the codebase + if len(self._contracts_name) == 2: + return next( + ( + (c, "__" + c + "_" * (38 - len(c))) + for c in self._contracts_name + if c != original_contract + ), + None, + ) + + return None + + def libraries_names(self, name: str) -> List[str]: + """ + Return the name of the libraries used by the contract + + :param name: contract + :return: list of libraries name + """ + + if name not in self._libraries: + init = re.findall(r"__.{36}__", self.bytecode_init(name)) + runtime = re.findall(r"__.{36}__", self.bytecode_runtime(name)) + libraires = [self._library_name_lookup(x, name) for x in set(init + runtime)] + self._libraries[name] = [lib for lib in libraires if lib] + return [name for (name, pattern) in self._libraries[name]] + + def libraries_names_and_patterns(self, name): + """ + Return the name of the libraries used by the contract + + :param name: contract + :return: list of (libraries name, pattern) + """ + + if name not in self._libraries: + init = re.findall(r"__.{36}__", self.bytecode_init(name)) + runtime = re.findall(r"__.{36}__", self.bytecode_runtime(name)) + libraires = [self._library_name_lookup(x, name) for x in set(init + runtime)] + self._libraries[name] = [lib for lib in libraires if lib] + return self._libraries[name] + + def _update_bytecode_with_libraries( + self, bytecode: str, libraries: Union[None, Dict[str, str]] + ) -> str: + """ + Patch the bytecode + + :param bytecode: + :param libraries: + :return: + """ + if libraries: + libraries = self._convert_libraries_names(libraries) + for library_found in re.findall(r"__.{36}__", bytecode): + if library_found in libraries: + bytecode = re.sub( + re.escape(library_found), + "{:040x}".format(libraries[library_found]), + bytecode, + ) + return bytecode + + # endregion + ################################################################################### + ################################################################################### + # region Hashes + ################################################################################### + ################################################################################### + + def hashes(self, name: str) -> Dict[str, int]: + """ + Return the hashes of the functions + + :param name: + :return: + """ + if not name in self._hashes: + self._compute_hashes(name) + return self._hashes[name] + + def _compute_hashes(self, name: str): + self._hashes[name] = {} + for sig in self.abi(name): + if "type" in sig: + if sig["type"] == "function": + sig_name = sig["name"] + arguments = ",".join([x["type"] for x in sig["inputs"]]) + sig = f"{sig_name}({arguments})" + sha3_result = sha3.keccak_256() + sha3_result.update(sig.encode("utf-8")) + self._hashes[name][sig] = int("0x" + sha3_result.hexdigest()[:8], 16) + + # endregion + ################################################################################### + ################################################################################### + # region Events + ################################################################################### + ################################################################################### + + def events_topics(self, name: str) -> Dict[str, Tuple[int, List[bool]]]: + """ + Return the topics of the contract'sevents + :param name: contract + :return: A dictionary {event signature -> topic hash, [is_indexed for each parameter]} + """ + if not name in self._events: + self._compute_topics_events(name) + return self._events[name] + + def _compute_topics_events(self, name: str): + self._events[name] = {} + for sig in self.abi(name): + if "type" in sig: + if sig["type"] == "event": + sig_name = sig["name"] + arguments = ",".join([x["type"] for x in sig["inputs"]]) + indexes = [x.get("indexed", False) for x in sig["inputs"]] + sig = f"{sig_name}({arguments})" + sha3_result = sha3.keccak_256() + sha3_result.update(sig.encode("utf-8")) + + self._events[name][sig] = (int("0x" + sha3_result.hexdigest()[:8], 16), indexes) + + # endregion + ################################################################################### + ################################################################################### + # region Metadata + ################################################################################### + ################################################################################### + + def remove_metadata(self): + """ + Init bytecode contains metadata that needs to be removed + see + http://solidity.readthedocs.io/en/v0.4.24/metadata.html#encoding-of-the-metadata-hash-in-the-bytecode + """ + self._init_bytecodes = { + key: re.sub(r"a165627a7a72305820.{64}0029", r"", bytecode) + for (key, bytecode) in self._init_bytecodes.items() + } + + self._runtime_bytecodes = { + key: re.sub(r"a165627a7a72305820.{64}0029", r"", bytecode) + for (key, bytecode) in self._runtime_bytecodes.items() + } + + # endregion + ################################################################################### + ################################################################################### + # region Compiler information + ################################################################################### + ################################################################################### + + @property + def compiler_version(self) -> "CompilerVersion": + """ + Return the compiler used as a namedtuple(compiler, version) + + :return: + """ + return self._compiler_version + + @compiler_version.setter + def compiler_version(self, compiler): + self._compiler_version = compiler diff --git a/crytic_compile/crytic_compile.py b/crytic_compile/crytic_compile.py index b9487aa3..dc0d3aaf 100644 --- a/crytic_compile/crytic_compile.py +++ b/crytic_compile/crytic_compile.py @@ -6,26 +6,23 @@ import json import logging import os -import re import subprocess from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type, Union -import sha3 - +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.platform import all_platforms, solc_standard_json from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.all_export import PLATFORMS_EXPORT from crytic_compile.platform.solc import Solc from crytic_compile.platform.standard import export_to_standard from crytic_compile.utils.naming import Filename -from crytic_compile.utils.natspec import Natspec from crytic_compile.utils.npm import get_package_name from crytic_compile.utils.zip import load_from_zip # Cycle dependency if TYPE_CHECKING: - from crytic_compile.compiler.compiler import CompilerVersion + pass LOGGER = logging.getLogger("CryticCompile") logging.basicConfig() @@ -56,7 +53,7 @@ def is_supported(target: str) -> bool: return any(platform.is_supported(target) for platform in platforms) or target.endswith(".zip") -# pylint: disable=too-many-instance-attributes,too-many-public-methods +# pylint: disable=too-many-instance-attributes class CryticCompile: """ Main class. @@ -69,33 +66,18 @@ def __init__(self, target: Union[str, AbstractPlatform], **kwargs: str): Keyword Args: See https://github.com/crytic/crytic-compile/wiki/Configuration """ - # ASTS are indexed by absolute path - self._asts: Dict = {} - - # ABI, bytecode and srcmap are indexed by contract_name - self._abis: Dict = {} - self._runtime_bytecodes: Dict = {} - self._init_bytecodes: Dict = {} - self._hashes: Dict = {} - self._events: Dict = {} - self._srcmaps: Dict[str, List[str]] = {} - self._srcmaps_runtime: Dict[str, List[str]] = {} - self._src_content: Dict = {} + # dependencies is needed for platform conversion self._dependencies: Set = set() - # set containing all the contract names - self._contracts_name: Set[str] = set() - # set containing all the contract name without the libraries - self._contracts_name_without_libraries: Optional[Set[str]] = None - # set containing all the filenames self._filenames: Set[Filename] = set() - # mapping from contract name to filename (naming.Filename) - self._contracts_filenames: Dict[str, Filename] = {} + # mapping from absolute/relative/used to filename self._filenames_lookup: Optional[Dict[str, Filename]] = None + self._src_content: Dict = {} + # Mapping each file to # offset -> line, column # This is not memory optimized, but allow an offset lookup in O(1) @@ -107,18 +89,6 @@ def __init__(self, target: Union[str, AbstractPlatform], **kwargs: str): # Note: line 1 is at index 0 self._cached_line_to_code: Dict[Filename, List[bytes]] = dict() - # Libraries used by the contract - # contract_name -> (library, pattern) - self._libraries: Dict[str, List[Tuple[str, str]]] = {} - - self._bytecode_only = False - - # Natspec - self._natspec: Dict[str, Natspec] = {} - - # compiler.compiler - self._compiler_version: Optional["CompilerVersion"] = None - self._working_dir = Path.cwd() if isinstance(target, str): @@ -130,9 +100,9 @@ def __init__(self, target: Union[str, AbstractPlatform], **kwargs: str): self._platform: AbstractPlatform = platform - # If its a exported archive, we use compilation index 0. - # if isinstance(target, dict): - # target = (target, 0) + self._compilation_units: Dict[str, CompilationUnit] = {} + + self._bytecode_only = False self._compile(**kwargs) @@ -145,20 +115,13 @@ def target(self) -> str: """ return self._platform.target - ################################################################################### - ################################################################################### - # region Natspec - ################################################################################### - ################################################################################### - @property - def natspec(self): + def compilation_units(self) -> Dict[str, CompilationUnit]: """ - Return the natspec of the contractse + Return the dict id -> compilation unit - :return: Dict[str, Natspec] """ - return self._natspec + return self._compilation_units ################################################################################### ################################################################################### @@ -177,70 +140,6 @@ def filenames(self) -> Set[Filename]: def filenames(self, all_filenames: Set[Filename]): self._filenames = all_filenames - @property - def contracts_filenames(self) -> Dict[str, Filename]: - """ - Return a dict contract_name -> Filename namedtuple (absolute, used) - - :return: dict(name -> utils.namings.Filename) - """ - return self._contracts_filenames - - @property - def contracts_absolute_filenames(self) -> Dict[str, str]: - """ - Return a dict (contract_name -> absolute filename) - - :return: - """ - return {k: f.absolute for (k, f) in self._contracts_filenames.items()} - - def filename_of_contract(self, name: str) -> Filename: - """ - :return: utils.namings.Filename - """ - return self._contracts_filenames[name] - - def absolute_filename_of_contract(self, name: str) -> str: - """ - :return: Absolute filename - """ - return self._contracts_filenames[name].absolute - - def used_filename_of_contract(self, name: str) -> str: - """ - :return: Used filename - """ - return self._contracts_filenames[name].used - - def find_absolute_filename_from_used_filename(self, used_filename: str) -> str: - """ - Return the absolute filename based on the used one - - :param used_filename: - :return: absolute filename - """ - # Note: we could memoize this function if the third party end up using it heavily - # If used_filename is already an absolute pathn no need to lookup - if used_filename in self._filenames: - return used_filename - d_file = {f.used: f.absolute for _, f in self._contracts_filenames.items()} - if used_filename not in d_file: - raise ValueError("f{filename} does not exist in {d}") - return d_file[used_filename] - - def relative_filename_from_absolute_filename(self, absolute_filename: str) -> str: - """ - Return the relative file based on the absolute name - - :param absolute_filename: - :return: - """ - d_file = {f.absolute: f.relative for _, f in self._contracts_filenames.items()} - if absolute_filename not in d_file: - raise ValueError("f{absolute_filename} does not exist in {d}") - return d_file[absolute_filename] - def filename_lookup(self, filename: str) -> Filename: """ Return a crytic_compile.naming.Filename from a any filename form (used/absolute/relative) @@ -304,7 +203,7 @@ def _get_cached_offset_to_line(self, file: Filename): source_code = self._cached_line_to_code[file] acc = 0 - lines_delimiters: Dict[int, Tuple[int, acc]] = dict() + lines_delimiters: Dict[int, Tuple[int, int]] = dict() for line_number, x in enumerate(source_code): for i in range(acc, acc + len(x)): lines_delimiters[i] = (line_number + 1, i - acc + 1) @@ -341,214 +240,6 @@ def get_code_from_line(self, filename: str, line: int) -> Optional[bytes]: return None return lines[line - 1] - # endregion - ################################################################################### - ################################################################################### - # region Contract Names - ################################################################################### - ################################################################################### - - @property - def contracts_names(self) -> Set[str]: - """ - Return the contracts names - - :return: - """ - return self._contracts_name - - @contracts_names.setter - def contracts_names(self, names: Set[str]): - """ - Return the contracts names - - :return: - """ - self._contracts_name = names - - @property - def contracts_names_without_libraries(self) -> Set[str]: - """ - Return the contracts names (without the librairies) - - :return: - """ - if self._contracts_name_without_libraries is None: - libraries: List[str] = [] - for contract_name in self._contracts_name: - libraries += self.libraries_names(contract_name) - self._contracts_name_without_libraries = { - l for l in self._contracts_name if l not in set(libraries) - } - return self._contracts_name_without_libraries - - # endregion - ################################################################################### - ################################################################################### - # region ABI - ################################################################################### - ################################################################################### - - @property - def abis(self) -> Dict: - """ - Return the ABIs - - :return: - """ - return self._abis - - def abi(self, name: str) -> Dict: - """ - Get the ABI from a contract - - :param name: - :return: - """ - return self._abis.get(name, None) - - # endregion - ################################################################################### - ################################################################################### - # region AST - ################################################################################### - ################################################################################### - - @property - def asts(self) -> Dict: - """ - Return the ASTs - - :return: dict (absolute filename -> AST) - """ - return self._asts - - @asts.setter - def asts(self, value: Dict): - self._asts = value - - def ast(self, path: str) -> Union[Dict, None]: - """ - Return of the file - - :param path: - :return: - """ - if path not in self._asts: - try: - path = self.find_absolute_filename_from_used_filename(path) - except ValueError: - pass - return self._asts.get(path, None) - - # endregion - ################################################################################### - ################################################################################### - # region Bytecode - ################################################################################### - ################################################################################### - - @property - def bytecodes_runtime(self) -> Dict[str, str]: - """ - Return the runtime bytecodes - - :return: - """ - return self._runtime_bytecodes - - @bytecodes_runtime.setter - def bytecodes_runtime(self, bytecodes: Dict[str, str]): - """ - Return the init bytecodes - - :return: - """ - self._runtime_bytecodes = bytecodes - - @property - def bytecodes_init(self) -> Dict[str, str]: - """ - Return the init bytecodes - - :return: - """ - return self._init_bytecodes - - @bytecodes_init.setter - def bytecodes_init(self, bytecodes: Dict[str, str]): - """ - Return the init bytecodes - - :return: - """ - self._init_bytecodes = bytecodes - - def bytecode_runtime(self, name: str, libraries: Union[None, Dict[str, str]] = None) -> str: - """ - Return the runtime bytecode of the contract. If library is provided, patch the bytecode - - :param name: - :param libraries: - :return: - """ - runtime = self._runtime_bytecodes.get(name, None) - return self._update_bytecode_with_libraries(runtime, libraries) - - def bytecode_init(self, name: str, libraries: Union[None, Dict[str, str]] = None) -> str: - """ - Return the init bytecode of the contract. If library is provided, patch the bytecode - - :param name: - :param libraries: - :return: - """ - init = self._init_bytecodes.get(name, None) - return self._update_bytecode_with_libraries(init, libraries) - - # endregion - ################################################################################### - ################################################################################### - # region Source mapping - ################################################################################### - ################################################################################### - - @property - def srcmaps_init(self) -> Dict[str, List[str]]: - """ - Return the init srcmap - - :return: - """ - return self._srcmaps - - @property - def srcmaps_runtime(self) -> Dict[str, List[str]]: - """ - Return the runtime srcmap - - :return: - """ - return self._srcmaps_runtime - - def srcmap_init(self, name: str) -> List[str]: - """ - Return the init srcmap - - :param name: - :return: - """ - return self._srcmaps.get(name, []) - - def srcmap_runtime(self, name: str) -> List[str]: - """ - Return the runtime srcmap - - :param name: - :return: - """ - return self._srcmaps_runtime.get(name, []) - @property def src_content(self) -> Dict[str, str]: """ @@ -618,19 +309,6 @@ def platform(self) -> AbstractPlatform: ################################################################################### ################################################################################### - @property - def compiler_version(self) -> Union[None, "CompilerVersion"]: - """ - Return the compiler used as a namedtuple(compiler, version) - - :return: - """ - return self._compiler_version - - @compiler_version.setter - def compiler_version(self, compiler): - self._compiler_version = compiler - @property def bytecode_only(self) -> bool: """ @@ -644,267 +322,6 @@ def bytecode_only(self) -> bool: def bytecode_only(self, bytecode): self._bytecode_only = bytecode - # endregion - ################################################################################### - ################################################################################### - # region Libraries - ################################################################################### - ################################################################################### - - @property - def libraries(self) -> Dict[str, List[Tuple[str, str]]]: - """ - Return the libraries used (contract_name -> [(library, pattern))]) - - :return: - """ - return self._libraries - - def _convert_libraries_names(self, libraries: Dict[str, str]) -> Dict[str, str]: - """ - :param libraries: list(name, addr). Name can be the library name, or filename:library_name - :return: - """ - new_names = {} - for (lib, addr) in libraries.items(): - # Prior solidity 0.5 - # libraries were on the format __filename:contract_name_____ - # From solidity 0.5, - # libraries are on the format __$kecckack(filename:contract_name)[34]$__ - # https://solidity.readthedocs.io/en/v0.5.7/050-breaking-changes.html#command-line-and-json-interfaces - - lib_4 = "__" + lib + "_" * (38 - len(lib)) - - sha3_result = sha3.keccak_256() - sha3_result.update(lib.encode("utf-8")) - lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" - - new_names[lib] = addr - new_names[lib_4] = addr - new_names[lib_5] = addr - - if lib in self.contracts_names: - lib_filename = self.contracts_filenames[lib] - - lib_with_abs_filename = lib_filename.absolute + ":" + lib - lib_with_abs_filename = lib_with_abs_filename[0:36] - - lib_4 = "__" + lib_with_abs_filename + "_" * (38 - len(lib_with_abs_filename)) - new_names[lib_4] = addr - - lib_with_used_filename = lib_filename.used + ":" + lib - lib_with_used_filename = lib_with_used_filename[0:36] - - lib_4 = "__" + lib_with_used_filename + "_" * (38 - len(lib_with_used_filename)) - new_names[lib_4] = addr - - sha3_result = sha3.keccak_256() - sha3_result.update(lib_with_abs_filename.encode("utf-8")) - lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" - new_names[lib_5] = addr - - sha3_result = sha3.keccak_256() - sha3_result.update(lib_with_used_filename.encode("utf-8")) - lib_5 = "__$" + sha3_result.hexdigest()[:34] + "$__" - new_names[lib_5] = addr - - return new_names - - def _library_name_lookup( - self, lib_name: str, original_contract: str - ) -> Optional[Tuple[str, str]]: - """ - Convert a library name to the contract - The library can be: - - the original contract name - - __X__ following Solidity 0.4 format - - __$..$__ following Solidity 0.5 format - - :param lib_name: - :return: (contract name, pattern) (None if not found) - """ - - for name in self.contracts_names: - if name == lib_name: - return name, name - - # Some platform use only the contract name - # Some use fimename:contract_name - name_with_absolute_filename = self.contracts_filenames[name].absolute + ":" + name - name_with_absolute_filename = name_with_absolute_filename[0:36] - - name_with_used_filename = self.contracts_filenames[name].used + ":" + name - name_with_used_filename = name_with_used_filename[0:36] - - # Solidity 0.4 - solidity_0_4 = "__" + name + "_" * (38 - len(name)) - if solidity_0_4 == lib_name: - return name, solidity_0_4 - - # Solidity 0.4 with filename - solidity_0_4_filename = ( - "__" + name_with_absolute_filename + "_" * (38 - len(name_with_absolute_filename)) - ) - if solidity_0_4_filename == lib_name: - return name, solidity_0_4_filename - - # Solidity 0.4 with filename - solidity_0_4_filename = ( - "__" + name_with_used_filename + "_" * (38 - len(name_with_used_filename)) - ) - if solidity_0_4_filename == lib_name: - return name, solidity_0_4_filename - - # Solidity 0.5 - sha3_result = sha3.keccak_256() - sha3_result.update(name.encode("utf-8")) - v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" - - if v5_name == lib_name: - return name, v5_name - - # Solidity 0.5 with filename - sha3_result = sha3.keccak_256() - sha3_result.update(name_with_absolute_filename.encode("utf-8")) - v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" - - if v5_name == lib_name: - return name, v5_name - - sha3_result = sha3.keccak_256() - sha3_result.update(name_with_used_filename.encode("utf-8")) - v5_name = "__$" + sha3_result.hexdigest()[:34] + "$__" - - if v5_name == lib_name: - return name, v5_name - - # handle specific case of collision for Solidity <0.4 - # We can only detect that the second contract is meant to be the library - # if there is only two contracts in the codebase - if len(self._contracts_name) == 2: - return next( - ( - (c, "__" + c + "_" * (38 - len(c))) - for c in self._contracts_name - if c != original_contract - ), - None, - ) - - return None - - def libraries_names(self, name: str) -> List[str]: - """ - Return the name of the libraries used by the contract - - :param name: contract - :return: list of libraries name - """ - - if name not in self._libraries: - init = re.findall(r"__.{36}__", self.bytecode_init(name)) - runtime = re.findall(r"__.{36}__", self.bytecode_runtime(name)) - libraires = [self._library_name_lookup(x, name) for x in set(init + runtime)] - self._libraries[name] = [lib for lib in libraires if lib] - return [name for (name, pattern) in self._libraries[name]] - - def libraries_names_and_patterns(self, name): - """ - Return the name of the libraries used by the contract - - :param name: contract - :return: list of (libraries name, pattern) - """ - - if name not in self._libraries: - init = re.findall(r"__.{36}__", self.bytecode_init(name)) - runtime = re.findall(r"__.{36}__", self.bytecode_runtime(name)) - libraires = [self._library_name_lookup(x, name) for x in set(init + runtime)] - self._libraries[name] = [lib for lib in libraires if lib] - return self._libraries[name] - - def _update_bytecode_with_libraries( - self, bytecode: str, libraries: Union[None, Dict[str, str]] - ) -> str: - """ - Patch the bytecode - - :param bytecode: - :param libraries: - :return: - """ - if libraries: - libraries = self._convert_libraries_names(libraries) - for library_found in re.findall(r"__.{36}__", bytecode): - if library_found in libraries: - bytecode = re.sub( - re.escape(library_found), - "{:040x}".format(libraries[library_found]), - bytecode, - ) - return bytecode - - # endregion - ################################################################################### - ################################################################################### - # region Hashes - ################################################################################### - ################################################################################### - - def hashes(self, name: str) -> Dict[str, int]: - """ - Return the hashes of the functions - - :param name: - :return: - """ - if not name in self._hashes: - self._compute_hashes(name) - return self._hashes[name] - - def _compute_hashes(self, name: str): - self._hashes[name] = {} - for sig in self.abi(name): - if "type" in sig: - if sig["type"] == "function": - sig_name = sig["name"] - arguments = ",".join([x["type"] for x in sig["inputs"]]) - sig = f"{sig_name}({arguments})" - sha3_result = sha3.keccak_256() - sha3_result.update(sig.encode("utf-8")) - self._hashes[name][sig] = int("0x" + sha3_result.hexdigest()[:8], 16) - - # endregion - ################################################################################### - ################################################################################### - # region Events - ################################################################################### - ################################################################################### - - def events_topics(self, name: str) -> Dict[str, Tuple[int, List[bool]]]: - """ - Return the topics of the contract'sevents - :param name: contract - :return: A dictionary {event signature -> topic hash, [is_indexed for each parameter]} - """ - if not name in self._events: - self._compute_topics_events(name) - return self._events[name] - - def _compute_topics_events(self, name: str): - self._events[name] = {} - for sig in self.abi(name): - if "type" in sig: - if sig["type"] == "event": - sig_name = sig["name"] - arguments = ",".join([x["type"] for x in sig["inputs"]]) - indexes = [x.get("indexed", False) for x in sig["inputs"]] - sig = f"{sig_name}({arguments})" - sha3_result = sha3.keccak_256() - sha3_result.update(sig.encode("utf-8")) - - self._events[name][sig] = (int("0x" + sha3_result.hexdigest()[:8], 16), indexes) - # endregion ################################################################################### ################################################################################### @@ -940,7 +357,7 @@ def import_archive_compilations(compiled_archive: Union[str, Dict]) -> List["Cry ################################################################################### ################################################################################### - def export(self, **kwargs: str) -> Optional[str]: + def export(self, **kwargs: str) -> List[str]: """ Export to json. The json format can be crytic-compile, solc or truffle. """ @@ -988,7 +405,8 @@ def _compile(self, **kwargs: str): remove_metadata = kwargs.get("compile_remove_metadata", False) if remove_metadata: - self._remove_metadata() + for compilation_unit in self._compilation_units.values(): + compilation_unit.remove_metadata() @staticmethod def _run_custom_build(custom_build: str): @@ -1005,29 +423,6 @@ def _run_custom_build(custom_build: str): if stderr: LOGGER.error("Custom build error: \n%s", stderr) - # endregion - ################################################################################### - ################################################################################### - # region Metadata - ################################################################################### - ################################################################################### - - def _remove_metadata(self): - """ - Init bytecode contains metadata that needs to be removed - see - http://solidity.readthedocs.io/en/v0.4.24/metadata.html#encoding-of-the-metadata-hash-in-the-bytecode - """ - self._init_bytecodes = { - key: re.sub(r"a165627a7a72305820.{64}0029", r"", bytecode) - for (key, bytecode) in self._init_bytecodes.items() - } - - self._runtime_bytecodes = { - key: re.sub(r"a165627a7a72305820.{64}0029", r"", bytecode) - for (key, bytecode) in self._runtime_bytecodes.items() - } - # endregion ################################################################################### ################################################################################### diff --git a/crytic_compile/platform/abstract_platform.py b/crytic_compile/platform/abstract_platform.py index 554925b0..c72df2da 100644 --- a/crytic_compile/platform/abstract_platform.py +++ b/crytic_compile/platform/abstract_platform.py @@ -2,7 +2,6 @@ Abstract Platform """ import abc -from pathlib import Path from typing import TYPE_CHECKING, List, Dict from crytic_compile.platform import Type @@ -49,7 +48,7 @@ def __init__(self, target: str, **_kwargs: str): ) self._target: str = target - self._cached_dependencies: Dict[Path, bool] = dict() + self._cached_dependencies: Dict[str, bool] = dict() # region Properties. ################################################################################### diff --git a/crytic_compile/platform/archive.py b/crytic_compile/platform/archive.py index 6d4b14a4..98382992 100644 --- a/crytic_compile/platform/archive.py +++ b/crytic_compile/platform/archive.py @@ -19,7 +19,7 @@ from crytic_compile import CryticCompile -def export_to_archive(crytic_compile: "CryticCompile", **kwargs) -> str: +def export_to_archive(crytic_compile: "CryticCompile", **kwargs) -> List[str]: """ Export the archive @@ -40,7 +40,7 @@ def export_to_archive(crytic_compile: "CryticCompile", **kwargs) -> str: with open(path, "w", encoding="utf8") as f_path: json.dump(output, f_path) - return path + return [path] class Archive(AbstractPlatform): diff --git a/crytic_compile/platform/brownie.py b/crytic_compile/platform/brownie.py index d18068fa..66b99cf2 100755 --- a/crytic_compile/platform/brownie.py +++ b/crytic_compile/platform/brownie.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -124,6 +125,9 @@ def _iterate_over_files(crytic_compile: "CryticCompile", target: str, filenames: compiler = "solc" version = None + compilation_unit = CompilationUnit(crytic_compile, str(target)) + crytic_compile.compilation_units[str(target)] = compilation_unit + for original_filename in filenames: with open(original_filename, encoding="utf8") as f_file: target_loaded: Dict = json.load(f_file) @@ -138,7 +142,7 @@ def _iterate_over_files(crytic_compile: "CryticCompile", target: str, filenames: optimized = compiler_d.get("optimize", False) version = _get_version(compiler_d) if "compiler" in target_loaded: - compiler_d: Dict = target_loaded["compiler"] + compiler_d = target_loaded["compiler"] optimized = compiler_d.get("optimize", False) version = _get_version(compiler_d) @@ -151,29 +155,29 @@ def _iterate_over_files(crytic_compile: "CryticCompile", target: str, filenames: filename_txt, _relative_to_short, crytic_compile, working_dir=target ) - crytic_compile.asts[filename.absolute] = target_loaded["ast"] + compilation_unit.asts[filename.absolute] = target_loaded["ast"] crytic_compile.filenames.add(filename) contract_name = target_loaded["contractName"] - crytic_compile.contracts_filenames[contract_name] = filename - crytic_compile.contracts_names.add(contract_name) - crytic_compile.abis[contract_name] = target_loaded["abi"] - crytic_compile.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( + compilation_unit.contracts_filenames[contract_name] = filename + compilation_unit.contracts_names.add(contract_name) + compilation_unit.abis[contract_name] = target_loaded["abi"] + compilation_unit.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( "0x", "" ) - crytic_compile.bytecodes_runtime[contract_name] = target_loaded[ + compilation_unit.bytecodes_runtime[contract_name] = target_loaded[ "deployedBytecode" ].replace("0x", "") - crytic_compile.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = target_loaded[ + compilation_unit.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = target_loaded[ "deployedSourceMap" ].split(";") userdoc = target_loaded.get("userdoc", {}) devdoc = target_loaded.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler=compiler, version=version, optimized=optimized ) diff --git a/crytic_compile/platform/buidler.py b/crytic_compile/platform/buidler.py index 74b449d8..e56a3bcd 100755 --- a/crytic_compile/platform/buidler.py +++ b/crytic_compile/platform/buidler.py @@ -6,18 +6,18 @@ import os import subprocess from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Tuple from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.exceptions import InvalidCompilation from crytic_compile.platform.types import Type from crytic_compile.utils.naming import convert_filename, extract_name from crytic_compile.utils.natspec import Natspec - from .abstract_platform import AbstractPlatform # Handle cycle from .solc import relative_to_short +from ..compilation_unit import CompilationUnit if TYPE_CHECKING: from crytic_compile import CryticCompile @@ -87,13 +87,15 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): txt = f"`buidler compile` failed. Can you run it?\n{os.path.join(self._target, target_solc_file)} not found" raise InvalidCompilation(txt) - (compiler, version_from_config, optimized) = _get_version_from_config(cache_directory) + compilation_unit = CompilationUnit(crytic_compile, str(target_solc_file)) + + (compiler, version_from_config, optimized) = _get_version_from_config(Path(cache_directory)) - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler=compiler, version=version_from_config, optimized=optimized ) - skip_filename = crytic_compile.compiler_version.version in [ + skip_filename = compilation_unit.compiler_version.version in [ f"0.4.{x}" for x in range(0, 10) ] @@ -118,26 +120,26 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): working_dir=buidler_working_dir, ) - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = info["abi"] - crytic_compile.bytecodes_init[contract_name] = info["evm"]["bytecode"][ + compilation_unit.abis[contract_name] = info["abi"] + compilation_unit.bytecodes_init[contract_name] = info["evm"]["bytecode"][ "object" ] - crytic_compile.bytecodes_runtime[contract_name] = info["evm"][ + compilation_unit.bytecodes_runtime[contract_name] = info["evm"][ "deployedBytecode" ]["object"] - crytic_compile.srcmaps_init[contract_name] = info["evm"]["bytecode"][ + compilation_unit.srcmaps_init[contract_name] = info["evm"]["bytecode"][ "sourceMap" ].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["evm"][ + compilation_unit.srcmaps_runtime[contract_name] = info["evm"][ "deployedBytecode" ]["sourceMap"].split(";") userdoc = info.get("userdoc", {}) devdoc = info.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec if "sources" in targets_json: for path, info in targets_json["sources"].items(): @@ -157,7 +159,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): path, relative_to_short, crytic_compile, working_dir=buidler_working_dir ) crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["ast"] + compilation_unit.asts[path.absolute] = info["ast"] @staticmethod def is_supported(target: str, **kwargs: str) -> bool: @@ -196,7 +198,7 @@ def _guessed_tests(self) -> List[str]: return ["buidler test"] -def _get_version_from_config(builder_directory: Path) -> Optional[Tuple[str, str, bool]]: +def _get_version_from_config(builder_directory: Path) -> Tuple[str, str, bool]: """ :return: (version, optimized) """ diff --git a/crytic_compile/platform/dapp.py b/crytic_compile/platform/dapp.py index 426e87bd..8fe917a7 100755 --- a/crytic_compile/platform/dapp.py +++ b/crytic_compile/platform/dapp.py @@ -13,6 +13,7 @@ # Cycle dependency from typing import TYPE_CHECKING, List +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.types import Type @@ -55,7 +56,9 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): if not dapp_ignore_compile: _run_dapp(self._target) - crytic_compile.compiler_version = _get_version(self._target) + compilation_unit = CompilationUnit(crytic_compile, str(self._target)) + + compilation_unit.compiler_version = _get_version(self._target) optimized = False @@ -77,24 +80,26 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): ): optimized |= metadata["settings"]["optimizer"]["enabled"] contract_name = extract_name(original_contract_name) - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = original_filename - - crytic_compile.abis[contract_name] = info["abi"] - crytic_compile.bytecodes_init[contract_name] = info["evm"]["bytecode"]["object"] - crytic_compile.bytecodes_runtime[contract_name] = info["evm"][ + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = original_filename + + compilation_unit.abis[contract_name] = info["abi"] + compilation_unit.bytecodes_init[contract_name] = info["evm"]["bytecode"][ + "object" + ] + compilation_unit.bytecodes_runtime[contract_name] = info["evm"][ "deployedBytecode" ]["object"] - crytic_compile.srcmaps_init[contract_name] = info["evm"]["bytecode"][ + compilation_unit.srcmaps_init[contract_name] = info["evm"]["bytecode"][ "sourceMap" ].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["evm"]["bytecode"][ + compilation_unit.srcmaps_runtime[contract_name] = info["evm"]["bytecode"][ "sourceMap" ].split(";") userdoc = info.get("userdoc", {}) devdoc = info.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec if version is None: metadata = json.loads(info["metadata"]) @@ -105,9 +110,9 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): path, _relative_to_short, crytic_compile, working_dir=self._target ) crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["ast"] + compilation_unit.asts[path.absolute] = info["ast"] - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler="solc", version=version, optimized=optimized ) diff --git a/crytic_compile/platform/embark.py b/crytic_compile/platform/embark.py index 21faf9b1..ff0c1ab2 100755 --- a/crytic_compile/platform/embark.py +++ b/crytic_compile/platform/embark.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, List +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -101,8 +102,9 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): "Embark did not generate the AST file. Is Embark installed " "(npm install -g embark)? Is embark-contract-info installed? (npm install -g embark)." ) + compilation_unit = CompilationUnit(crytic_compile, str(self._target)) - crytic_compile.compiler_version = _get_version(self._target) + compilation_unit.compiler_version = _get_version(self._target) with open(infile, "r", encoding="utf8") as file_desc: targets_loaded = json.load(file_desc) @@ -110,7 +112,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): filename = convert_filename( k, _relative_to_short, crytic_compile, working_dir=self._target ) - crytic_compile.asts[filename.absolute] = ast + compilation_unit.asts[filename.absolute] = ast crytic_compile.filenames.add(filename) if not "contracts" in targets_loaded: @@ -128,28 +130,28 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): contract_filename, _relative_to_short, crytic_compile, working_dir=self._target ) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.contracts_names.add(contract_name) if "abi" in info: - crytic_compile.abis[contract_name] = info["abi"] + compilation_unit.abis[contract_name] = info["abi"] if "bin" in info: - crytic_compile.bytecodes_init[contract_name] = info["bin"].replace("0x", "") + compilation_unit.bytecodes_init[contract_name] = info["bin"].replace("0x", "") if "bin-runtime" in info: - crytic_compile.bytecodes_runtime[contract_name] = info["bin-runtime"].replace( + compilation_unit.bytecodes_runtime[contract_name] = info["bin-runtime"].replace( "0x", "" ) if "srcmap" in info: - crytic_compile.srcmaps_init[contract_name] = info["srcmap"].split(";") + compilation_unit.srcmaps_init[contract_name] = info["srcmap"].split(";") if "srcmap-runtime" in info: - crytic_compile.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split( + compilation_unit.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split( ";" ) userdoc = info.get("userdoc", {}) devdoc = info.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec @staticmethod def is_supported(target: str, **kwargs: str) -> bool: diff --git a/crytic_compile/platform/etherlime.py b/crytic_compile/platform/etherlime.py index 07e4f651..1f74934a 100755 --- a/crytic_compile/platform/etherlime.py +++ b/crytic_compile/platform/etherlime.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING, List, Optional +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -92,6 +93,8 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): version = None compiler = "solc-js" + compilation_unit = CompilationUnit(crytic_compile, str(self._target)) + for file in filenames: with open(file, encoding="utf8") as file_desc: target_loaded = json.load(file_desc) @@ -108,29 +111,29 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): filename_txt = target_loaded["ast"]["absolutePath"] filename = convert_filename(filename_txt, _relative_to_short, crytic_compile) - crytic_compile.asts[filename.absolute] = target_loaded["ast"] + compilation_unit.asts[filename.absolute] = target_loaded["ast"] crytic_compile.filenames.add(filename) contract_name = target_loaded["contractName"] - crytic_compile.contracts_filenames[contract_name] = filename - crytic_compile.contracts_names.add(contract_name) - crytic_compile.abis[contract_name] = target_loaded["abi"] - crytic_compile.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( + compilation_unit.contracts_filenames[contract_name] = filename + compilation_unit.contracts_names.add(contract_name) + compilation_unit.abis[contract_name] = target_loaded["abi"] + compilation_unit.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( "0x", "" ) - crytic_compile.bytecodes_runtime[contract_name] = target_loaded[ + compilation_unit.bytecodes_runtime[contract_name] = target_loaded[ "deployedBytecode" ].replace("0x", "") - crytic_compile.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = target_loaded[ + compilation_unit.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = target_loaded[ "deployedSourceMap" ].split(";") userdoc = target_loaded.get("userdoc", {}) devdoc = target_loaded.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler=compiler, version=version, optimized=_is_optimized(compile_arguments) ) diff --git a/crytic_compile/platform/etherscan.py b/crytic_compile/platform/etherscan.py index 004a57dd..cd8f22ce 100644 --- a/crytic_compile/platform/etherscan.py +++ b/crytic_compile/platform/etherscan.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Union, Tuple +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -55,15 +56,17 @@ def _handle_bytecode(crytic_compile: "CryticCompile", target: str, result_b: byt contract_filename = Filename(absolute="", relative="", short="", used="") - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = {} - crytic_compile.bytecodes_init[contract_name] = bytecode - crytic_compile.bytecodes_runtime[contract_name] = "" - crytic_compile.srcmaps_init[contract_name] = [] - crytic_compile.srcmaps_runtime[contract_name] = [] + compilation_unit = CompilationUnit(crytic_compile, str(target)) - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.abis[contract_name] = {} + compilation_unit.bytecodes_init[contract_name] = bytecode + compilation_unit.bytecodes_runtime[contract_name] = "" + compilation_unit.srcmaps_init[contract_name] = [] + compilation_unit.srcmaps_runtime[contract_name] = [] + + compilation_unit.compiler_version = CompilerVersion( compiler="unknown", version="", optimized=None ) @@ -242,10 +245,6 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): optimized_run = int(result["Runs"]) solc_arguments = f"--optimize --optimize-runs {optimized_run}" - crytic_compile.compiler_version = CompilerVersion( - compiler="solc", version=compiler_version, optimized=optimization_used - ) - working_dir = None try: # etherscan might return an object with two curly braces, {{ content }} @@ -263,8 +262,10 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): except JSONDecodeError: filename = _handle_single_file(source_code, addr, prefix, contract_name, export_dir) + compilation_unit = CompilationUnit(crytic_compile, str(filename)) + targets_json = _run_solc( - crytic_compile, + compilation_unit, filename, solc=solc, solc_disable_warnings=False, @@ -273,31 +274,35 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): working_dir=working_dir, ) + compilation_unit.compiler_version = CompilerVersion( + compiler="solc", version=compiler_version, optimized=optimization_used + ) + for original_contract_name, info in targets_json["contracts"].items(): contract_name = extract_name(original_contract_name) contract_filename = extract_filename(original_contract_name) contract_filename = convert_filename( contract_filename, _relative_to_short, crytic_compile, working_dir=working_dir ) - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = json.loads(info["abi"]) - crytic_compile.bytecodes_init[contract_name] = info["bin"] - crytic_compile.bytecodes_runtime[contract_name] = info["bin-runtime"] - crytic_compile.srcmaps_init[contract_name] = info["srcmap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split(";") + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.abis[contract_name] = json.loads(info["abi"]) + compilation_unit.bytecodes_init[contract_name] = info["bin"] + compilation_unit.bytecodes_runtime[contract_name] = info["bin-runtime"] + compilation_unit.srcmaps_init[contract_name] = info["srcmap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split(";") userdoc = json.loads(info.get("userdoc", "{}")) devdoc = json.loads(info.get("devdoc", "{}")) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec for path, info in targets_json["sources"].items(): path = convert_filename( path, _relative_to_short, crytic_compile, working_dir=working_dir ) crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["AST"] + compilation_unit.asts[path.absolute] = info["AST"] @staticmethod def is_supported(target: str, **kwargs: str) -> bool: diff --git a/crytic_compile/platform/hardhat.py b/crytic_compile/platform/hardhat.py index 7fbcc7b4..708e772a 100755 --- a/crytic_compile/platform/hardhat.py +++ b/crytic_compile/platform/hardhat.py @@ -17,6 +17,7 @@ # Handle cycle from .solc import relative_to_short +from ..compilation_unit import CompilationUnit if TYPE_CHECKING: from crytic_compile import CryticCompile @@ -46,9 +47,6 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): "ignore_compile", False ) - cache_directory = kwargs.get("hardhat_cache_directory", "cache") - config_file = Path(cache_directory, "solidity-files-cache.json") - build_directory = Path(kwargs.get("hardhat_artifacts_directory", "artifacts"), "build-info") hardhat_working_dir = kwargs.get("hardhat_working_dir", None) @@ -79,75 +77,88 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): if stderr: LOGGER.error(stderr) - (compiler, version_from_config, optimized) = _get_version_from_config(config_file) - - crytic_compile.compiler_version = CompilerVersion( - compiler=compiler, version=version_from_config, optimized=optimized - ) - - skip_filename = crytic_compile.compiler_version.version in [ - f"0.4.{x}" for x in range(0, 10) - ] - files = sorted( os.listdir(build_directory), key=lambda x: os.path.getmtime(Path(build_directory, x)) ) + files = [f for f in files if f.endswith(".json")] if not files: txt = f"`hardhat compile` failed. Can you run it?\n{build_directory} is empty" raise InvalidCompilation(txt) - build_info = Path(build_directory, files[0]) - with open(build_info, encoding="utf8") as file_desc: - targets_json = json.load(file_desc)["output"] - - if "contracts" in targets_json: - for original_filename, contracts_info in targets_json["contracts"].items(): - for original_contract_name, info in contracts_info.items(): - contract_name = extract_name(original_contract_name) - - contract_filename = convert_filename( - original_filename, - relative_to_short, - crytic_compile, - working_dir=hardhat_working_dir, - ) - - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - - crytic_compile.abis[contract_name] = info["abi"] - crytic_compile.bytecodes_init[contract_name] = info["evm"]["bytecode"][ - "object" - ] - crytic_compile.bytecodes_runtime[contract_name] = info["evm"][ - "deployedBytecode" - ]["object"] - crytic_compile.srcmaps_init[contract_name] = info["evm"]["bytecode"][ - "sourceMap" - ].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["evm"][ - "deployedBytecode" - ]["sourceMap"].split(";") - userdoc = info.get("userdoc", {}) - devdoc = info.get("devdoc", {}) - natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec - - if "sources" in targets_json: - for path, info in targets_json["sources"].items(): - if skip_filename: - path = convert_filename( - self._target, - relative_to_short, - crytic_compile, - working_dir=hardhat_working_dir, - ) - else: - path = convert_filename( - path, relative_to_short, crytic_compile, working_dir=hardhat_working_dir - ) - crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["ast"] + for file in files: + build_info = Path(build_directory, file) + + compilation_unit = CompilationUnit(crytic_compile, file) + + with open(build_info, encoding="utf8") as file_desc: + loaded_json = json.load(file_desc) + + targets_json = loaded_json["output"] + + version_from_config = loaded_json["solcVersion"] # TODO supper vyper + input_json = loaded_json["input"] + compiler = "solc" if input_json["language"] == "Solidity" else "vyper" + optimized = input_json["settings"]["optimizer"]["enabled"] + + compilation_unit.compiler_version = CompilerVersion( + compiler=compiler, version=version_from_config, optimized=optimized + ) + + skip_filename = compilation_unit.compiler_version.version in [ + f"0.4.{x}" for x in range(0, 10) + ] + + if "contracts" in targets_json: + for original_filename, contracts_info in targets_json["contracts"].items(): + for original_contract_name, info in contracts_info.items(): + contract_name = extract_name(original_contract_name) + + contract_filename = convert_filename( + original_filename, + relative_to_short, + crytic_compile, + working_dir=hardhat_working_dir, + ) + + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + + compilation_unit.abis[contract_name] = info["abi"] + compilation_unit.bytecodes_init[contract_name] = info["evm"][ + "bytecode" + ]["object"] + compilation_unit.bytecodes_runtime[contract_name] = info["evm"][ + "deployedBytecode" + ]["object"] + compilation_unit.srcmaps_init[contract_name] = info["evm"]["bytecode"][ + "sourceMap" + ].split(";") + compilation_unit.srcmaps_runtime[contract_name] = info["evm"][ + "deployedBytecode" + ]["sourceMap"].split(";") + userdoc = info.get("userdoc", {}) + devdoc = info.get("devdoc", {}) + natspec = Natspec(userdoc, devdoc) + compilation_unit.natspec[contract_name] = natspec + + if "sources" in targets_json: + for path, info in targets_json["sources"].items(): + if skip_filename: + path = convert_filename( + self._target, + relative_to_short, + crytic_compile, + working_dir=hardhat_working_dir, + ) + else: + path = convert_filename( + path, + relative_to_short, + crytic_compile, + working_dir=hardhat_working_dir, + ) + crytic_compile.filenames.add(path) + compilation_unit.asts[path.absolute] = info["ast"] @staticmethod def is_supported(target: str, **kwargs: str) -> bool: diff --git a/crytic_compile/platform/solc.py b/crytic_compile/platform/solc.py index 2e4590a9..4637c530 100644 --- a/crytic_compile/platform/solc.py +++ b/crytic_compile/platform/solc.py @@ -8,6 +8,7 @@ import subprocess from typing import TYPE_CHECKING, Dict, List, Optional, Union +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -28,38 +29,33 @@ LOGGER = logging.getLogger("CryticCompile") -def export_to_solc(crytic_compile: "CryticCompile", **kwargs: str) -> Union[str, None]: - """ - Export the project to the solc format - - :param crytic_compile: - :param kwargs: - :return: - """ - # Obtain objects to represent each contract +def export_to_solc_from_compilation_unit( + compilation_unit: "CompilationUnit", key: str, export_dir: str +) -> Optional[str]: contracts = dict() - for contract_name in crytic_compile.contracts_names: - abi = str(crytic_compile.abi(contract_name)) + + for contract_name in compilation_unit.contracts_names: + abi = str(compilation_unit.abi(contract_name)) abi = abi.replace("'", '"') abi = abi.replace("True", "true") abi = abi.replace("False", "false") abi = abi.replace(" ", "") exported_name = combine_filename_name( - crytic_compile.contracts_filenames[contract_name].absolute, contract_name + compilation_unit.contracts_filenames[contract_name].absolute, contract_name ) contracts[exported_name] = { - "srcmap": ";".join(crytic_compile.srcmap_init(contract_name)), - "srcmap-runtime": ";".join(crytic_compile.srcmap_runtime(contract_name)), + "srcmap": ";".join(compilation_unit.srcmap_init(contract_name)), + "srcmap-runtime": ";".join(compilation_unit.srcmap_runtime(contract_name)), "abi": abi, - "bin": crytic_compile.bytecode_init(contract_name), - "bin-runtime": crytic_compile.bytecode_runtime(contract_name), - "userdoc": crytic_compile.natspec[contract_name].userdoc.export(), - "devdoc": crytic_compile.natspec[contract_name].devdoc.export(), + "bin": compilation_unit.bytecode_init(contract_name), + "bin-runtime": compilation_unit.bytecode_runtime(contract_name), + "userdoc": compilation_unit.natspec[contract_name].userdoc.export(), + "devdoc": compilation_unit.natspec[contract_name].devdoc.export(), } # Create additional informational objects. - sources = {filename: {"AST": ast} for (filename, ast) in crytic_compile.asts.items()} - source_list = [x.absolute for x in crytic_compile.filenames] + sources = {filename: {"AST": ast} for (filename, ast) in compilation_unit.asts.items()} + source_list = [x.absolute for x in compilation_unit.crytic_compile.filenames] # needed for Echidna, see https://github.com/crytic/crytic-compile/issues/112 first_source_list = list(filter(lambda f: "@" in f, source_list)) @@ -72,11 +68,10 @@ def export_to_solc(crytic_compile: "CryticCompile", **kwargs: str) -> Union[str, output = {"sources": sources, "sourceList": source_list, "contracts": contracts} # If we have an export directory specified, we output the JSON. - export_dir = kwargs.get("export_dir", "crytic-export") if export_dir: if not os.path.exists(export_dir): os.makedirs(export_dir) - path = os.path.join(export_dir, "combined_solc.json") + path = os.path.join(export_dir, f"{key}.json") with open(path, "w", encoding="utf8") as file_desc: json.dump(output, file_desc) @@ -84,6 +79,26 @@ def export_to_solc(crytic_compile: "CryticCompile", **kwargs: str) -> Union[str, return None +def export_to_solc(crytic_compile: "CryticCompile", **kwargs: str) -> List[str]: + """ + Export the project to the solc format + + :param crytic_compile: + :param kwargs: + :return: + """ + # Obtain objects to represent each contract + + paths = [] + export_dir = kwargs.get("export_dir", "crytic-export") + + for key, compilation_unit in crytic_compile.compilation_units.items(): + path = export_to_solc_from_compilation_unit(compilation_unit, key, export_dir) + if path: + paths.append(path) + return paths + + class Solc(AbstractPlatform): """ Solc platform @@ -104,19 +119,20 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): solc_working_dir = kwargs.get("solc_working_dir", None) force_legacy_json = kwargs.get("solc_force_legacy_json", False) + compilation_unit = CompilationUnit(crytic_compile, str(self._target)) - targets_json = _get_targets_json(crytic_compile, self._target, **kwargs) + targets_json = _get_targets_json(compilation_unit, self._target, **kwargs) # there have been a couple of changes in solc starting from 0.8.x, - if force_legacy_json and _is_at_or_above_minor_version(crytic_compile, 8): + if force_legacy_json and _is_at_or_above_minor_version(compilation_unit, 8): raise InvalidCompilation("legacy JSON not supported from 0.8.x onwards") - skip_filename = crytic_compile.compiler_version.version in [ + skip_filename = compilation_unit.compiler_version.version in [ f"0.4.{x}" for x in range(0, 10) ] _handle_contracts( - targets_json, skip_filename, crytic_compile, self._target, solc_working_dir + targets_json, skip_filename, compilation_unit, self._target, solc_working_dir ) if "sources" in targets_json: @@ -133,7 +149,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): path, relative_to_short, crytic_compile, working_dir=solc_working_dir ) crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["AST"] + compilation_unit.asts[path.absolute] = info["AST"] @staticmethod def is_supported(target: str, **kwargs: str) -> bool: @@ -163,7 +179,7 @@ def _guessed_tests(self) -> List[str]: return [] -def _get_targets_json(crytic_compile: "CryticCompile", target: str, **kwargs): +def _get_targets_json(compilation_unit: "CompilationUnit", target: str, **kwargs): solc = kwargs.get("solc", "solc") solc_disable_warnings = kwargs.get("solc_disable_warnings", False) solc_arguments = kwargs.get("solc_args", "") @@ -182,7 +198,7 @@ def _get_targets_json(crytic_compile: "CryticCompile", target: str, **kwargs): if isinstance(solcs_path, str): solcs_path = solcs_path.split(",") return _run_solcs_path( - crytic_compile, + compilation_unit, target, solcs_path, solc_disable_warnings, @@ -195,7 +211,7 @@ def _get_targets_json(crytic_compile: "CryticCompile", target: str, **kwargs): if solcs_env: solcs_env_list = solcs_env.split(",") return _run_solcs_env( - crytic_compile, + compilation_unit, target, solc, solc_disable_warnings, @@ -207,7 +223,7 @@ def _get_targets_json(crytic_compile: "CryticCompile", target: str, **kwargs): ) return _run_solc( - crytic_compile, + compilation_unit, target, solc, solc_disable_warnings, @@ -221,12 +237,14 @@ def _get_targets_json(crytic_compile: "CryticCompile", target: str, **kwargs): def _handle_contracts( targets_json: Dict, skip_filename: bool, - crytic_compile: "CryticCompile", + compilation_unit: "CompilationUnit", target: str, solc_working_dir: Optional[str], ): - is_above_0_8 = _is_at_or_above_minor_version(crytic_compile, 8) + is_above_0_8 = _is_at_or_above_minor_version(compilation_unit, 8) + if "contracts" in targets_json: + for original_contract_name, info in targets_json["contracts"].items(): contract_name = extract_name(original_contract_name) contract_filename = extract_filename(original_contract_name) @@ -235,32 +253,32 @@ def _handle_contracts( contract_filename = convert_filename( target, relative_to_short, - crytic_compile, + compilation_unit.crytic_compile, working_dir=solc_working_dir, ) else: contract_filename = convert_filename( contract_filename, relative_to_short, - crytic_compile, + compilation_unit.crytic_compile, working_dir=solc_working_dir, ) - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = ( + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.abis[contract_name] = ( json.loads(info["abi"]) if not is_above_0_8 else info["abi"] ) - crytic_compile.bytecodes_init[contract_name] = info["bin"] - crytic_compile.bytecodes_runtime[contract_name] = info["bin-runtime"] - crytic_compile.srcmaps_init[contract_name] = info["srcmap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split(";") + compilation_unit.bytecodes_init[contract_name] = info["bin"] + compilation_unit.bytecodes_runtime[contract_name] = info["bin-runtime"] + compilation_unit.srcmaps_init[contract_name] = info["srcmap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = info["srcmap-runtime"].split(";") userdoc = json.loads(info.get("userdoc", "{}")) if not is_above_0_8 else info["userdoc"] devdoc = json.loads(info.get("devdoc", "{}")) if not is_above_0_8 else info["devdoc"] natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec -def _is_at_or_above_minor_version(crytic_compile: "CryticCompile", version: int) -> bool: +def _is_at_or_above_minor_version(compilation_unit: "CompilationUnit", version: int) -> bool: """ Checks if the solc version is at or above(=newer) a given minor (0.x.0) version @@ -269,7 +287,7 @@ def _is_at_or_above_minor_version(crytic_compile: "CryticCompile", version: int) :return: """ - return int(crytic_compile.compiler_version.version.split(".")[1]) >= version + return int(compilation_unit.compiler_version.version.split(".")[1]) >= version def get_version(solc: str, env: Dict[str, str]) -> str: @@ -307,7 +325,7 @@ def is_optimized(solc_arguments: str) -> bool: # pylint: disable=too-many-arguments,too-many-locals,too-many-branches def _run_solc( - crytic_compile: "CryticCompile", + compilation_unit: "CompilationUnit", filename: str, solc: str, solc_disable_warnings, @@ -340,11 +358,11 @@ def _run_solc( if not filename.endswith(".sol"): raise InvalidCompilation("Incorrect file format") - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler="solc", version=get_version(solc, env), optimized=is_optimized(solc_arguments) ) - compiler_version = crytic_compile.compiler_version + compiler_version = compilation_unit.compiler_version assert compiler_version old_04_versions = [f"0.4.{x}" for x in range(0, 12)] if compiler_version.version in old_04_versions or compiler_version.version.startswith("0.3"): @@ -415,7 +433,7 @@ def _run_solc( # pylint: disable=too-many-arguments def _run_solcs_path( - crytic_compile, + compilation_unit: "CompilationUnit", filename, solcs_path, solc_disable_warnings, @@ -433,7 +451,7 @@ def _run_solcs_path( continue try: targets_json = _run_solc( - crytic_compile, + compilation_unit, filename, solcs_path[guessed_solc], solc_disable_warnings, @@ -452,7 +470,7 @@ def _run_solcs_path( for solc_bin in solc_bins: try: targets_json = _run_solc( - crytic_compile, + compilation_unit, filename, solc_bin, solc_disable_warnings, @@ -475,7 +493,7 @@ def _run_solcs_path( # pylint: disable=too-many-arguments def _run_solcs_env( - crytic_compile, + compilation_unit: "CompilationUnit", filename, solc, solc_disable_warnings, @@ -495,7 +513,7 @@ def _run_solcs_env( try: env["SOLC_VERSION"] = guessed_solc targets_json = _run_solc( - crytic_compile, + compilation_unit, filename, solc, solc_disable_warnings, @@ -515,7 +533,7 @@ def _run_solcs_env( try: env["SOLC_VERSION"] = version_env targets_json = _run_solc( - crytic_compile, + compilation_unit, filename, solc, solc_disable_warnings, diff --git a/crytic_compile/platform/solc_standard_json.py b/crytic_compile/platform/solc_standard_json.py index fc4dce8a..8e56c545 100644 --- a/crytic_compile/platform/solc_standard_json.py +++ b/crytic_compile/platform/solc_standard_json.py @@ -7,6 +7,7 @@ import subprocess from typing import TYPE_CHECKING, Dict, List, Optional, Union +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.exceptions import InvalidCompilation from crytic_compile.platform.solc import Solc, get_version, is_optimized, relative_to_short @@ -127,11 +128,15 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): solc_remaps: Optional[Union[str, List[str]]] = kwargs.get("solc_remaps", None) solc_working_dir = kwargs.get("solc_working_dir", None) - crytic_compile.compiler_version = CompilerVersion( - compiler="solc", version=get_version(solc, None), optimized=is_optimized(solc_arguments) + compilation_unit = CompilationUnit(crytic_compile, "standard_json") + + compilation_unit.compiler_version = CompilerVersion( + compiler="solc", + version=get_version(solc, dict()), + optimized=is_optimized(solc_arguments), ) - skip_filename = crytic_compile.compiler_version.version in [ + skip_filename = compilation_unit.compiler_version.version in [ f"0.4.{x}" for x in range(0, 10) ] @@ -165,25 +170,27 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): crytic_compile, working_dir=solc_working_dir, ) - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = info["abi"] + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.abis[contract_name] = info["abi"] userdoc = info.get("userdoc", {}) devdoc = info.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec - crytic_compile.bytecodes_init[contract_name] = info["evm"]["bytecode"]["object"] - crytic_compile.bytecodes_runtime[contract_name] = info["evm"][ + compilation_unit.bytecodes_init[contract_name] = info["evm"]["bytecode"][ + "object" + ] + compilation_unit.bytecodes_runtime[contract_name] = info["evm"][ "deployedBytecode" ]["object"] - crytic_compile.srcmaps_init[contract_name] = info["evm"]["bytecode"][ - "sourceMap" - ].split(";") - crytic_compile.srcmaps_runtime[contract_name] = info["evm"]["deployedBytecode"][ + compilation_unit.srcmaps_init[contract_name] = info["evm"]["bytecode"][ "sourceMap" ].split(";") + compilation_unit.srcmaps_runtime[contract_name] = info["evm"][ + "deployedBytecode" + ]["sourceMap"].split(";") if "sources" in targets_json: for path, info in targets_json["sources"].items(): @@ -199,7 +206,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): path, relative_to_short, crytic_compile, working_dir=solc_working_dir ) crytic_compile.filenames.add(path) - crytic_compile.asts[path.absolute] = info["ast"] + compilation_unit.asts[path.absolute] = info["ast"] def _guessed_tests(self) -> List[str]: """ diff --git a/crytic_compile/platform/standard.py b/crytic_compile/platform/standard.py index 41ad69f3..aefba9c6 100644 --- a/crytic_compile/platform/standard.py +++ b/crytic_compile/platform/standard.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Tuple, Type +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform import Type as PlatformType from crytic_compile.platform.abstract_platform import AbstractPlatform @@ -18,7 +19,7 @@ from crytic_compile import CryticCompile -def export_to_standard(crytic_compile: "CryticCompile", **kwargs: str) -> str: +def export_to_standard(crytic_compile: "CryticCompile", **kwargs: str) -> List[str]: """ Export the project to the standard crytic compile format :param crytic_compile: @@ -43,7 +44,7 @@ def export_to_standard(crytic_compile: "CryticCompile", **kwargs: str) -> str: with open(path, "w", encoding="utf8") as file_desc: json.dump(output, file_desc) - return path + return [path] class Standard(AbstractPlatform): @@ -136,41 +137,48 @@ def generate_standard_export(crytic_compile: "CryticCompile") -> Dict: :param crytic_compile: :return: """ - contracts = dict() - for contract_name in crytic_compile.contracts_names: - filename = crytic_compile.filename_of_contract(contract_name) - libraries = crytic_compile.libraries_names_and_patterns(contract_name) - contracts[contract_name] = { - "abi": crytic_compile.abi(contract_name), - "bin": crytic_compile.bytecode_init(contract_name), - "bin-runtime": crytic_compile.bytecode_runtime(contract_name), - "srcmap": ";".join(crytic_compile.srcmap_init(contract_name)), - "srcmap-runtime": ";".join(crytic_compile.srcmap_runtime(contract_name)), - "filenames": { - "absolute": filename.absolute, - "used": filename.used, - "short": filename.short, - "relative": filename.relative, - }, - "libraries": dict(libraries) if libraries else dict(), - "is_dependency": crytic_compile.is_dependency(filename.absolute), - "userdoc": crytic_compile.natspec[contract_name].userdoc.export(), - "devdoc": crytic_compile.natspec[contract_name].devdoc.export(), + compilation_units = {} + for key, compilation_unit in crytic_compile.compilation_units.items(): + contracts = dict() + for contract_name in compilation_unit.contracts_names: + filename = compilation_unit.filename_of_contract(contract_name) + libraries = compilation_unit.libraries_names_and_patterns(contract_name) + contracts[contract_name] = { + "abi": compilation_unit.abi(contract_name), + "bin": compilation_unit.bytecode_init(contract_name), + "bin-runtime": compilation_unit.bytecode_runtime(contract_name), + "srcmap": ";".join(compilation_unit.srcmap_init(contract_name)), + "srcmap-runtime": ";".join(compilation_unit.srcmap_runtime(contract_name)), + "filenames": { + "absolute": filename.absolute, + "used": filename.used, + "short": filename.short, + "relative": filename.relative, + }, + "libraries": dict(libraries) if libraries else dict(), + "is_dependency": crytic_compile.is_dependency(filename.absolute), + "userdoc": compilation_unit.natspec[contract_name].userdoc.export(), + "devdoc": compilation_unit.natspec[contract_name].devdoc.export(), + } + + # Create our root object to contain the contracts and other information. + + compiler: Dict = dict() + if compilation_unit.compiler_version: + compiler = { + "compiler": compilation_unit.compiler_version.compiler, + "version": compilation_unit.compiler_version.version, + "optimized": compilation_unit.compiler_version.optimized, + } + + compilation_units[key] = { + "compiler": compiler, + "asts": compilation_unit.asts, + "contracts": contracts, } - # Create our root object to contain the contracts and other information. - - compiler: Dict = dict() - if crytic_compile.compiler_version: - compiler = { - "compiler": crytic_compile.compiler_version.compiler, - "version": crytic_compile.compiler_version.version, - "optimized": crytic_compile.compiler_version.optimized, - } output = { - "asts": crytic_compile.asts, - "contracts": contracts, - "compiler": compiler, + "compilation_units": compilation_units, "package": crytic_compile.package, "working_dir": str(crytic_compile.working_dir), "type": int(crytic_compile.platform.platform_type_used), @@ -179,50 +187,96 @@ def generate_standard_export(crytic_compile: "CryticCompile") -> Dict: return output -def load_from_compile(crytic_compile: "CryticCompile", loaded_json: Dict) -> Tuple[int, List[str]]: - """ - Load from json - - :param crytic_compile: - :param loaded_json: - :return: - """ - crytic_compile.package_name = loaded_json.get("package", None) - crytic_compile.asts = loaded_json["asts"] - crytic_compile.compiler_version = CompilerVersion( +def _load_from_compile_legacy(crytic_compile: "CryticCompile", loaded_json: Dict): + compilation_unit = CompilationUnit(crytic_compile, "legacy") + compilation_unit.asts = loaded_json["asts"] + compilation_unit.compiler_version = CompilerVersion( compiler=loaded_json["compiler"]["compiler"], version=loaded_json["compiler"]["version"], optimized=loaded_json["compiler"]["optimized"], ) for contract_name, contract in loaded_json["contracts"].items(): - crytic_compile.contracts_names.add(contract_name) + compilation_unit.contracts_names.add(contract_name) filename = Filename( absolute=contract["filenames"]["absolute"], relative=contract["filenames"]["relative"], short=contract["filenames"]["short"], used=contract["filenames"]["used"], ) - crytic_compile.contracts_filenames[contract_name] = filename + compilation_unit.contracts_filenames[contract_name] = filename - crytic_compile.abis[contract_name] = contract["abi"] - crytic_compile.bytecodes_init[contract_name] = contract["bin"] - crytic_compile.bytecodes_runtime[contract_name] = contract["bin-runtime"] - crytic_compile.srcmaps_init[contract_name] = contract["srcmap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = contract["srcmap-runtime"].split(";") - crytic_compile.libraries[contract_name] = contract["libraries"] + compilation_unit.abis[contract_name] = contract["abi"] + compilation_unit.bytecodes_init[contract_name] = contract["bin"] + compilation_unit.bytecodes_runtime[contract_name] = contract["bin-runtime"] + compilation_unit.srcmaps_init[contract_name] = contract["srcmap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = contract["srcmap-runtime"].split(";") + compilation_unit.libraries[contract_name] = contract["libraries"] userdoc = contract.get("userdoc", {}) devdoc = contract.get("devdoc", {}) - crytic_compile.natspec[contract_name] = Natspec(userdoc, devdoc) + compilation_unit.natspec[contract_name] = Natspec(userdoc, devdoc) if contract["is_dependency"]: - crytic_compile.dependencies.add(filename.absolute) - crytic_compile.dependencies.add(filename.relative) - crytic_compile.dependencies.add(filename.short) - crytic_compile.dependencies.add(filename.used) + compilation_unit.crytic_compile.dependencies.add(filename.absolute) + compilation_unit.crytic_compile.dependencies.add(filename.relative) + compilation_unit.crytic_compile.dependencies.add(filename.short) + compilation_unit.crytic_compile.dependencies.add(filename.used) + + +def load_from_compile(crytic_compile: "CryticCompile", loaded_json: Dict) -> Tuple[int, List[str]]: + """ + Load from json + + :param crytic_compile: + :param loaded_json: + :return: + """ + crytic_compile.package_name = loaded_json.get("package", None) + + if "compilation_units" not in loaded_json: + _load_from_compile_legacy(crytic_compile, loaded_json) + + else: + for key, compilation_unit in loaded_json["compilation_units"]: + compilation_unit = CompilationUnit(crytic_compile, key) + compilation_unit.compiler_version = CompilerVersion( + compiler=loaded_json["compiler"]["compiler"], + version=loaded_json["compiler"]["version"], + optimized=loaded_json["compiler"]["optimized"], + ) + for contract_name, contract in loaded_json["contracts"].items(): + compilation_unit.contracts_names.add(contract_name) + filename = Filename( + absolute=contract["filenames"]["absolute"], + relative=contract["filenames"]["relative"], + short=contract["filenames"]["short"], + used=contract["filenames"]["used"], + ) + compilation_unit.contracts_filenames[contract_name] = filename + + compilation_unit.abis[contract_name] = contract["abi"] + compilation_unit.bytecodes_init[contract_name] = contract["bin"] + compilation_unit.bytecodes_runtime[contract_name] = contract["bin-runtime"] + compilation_unit.srcmaps_init[contract_name] = contract["srcmap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = contract["srcmap-runtime"].split( + ";" + ) + compilation_unit.libraries[contract_name] = contract["libraries"] + + userdoc = contract.get("userdoc", {}) + devdoc = contract.get("devdoc", {}) + compilation_unit.natspec[contract_name] = Natspec(userdoc, devdoc) + + if contract["is_dependency"]: + crytic_compile.dependencies.add(filename.absolute) + crytic_compile.dependencies.add(filename.relative) + crytic_compile.dependencies.add(filename.short) + crytic_compile.dependencies.add(filename.used) + compilation_unit.asts = loaded_json["asts"] # Set our filenames - crytic_compile.filenames = set(crytic_compile.contracts_filenames.values()) + for compilation_unit in crytic_compile.compilation_units.values(): + crytic_compile.filenames |= set(compilation_unit.contracts_filenames.values()) crytic_compile.working_dir = loaded_json["working_dir"] diff --git a/crytic_compile/platform/truffle.py b/crytic_compile/platform/truffle.py index 3b429163..57d05fb0 100755 --- a/crytic_compile/platform/truffle.py +++ b/crytic_compile/platform/truffle.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform import solc from crytic_compile.platform.abstract_platform import AbstractPlatform @@ -28,7 +29,7 @@ LOGGER = logging.getLogger("CryticCompile") -def export_to_truffle(crytic_compile: "CryticCompile", **kwargs: str) -> Optional[str]: +def export_to_truffle(crytic_compile: "CryticCompile", **kwargs: str) -> List[str]: """ Export to the truffle format @@ -41,19 +42,24 @@ def export_to_truffle(crytic_compile: "CryticCompile", **kwargs: str) -> Optiona if export_dir and not os.path.exists(export_dir): os.makedirs(export_dir) + compilation_units = list(crytic_compile.compilation_units.values()) + if len(compilation_units) != 1: + raise InvalidCompilation("Truffle export require 1 compilation unit") + compilation_unit = compilation_units[0] + # Loop for each contract filename. results: List[Dict] = [] - for contract_name in crytic_compile.contracts_names: + for contract_name in compilation_unit.contracts_names: # Create the informational object to output for this contract - filename = crytic_compile.contracts_filenames[contract_name] + filename = compilation_unit.contracts_filenames[contract_name] output = { "contractName": contract_name, - "abi": crytic_compile.abi(contract_name), - "bytecode": "0x" + crytic_compile.bytecode_init(contract_name), - "deployedBytecode": "0x" + crytic_compile.bytecode_runtime(contract_name), - "ast": crytic_compile.ast(filename.absolute), - "userdoc": crytic_compile.natspec[contract_name].userdoc.export(), - "devdoc": crytic_compile.natspec[contract_name].devdoc.export(), + "abi": compilation_unit.abi(contract_name), + "bytecode": "0x" + compilation_unit.bytecode_init(contract_name), + "deployedBytecode": "0x" + compilation_unit.bytecode_runtime(contract_name), + "ast": compilation_unit.ast(filename.absolute), + "userdoc": compilation_unit.natspec[contract_name].userdoc.export(), + "devdoc": compilation_unit.natspec[contract_name].devdoc.export(), } results.append(output) @@ -63,7 +69,7 @@ def export_to_truffle(crytic_compile: "CryticCompile", **kwargs: str) -> Optiona with open(path, "w", encoding="utf8") as file_desc: json.dump(output, file_desc) - return export_dir + return [export_dir] class Truffle(AbstractPlatform): @@ -162,6 +168,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): ) # convert bytestrings to unicode strings if truffle_overwrite_config: + assert config_used _reload_config(Path(self._target), config_saved, config_used) LOGGER.info(stdout) @@ -179,6 +186,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): version = None compiler = None + compilation_unit = CompilationUnit(crytic_compile, str(self._target)) for filename_txt in filenames: with open(filename_txt, encoding="utf8") as file_desc: @@ -214,21 +222,21 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): # pylint: disable=raise-missing-from raise InvalidCompilation(txt) - crytic_compile.asts[filename.absolute] = target_loaded["ast"] + compilation_unit.asts[filename.absolute] = target_loaded["ast"] crytic_compile.filenames.add(filename) contract_name = target_loaded["contractName"] - crytic_compile.natspec[contract_name] = natspec - crytic_compile.contracts_filenames[contract_name] = filename - crytic_compile.contracts_names.add(contract_name) - crytic_compile.abis[contract_name] = target_loaded["abi"] - crytic_compile.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( + compilation_unit.natspec[contract_name] = natspec + compilation_unit.contracts_filenames[contract_name] = filename + compilation_unit.contracts_names.add(contract_name) + compilation_unit.abis[contract_name] = target_loaded["abi"] + compilation_unit.bytecodes_init[contract_name] = target_loaded["bytecode"].replace( "0x", "" ) - crytic_compile.bytecodes_runtime[contract_name] = target_loaded[ + compilation_unit.bytecodes_runtime[contract_name] = target_loaded[ "deployedBytecode" ].replace("0x", "") - crytic_compile.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") - crytic_compile.srcmaps_runtime[contract_name] = target_loaded[ + compilation_unit.srcmaps_init[contract_name] = target_loaded["sourceMap"].split(";") + compilation_unit.srcmaps_runtime[contract_name] = target_loaded[ "deployedSourceMap" ].split(";") @@ -246,7 +254,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): else: version, compiler = _get_version(base_cmd, cwd=self._target) - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler=compiler, version=version, optimized=optimized ) @@ -328,15 +336,15 @@ def _get_version(truffle_call: List[str], cwd: str) -> Tuple[str, str]: except OSError as error: # pylint: disable=raise-missing-from raise InvalidCompilation(f"Truffle failed: {error}") - stdout, _ = process.communicate() - stdout = stdout.decode() # convert bytestrings to unicode strings - if not stdout: + sstdout, _ = process.communicate() + ssstdout = sstdout.decode() # convert bytestrings to unicode strings + if not ssstdout: raise InvalidCompilation("Truffle failed to run: 'truffle version'") - stdout = stdout.split("\n") + stdout = ssstdout.split("\n") for line in stdout: if "Solidity" in line: if "native" in line: - return solc.get_version("solc", None), "solc-native" + return solc.get_version("solc", dict()), "solc-native" version = re.findall(r"\d+\.\d+\.\d+", line)[0] compiler = re.findall(r"(solc[a-z\-]*)", line) if len(compiler) > 0: @@ -359,11 +367,11 @@ def _save_config(cwd: Path) -> Tuple[Optional[Path], Optional[Path]]: unique_filename = str(uuid.uuid4()) if Path(cwd, "truffle-config.js").exists(): - shutil.move(Path(cwd, "truffle-config.js"), Path(cwd, unique_filename)) + shutil.move(str(Path(cwd, "truffle-config.js")), str(Path(cwd, unique_filename))) return Path("truffle-config.js"), Path(unique_filename) if Path(cwd, "truffle.js").exists(): - shutil.move(Path(cwd, "truffle.js"), Path(cwd, unique_filename)) + shutil.move(str(Path(cwd, "truffle.js")), str(Path(cwd, unique_filename))) return Path("truffle.js"), Path(unique_filename) return None, None @@ -379,7 +387,7 @@ def _reload_config(cwd: Path, original_config: Optional[Path], tmp_config: Path) """ os.remove(Path(cwd, tmp_config)) if original_config is not None: - shutil.move(Path(cwd, original_config), Path(cwd, tmp_config)) + shutil.move(str(Path(cwd, original_config)), str(Path(cwd, tmp_config))) def _write_config(cwd: Path, original_config: Path, version: Optional[str]): diff --git a/crytic_compile/platform/vyper.py b/crytic_compile/platform/vyper.py index 5978eef9..31aafceb 100644 --- a/crytic_compile/platform/vyper.py +++ b/crytic_compile/platform/vyper.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -46,7 +47,9 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): targets_json = _run_vyper(target, vyper) assert "version" in targets_json - crytic_compile.compiler_version = CompilerVersion( + compilation_unit = CompilationUnit(crytic_compile, str(target)) + + compilation_unit.compiler_version = CompilerVersion( compiler="vyper", version=targets_json["version"], optimized=False ) @@ -57,21 +60,23 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): contract_name = Path(target).parts[-1] - crytic_compile.contracts_names.add(contract_name) - crytic_compile.contracts_filenames[contract_name] = contract_filename - crytic_compile.abis[contract_name] = info["abi"] - crytic_compile.bytecodes_init[contract_name] = info["bytecode"].replace("0x", "") - crytic_compile.bytecodes_runtime[contract_name] = info["bytecode_runtime"].replace("0x", "") - crytic_compile.srcmaps_init[contract_name] = [] - crytic_compile.srcmaps_runtime[contract_name] = [] + compilation_unit.contracts_names.add(contract_name) + compilation_unit.contracts_filenames[contract_name] = contract_filename + compilation_unit.abis[contract_name] = info["abi"] + compilation_unit.bytecodes_init[contract_name] = info["bytecode"].replace("0x", "") + compilation_unit.bytecodes_runtime[contract_name] = info["bytecode_runtime"].replace( + "0x", "" + ) + compilation_unit.srcmaps_init[contract_name] = [] + compilation_unit.srcmaps_runtime[contract_name] = [] crytic_compile.filenames.add(contract_filename) # Natspec not yet handled for vyper - crytic_compile.natspec[contract_name] = Natspec({}, {}) + compilation_unit.natspec[contract_name] = Natspec({}, {}) ast = _get_vyper_ast(target, vyper) - crytic_compile.asts[contract_filename.absolute] = ast + compilation_unit.asts[contract_filename.absolute] = ast def is_dependency(self, _path): """ diff --git a/crytic_compile/platform/waffle.py b/crytic_compile/platform/waffle.py index d9037f63..674b0729 100755 --- a/crytic_compile/platform/waffle.py +++ b/crytic_compile/platform/waffle.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Dict, List +from crytic_compile.compilation_unit import CompilationUnit from crytic_compile.compiler.compiler import CompilerVersion from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.exceptions import InvalidCompilation @@ -64,7 +65,7 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): potential_config_files = list(Path(target).rglob("*waffle*.json")) if potential_config_files and len(potential_config_files) == 1: - config_file = potential_config_files[0] + config_file = str(potential_config_files[0]) # Read config file if config_file: @@ -172,6 +173,8 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): optimized = None + compilation_unit = CompilationUnit(crytic_compile, str(target)) + for contract in target_all["contracts"]: target_loaded = target_all["contracts"][contract] contract = contract.split(":") @@ -181,31 +184,31 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str): contract_name = contract[1] - crytic_compile.asts[filename.absolute] = target_all["sources"][contract[0]]["AST"] + compilation_unit.asts[filename.absolute] = target_all["sources"][contract[0]]["AST"] crytic_compile.filenames.add(filename) - crytic_compile.contracts_filenames[contract_name] = filename - crytic_compile.contracts_names.add(contract_name) - crytic_compile.abis[contract_name] = target_loaded["abi"] + compilation_unit.contracts_filenames[contract_name] = filename + compilation_unit.contracts_names.add(contract_name) + compilation_unit.abis[contract_name] = target_loaded["abi"] userdoc = target_loaded.get("userdoc", {}) devdoc = target_loaded.get("devdoc", {}) natspec = Natspec(userdoc, devdoc) - crytic_compile.natspec[contract_name] = natspec + compilation_unit.natspec[contract_name] = natspec - crytic_compile.bytecodes_init[contract_name] = target_loaded["evm"]["bytecode"][ + compilation_unit.bytecodes_init[contract_name] = target_loaded["evm"]["bytecode"][ "object" ] - crytic_compile.srcmaps_init[contract_name] = target_loaded["evm"]["bytecode"][ + compilation_unit.srcmaps_init[contract_name] = target_loaded["evm"]["bytecode"][ "sourceMap" ].split(";") - crytic_compile.bytecodes_runtime[contract_name] = target_loaded["evm"][ + compilation_unit.bytecodes_runtime[contract_name] = target_loaded["evm"][ "deployedBytecode" ]["object"] - crytic_compile.srcmaps_runtime[contract_name] = target_loaded["evm"][ + compilation_unit.srcmaps_runtime[contract_name] = target_loaded["evm"][ "deployedBytecode" ]["sourceMap"].split(";") - crytic_compile.compiler_version = CompilerVersion( + compilation_unit.compiler_version = CompilerVersion( compiler=compiler, version=version, optimized=optimized ) From f2a8952d066cb3f6eb443ec3650edd3d406fb3a4 Mon Sep 17 00:00:00 2001 From: Josselin Date: Fri, 9 Apr 2021 17:21:13 +0200 Subject: [PATCH 2/6] Minor --- crytic_compile/__init__.py | 1 + crytic_compile/__main__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crytic_compile/__init__.py b/crytic_compile/__init__.py index 1c9cbeee..36b81997 100644 --- a/crytic_compile/__init__.py +++ b/crytic_compile/__init__.py @@ -3,6 +3,7 @@ """ from .crytic_compile import CryticCompile, compile_all, is_supported +from .compilation_unit import CompilationUnit from .cryticparser import cryticparser from .platform import InvalidCompilation from .utils.zip import save_to_zip diff --git a/crytic_compile/__main__.py b/crytic_compile/__main__.py index e439623b..4493d8c0 100644 --- a/crytic_compile/__main__.py +++ b/crytic_compile/__main__.py @@ -153,7 +153,7 @@ def __call__(self, parser, args, values, option_string=None): def _print_filenames(compilation: "CryticCompile"): printed_filenames = set() for compilation_id, compilation_unit in compilation.compilation_units.items(): - print(f"Compilation unit: {compilation_id} ({len(compilation_unit.contracts_names)} files)") + print(f"Compilation unit: {compilation_id} ({len(compilation_unit.contracts_names)} files, solc {compilation_unit.compiler_version.version})") for contract in compilation_unit.contracts_names: filename = compilation_unit.filename_of_contract(contract) unique_id = f"{contract} - {filename} - {compilation_id}" From f61410f1217be1c273838c613fec68f4da99c7b2 Mon Sep 17 00:00:00 2001 From: Josselin Date: Fri, 9 Apr 2021 17:21:37 +0200 Subject: [PATCH 3/6] black --- crytic_compile/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crytic_compile/__main__.py b/crytic_compile/__main__.py index 4493d8c0..c87b086b 100644 --- a/crytic_compile/__main__.py +++ b/crytic_compile/__main__.py @@ -153,7 +153,9 @@ def __call__(self, parser, args, values, option_string=None): def _print_filenames(compilation: "CryticCompile"): printed_filenames = set() for compilation_id, compilation_unit in compilation.compilation_units.items(): - print(f"Compilation unit: {compilation_id} ({len(compilation_unit.contracts_names)} files, solc {compilation_unit.compiler_version.version})") + print( + f"Compilation unit: {compilation_id} ({len(compilation_unit.contracts_names)} files, solc {compilation_unit.compiler_version.version})" + ) for contract in compilation_unit.contracts_names: filename = compilation_unit.filename_of_contract(contract) unique_id = f"{contract} - {filename} - {compilation_id}" From a19c75c3b3368dc142575786945b01269f8a12fe Mon Sep 17 00:00:00 2001 From: Josselin Date: Mon, 26 Apr 2021 10:52:54 +0200 Subject: [PATCH 4/6] Generate one combined_solc.json file in solc export if there is 1 compil unit --- crytic_compile/platform/solc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crytic_compile/platform/solc.py b/crytic_compile/platform/solc.py index 9bccbe36..788747bb 100644 --- a/crytic_compile/platform/solc.py +++ b/crytic_compile/platform/solc.py @@ -88,10 +88,16 @@ def export_to_solc(crytic_compile: "CryticCompile", **kwargs: str) -> List[str]: :return: """ # Obtain objects to represent each contract - - paths = [] export_dir = kwargs.get("export_dir", "crytic-export") + if len(crytic_compile.compilation_units) == 1: + compilation_unit = list(crytic_compile.compilation_units.values())[0] + path = export_to_solc_from_compilation_unit(compilation_unit, "combined_solc", export_dir) + if path: + return [path] + return [] + + paths = [] for key, compilation_unit in crytic_compile.compilation_units.items(): path = export_to_solc_from_compilation_unit(compilation_unit, key, export_dir) if path: From 85f7ce31efb2db4e784be9ed99eab5c9ef5c16e7 Mon Sep 17 00:00:00 2001 From: Josselin Date: Mon, 26 Apr 2021 13:55:56 +0200 Subject: [PATCH 5/6] Use random ID instead of "." --- crytic_compile/compilation_unit.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crytic_compile/compilation_unit.py b/crytic_compile/compilation_unit.py index 7066963c..e2823281 100644 --- a/crytic_compile/compilation_unit.py +++ b/crytic_compile/compilation_unit.py @@ -1,4 +1,5 @@ import re +import uuid from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union import sha3 @@ -47,6 +48,10 @@ def __init__(self, crytic_compile: "CryticCompile", unique_id: str): ) self._crytic_compile: "CryticCompile" = crytic_compile + + if unique_id == ".": + unique_id = uuid.uuid4() + crytic_compile.compilation_units[unique_id] = self self._unique_id = unique_id From 165df536b601d8294c7b57f4a1d1944cff5c05c9 Mon Sep 17 00:00:00 2001 From: Josselin Date: Mon, 26 Apr 2021 14:13:15 +0200 Subject: [PATCH 6/6] Add is_is_multiple_compilation_unit API --- crytic_compile/crytic_compile.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crytic_compile/crytic_compile.py b/crytic_compile/crytic_compile.py index dc0d3aaf..df53c4d2 100644 --- a/crytic_compile/crytic_compile.py +++ b/crytic_compile/crytic_compile.py @@ -123,6 +123,17 @@ def compilation_units(self) -> Dict[str, CompilationUnit]: """ return self._compilation_units + def is_in_multiple_compilation_unit(self, contract: str) -> bool: + """ + Check if the contract is share by multiple compilation unit + + """ + count = 0 + for compilation_unit in self._compilation_units.values(): + if contract in compilation_unit.contracts_names: + count += 1 + return count >= 2 + ################################################################################### ################################################################################### # region Filenames