diff --git a/.github/actions/docker-setup/action.yaml b/.github/actions/docker-setup/action.yaml new file mode 100644 index 000000000..f51ff4488 --- /dev/null +++ b/.github/actions/docker-setup/action.yaml @@ -0,0 +1,32 @@ +name: "Docker build setup" +description: | + Runs an opinionated and unified docker build setup action. It does the following: + * Logs in to docker image registries GCP GAR +inputs: + # GCP auth + GCP_WORKLOAD_IDENTITY_PROVIDER: + required: true + description: "GCP Workload Identity provider" + GCP_SERVICE_ACCOUNT_EMAIL: + required: true + description: "GCP service account email" + +runs: + using: composite + steps: + - id: auth + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@dac4e13deb3640f22e3ffe758fd3f95e6e89f712" # pin@v0 + with: + create_credentials_file: false + token_format: "access_token" + access_token_lifetime: 5400 # setting this to 1.5h since sometimes docker builds (special performance builds etc.) take that long. Default is 1h. + workload_identity_provider: ${{ inputs.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ inputs.GCP_SERVICE_ACCOUNT_EMAIL }} + + - name: Login to US multi-region Google Artifact Registry + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # pin@v2 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml new file mode 100644 index 000000000..a4efa473c --- /dev/null +++ b/.github/workflows/build-images.yaml @@ -0,0 +1,36 @@ +name: "Build Docker Images" +on: + # Allow us to run this specific workflow without a PR + workflow_dispatch: + pull_request: + push: + branches: + - main + +# cancel redundant builds +concurrency: + # for push and workflow_dispatch events we use `github.sha` in the concurrency group and don't really cancel each other out/limit concurrency + # for pull_request events newer jobs cancel earlier jobs to save on CI etc. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha || github.head_ref || github.ref }} + cancel-in-progress: true + +env: + GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +permissions: + contents: read + id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry + +jobs: + Build: + strategy: + matrix: + example: [python] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/docker-setup + with: + GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + - run: ./scripts/build-and-push-images.sh ${{ matrix.example }} diff --git a/.vscode/indexer-client-examples.code-workspace b/.vscode/indexer-client-examples.code-workspace new file mode 100644 index 000000000..6720e9ffe --- /dev/null +++ b/.vscode/indexer-client-examples.code-workspace @@ -0,0 +1,26 @@ +{ + "folders": [ + { + "name": "ROOT", + "path": "../" + }, + { + "name": "typescript", + "path": "../typescript" + }, + { + "name": "python", + "path": "../python" + } + ], + "settings": { + "python.formatting.provider": "black", + "typescript.preferences.importModuleSpecifierEnding": "js", + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + } + }, + "extensions": { + "recommendations": ["ms-python.python", "esbenp.prettier-vscode"] + } +} diff --git a/config.yaml b/config.yaml index 7e613352d..d270440b1 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,6 @@ -chain-id: 1 -indexer-endpoint: "34.30.218.153:50051" -x-aptos-data-authorization: "YOUR_TOKEN" -starting-version: 10000 -tablename: "YOUR_TABLENAME" -cursor-filename: "cursor.txt" +chain_id: 1 +indexer_endpoint: "34.30.218.153:50051" +indexer_api_key: "" +starting_version: 10000 +db_connection_uri: "postgresql://" +cursor_filename: "cursor.txt" diff --git a/python/Dockerfile b/python/Dockerfile index d628595e2..5d7372aff 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -4,7 +4,7 @@ FROM python:3.11 RUN pip install "poetry==1.4.2" WORKDIR /app -COPY python/poetry.lock python/pyproject.toml /app/ +COPY poetry.lock pyproject.toml /app/ # Project initialization @@ -12,8 +12,7 @@ RUN poetry config virtualenvs.create false \ && poetry install --only main # Copy files and folders -COPY ./config.yaml /app/configs/config.yaml -COPY python/*.py /app/ -COPY python/aptos /app/aptos +COPY *.py /app/ +COPY /aptos /app/aptos -CMD ["poetry", "run", "python", "grpc_client.py", "--config", "configs/config.yaml"] \ No newline at end of file +CMD ["poetry", "run", "python", "grpc_client.py", "--config", "/app/config/config.yaml"] diff --git a/python/config.py b/python/config.py new file mode 100644 index 000000000..0b0c2c481 --- /dev/null +++ b/python/config.py @@ -0,0 +1,30 @@ +import yaml +from pydantic import BaseSettings +from pydantic.env_settings import SettingsSourceCallable + + +class Config(BaseSettings): + chain_id: int + indexer_endpoint: str + indexer_api_key: str + starting_version: int + db_connection_uri: str + cursor_filename: str + + class Config: + # change order of priority of settings sources such that environment variables take precedence over config file settings + # inspired by https://docs.pydantic.dev/usage/settings/#changing-priority + @classmethod + def customise_sources( + cls, + init_settings: SettingsSourceCallable, + env_settings: SettingsSourceCallable, + file_secret_settings: SettingsSourceCallable, + ) -> tuple[SettingsSourceCallable, ...]: + return env_settings, init_settings, file_secret_settings + + @classmethod + def from_yaml_file(cls, path: str): + with open(path, "r") as file: + config = yaml.safe_load(file) + return cls(**config) diff --git a/python/create_table.py b/python/create_table.py index c8b4fd672..127e8bad9 100644 --- a/python/create_table.py +++ b/python/create_table.py @@ -2,16 +2,20 @@ from sqlalchemy.orm import declarative_base import argparse -import yaml + +from config import Config parser = argparse.ArgumentParser() -parser.add_argument('-c', '--config', help='Path to config file', required=True) +parser.add_argument("-c", "--config", help="Path to config file", required=True) args = parser.parse_args() +config = Config.from_yaml_file(args.config) + Base = declarative_base() + class Event(Base): - __tablename__ = 'events' + __tablename__ = "events" sequence_number = Column(BigInteger, primary_key=True) creation_number = Column(BigInteger, primary_key=True) @@ -23,8 +27,6 @@ class Event(Base): inserted_at = Column(DateTime) event_index = Column(BigInteger) -with open(args.config, 'r') as file: - config = yaml.safe_load(file) -engine = create_engine(config['tablename']) -Base.metadata.create_all(engine) \ No newline at end of file +engine = create_engine(config.db_connection_uri) +Base.metadata.create_all(engine) diff --git a/python/docker-compose.yml b/python/docker-compose.yml new file mode 100644 index 000000000..db1dcbd6a --- /dev/null +++ b/python/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + index-processor: + build: . + environment: + DB_CONNECTION_URI: postgresql://postgres:postgres@db:5432/postgres + depends_on: + - db + volumes: + - ../config.yaml:/app/config/config.yaml + db: + image: postgres:15.2 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - db-data:/var/lib/postgresql/data +volumes: + db-data: diff --git a/python/grpc_client.py b/python/grpc_client.py index ad98b1e58..49b1328c5 100644 --- a/python/grpc_client.py +++ b/python/grpc_client.py @@ -1,3 +1,4 @@ +from config import Config from grpc_parser import parse from aptos.datastream.v1 import datastream_pb2_grpc @@ -11,39 +12,52 @@ import argparse import base64 import datetime -import yaml +import json parser = argparse.ArgumentParser() -parser.add_argument('-c', '--config', help='Path to config file', required=True) +parser.add_argument("-c", "--config", help="Path to config file", required=True) args = parser.parse_args() +config = Config.from_yaml_file(args.config) -with open(args.config, 'r') as file: - config = yaml.safe_load(file) +metadata = (("x-aptos-data-authorization", config.indexer_api_key),) +options = [("grpc.max_receive_message_length", -1)] +engine = create_engine(config.db_connection_uri) -metadata = (("x-aptos-data-authorization", config["x-aptos-data-authorization"]),) -options = [('grpc.max_receive_message_length', -1)] -engine = create_engine(config['tablename']) - -with grpc.insecure_channel(config["indexer-endpoint"], options=options) as channel: +with grpc.insecure_channel(config.indexer_endpoint, options=options) as channel: stub = datastream_pb2_grpc.IndexerStreamStub(channel) - current_transaction_version = config["starting-version"] + current_transaction_version = config.starting_version - for response in stub.RawDatastream(datastream_pb2.RawDatastreamRequest(starting_version=config["starting-version"]), metadata=metadata): + for response in stub.RawDatastream( + datastream_pb2.RawDatastreamRequest(starting_version=config.starting_version), + metadata=metadata, + ): chain_id = response.chain_id - if chain_id != config["chain-id"]: - raise Exception("Chain ID mismatch. Expected chain ID is: " + str(config["chain-id"]) + ", but received chain ID is: " + str(chain_id)) - + if chain_id != config.chain_id: + raise Exception( + "Chain ID mismatch. Expected chain ID is: " + + str(config.chain_id) + + ", but received chain ID is: " + + str(chain_id) + ) + transactions_output = response.data for transaction_output in transactions_output.transactions: # Decode transaction data - decoded_transaction = base64.b64decode(transaction_output.encoded_proto_data) + decoded_transaction = base64.b64decode( + transaction_output.encoded_proto_data + ) transaction = transaction_pb2.Transaction() transaction.ParseFromString(decoded_transaction) transaction_version = transaction.version if transaction_version != current_transaction_version: - raise Exception("Transaction version mismatch. Expected transaction version is: " + str(current_transaction_version) + ", but received transaction version is: " + str(transaction_version)) + raise Exception( + "Transaction version mismatch. Expected transaction version is: " + + str(current_transaction_version) + + ", but received transaction version is: " + + str(transaction_version) + ) current_transaction_version += 1 parsed_objs = parse(transaction) @@ -53,8 +67,24 @@ with Session(engine) as session, session.begin(): session.add_all(parsed_objs) + if (current_transaction_version % 1000) == 0: + print( + json.dumps( + { + "message": "Successfully processed transaction", + "last_success_transaction_version": current_transaction_version, + } + ) + ) # Keep track of last successfully processed transaction version - cursor_file = open(config["cursor-filename"], 'w+') - cursor_file.write("last_success_transaction_version=" + str(current_transaction_version) + "\n") - cursor_file.write("last_updated=" + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')) - cursor_file.close() + cursor_file = open(config.cursor_filename, "w+") + cursor_file.write( + "last_success_transaction_version=" + + str(current_transaction_version) + + "\n" + ) + cursor_file.write( + "last_updated=" + + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + ) + cursor_file.close() diff --git a/python/grpc_parser.py b/python/grpc_parser.py index 382d6149a..c936f81b0 100644 --- a/python/grpc_parser.py +++ b/python/grpc_parser.py @@ -3,12 +3,13 @@ from create_table import Event import datetime + def parse(transaction: transaction_pb2.Transaction): # Custom filtering # Here we filter out all transactions that are not of type TRANSACTION_TYPE_USER if transaction.type != transaction_pb2.Transaction.TRANSACTION_TYPE_USER: return - + # Parse Transaction struct transaction_version = transaction.version transaction_block_height = transaction.block_height @@ -16,7 +17,7 @@ def parse(transaction: transaction_pb2.Transaction): user_transaction = transaction.user # Parse Event struct - event_db_objs = [] + event_db_objs: list[Event] = [] for event_index, event in enumerate(user_transaction.events): creation_number = event.key.creation_number sequence_number = event.sequence_number @@ -37,12 +38,16 @@ def parse(transaction: transaction_pb2.Transaction): event_index=event_index, ) event_db_objs.append(event_db_obj) - + return event_db_objs + def parse_timestamp(timestamp: timestamp_pb2.Timestamp): - datetime_obj = datetime.datetime.fromtimestamp(timestamp.seconds + timestamp.nanos * 1e-9) - return datetime_obj.strftime('%Y-%m-%d %H:%M:%S.%f') + datetime_obj = datetime.datetime.fromtimestamp( + timestamp.seconds + timestamp.nanos * 1e-9 + ) + return datetime_obj.strftime("%Y-%m-%d %H:%M:%S.%f") + def standardize_address(address: str): return "0x" + address diff --git a/python/poetry.lock b/python/poetry.lock index 731a2b481..6a0ca04e9 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -1,5 +1,80 @@ # This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + [[package]] name = "greenlet" version = "2.0.2" @@ -192,6 +267,58 @@ grpcio = ">=1.53.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + +[[package]] +name = "platformdirs" +version = "3.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, +] + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + [[package]] name = "protobuf" version = "4.22.3" @@ -238,6 +365,59 @@ files = [ {file = "psycopg2-2.9.6.tar.gz", hash = "sha256:f15158418fd826831b28585e2ab48ed8df2d0d98f502a2b4fe619e7d5ca29011"}, ] +[[package]] +name = "pydantic" +version = "1.10.7" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d"}, + {file = "pydantic-1.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"}, + {file = "pydantic-1.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca"}, + {file = "pydantic-1.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d"}, + {file = "pydantic-1.10.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a"}, + {file = "pydantic-1.10.7-cp37-cp37m-win_amd64.whl", hash = "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209"}, + {file = "pydantic-1.10.7-cp38-cp38-win_amd64.whl", hash = "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5"}, + {file = "pydantic-1.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e"}, + {file = "pydantic-1.10.7-py3-none-any.whl", hash = "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6"}, + {file = "pydantic-1.10.7.tar.gz", hash = "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pyyaml" version = "6.0" @@ -398,4 +578,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "a0f4f60d49489437dd6cbf7943352fd5aa8eecf3c1dc3ec3f27183bcfb89efe1" +content-hash = "f34a2d7a5d23cadc3b429711c473a77c510232e8f1a4ec7f269f0074254d9981" diff --git a/python/pyproject.toml b/python/pyproject.toml index d80438a55..4337f340c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -13,9 +13,11 @@ PyYAML = "^6.0" SQLAlchemy = "^2.0.9" grpcio-tools = "^1.53.0" protobuf = "^4.22.3" +pydantic = "^1.10.7" [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.53.0" +black = "^23.3.0" [build-system] requires = ["poetry-core>=1.4.2"] diff --git a/scripts/build-and-push-images.sh b/scripts/build-and-push-images.sh new file mode 100755 index 000000000..193460c1a --- /dev/null +++ b/scripts/build-and-push-images.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) Aptos +# SPDX-License-Identifier: Apache-2.0 + +# This script is to build and push all the docker images of all indexer client examples +# You need to execute this from the repository root as working directory +# E.g. scripts/build-and-push-images.sh +# If you want to build a specific example only: +# scripts/build-and-push-images.sh +# E.g. scripts/build-and-push-images.sh python +# Note that this uses kaniko (https://github.com/GoogleContainerTools/kaniko) instead of vanilla docker to build the images, which has good remote caching support + +set -ex + +TARGET_REGISTRY="us-docker.pkg.dev/aptos-registry/docker/indexer-client-examples" +# take GIT_SHA from environment variable if set, otherwise use git rev-parse HEAD +GIT_SHA="${GIT_SHA:-$(git rev-parse HEAD)}" +ALL_EXAMPLES="python" +EXAMPLE_TO_BUILD_ARG="${1:-all}" + +if [ "$EXAMPLE_TO_BUILD_ARG" == "all" ]; then + EXAMPLES_TO_BUILD="$ALL_EXAMPLES" +else + EXAMPLES_TO_BUILD="$EXAMPLE_TO_BUILD_ARG" +fi + +if [ "$CI" == "true" ]; then + CREDENTIAL_MOUNT="$HOME/.docker/:/kaniko/.docker/:ro" +else + # locally we mount gcloud config credentials + CREDENTIAL_MOUNT="$HOME/.config/gcloud:/root/.config/gcloud:ro" +fi + +for example in $EXAMPLES_TO_BUILD; do + docker run \ + --rm \ + -v $CREDENTIAL_MOUNT \ + -v $(pwd)/$example:/workspace \ + gcr.io/kaniko-project/executor:latest \ + --dockerfile /workspace/Dockerfile \ + --destination "$TARGET_REGISTRY/$example:$GIT_SHA" \ + --context dir:///workspace/ \ + --cache=true +done