Event Stream w/ Context for Coins

.github/actions/protobuf-compatibility-check/action.yaml

@@ -0,0 +1,59 @@
+name: "Protobuf compatibility check"
+description: |
+  "Runs a compatibility check for protobuf files."
+inputs:
+  upstream_commit_version:
+    description: "Commit version of aptos-core that upstream is using"
+    required: true
+    default: "main"
+
+runs:
+  using: composite
+  steps:
+    - name: Install the buf CLI
+      shell: bash
+      run: |
+        BIN="/usr/local/bin" && \
+        VERSION="1.30.0" && \
+        curl -sSL \
+        "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m)" \
+        -o "${BIN}/buf" && \
+        chmod +x "${BIN}/buf"
+
+    # Checkout current repo with current commit
+    - name: Checkout current repo
+      uses: actions/checkout@v4
+      with:
+        path: "aptos-indexer-processors"
+
+    - name: Parse the toml in this repo
+      id: get_tag_output
+      shell: bash
+      run: |
+        set -ex
+        curl -sSLf "$(curl -sSLf https://api.github.com/repos/tomwright/dasel/releases/latest | grep browser_download_url | grep linux_amd64 | grep -v .gz | cut -d\" -f 4)" -L -o dasel && chmod +x dasel
+        mv ./dasel /usr/local/bin/dasel
+        cd aptos-indexer-processors
+        tag_output=$(dasel -r toml  -f rust/Cargo.toml workspace.dependencies.aptos-protos.rev -w - )
+        echo "::set-output name=tag_output::$tag_output"
+
+    - name: Checkout aptos-core
+      uses: actions/checkout@v4
+      with:
+        repository: "aptos-labs/aptos-core"
+        path: "aptos-core"
+        ref: ${{ steps.get_tag_output.outputs.tag_output }}
+
+    - name: Check compatibility
+      shell: bash
+      run: |
+        set -ex
+        cd aptos-core/protos/proto 
+        repo_url="https://github.com/aptos-labs/aptos-core.git#tag=${{  inputs.upstream_commit_version }},subdir=protos/proto"
+        if buf breaking --against "$repo_url" --verbose; then
+          echo "No breaking changes found"
+        else
+          echo "Breaking changes found"
+          echo "Did new oneof/enum fields get added?"
+          exit 1
+        fi

.github/workflows/build-images.yaml

@@ -29,7 +29,7 @@ jobs:
   Build:
     strategy:
       matrix:
-        example: [python, rust]
+        example: [rust]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check-protobuf-compatibility.yaml

@@ -0,0 +1,27 @@
+name: "Check Protobuf Compatibility"
+on:
+  workflow_dispatch:
+    inputs:
+      upstream_commit_version:
+        description: 'The commit version to check compatibility against'
+        required: false
+        default: 'main'
+  pull_request:
+
+# cancel redundant builds
+concurrency:
+  # for push and workflow_dispatch events we use `github.sha` in the concurrency group and don't really cancel each other out/limit concurrency
+  # for pull_request events newer jobs cancel earlier jobs to save on CI etc.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha || github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+
+jobs:
+  CheckProtobufCompatibilityAgainstCurrentPR:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/protobuf-compatibility-check/
+        with:
+          # Either current PR or the latest commit on the branch
+          upstream_commit_version: ${{ github.event.inputs.upstream_commit_version || 'main' }}

.github/workflows/copy-processor-images-to-dockerhub-release.yaml

@@ -17,7 +17,7 @@ jobs:
   copy-images-to-docker-hub:
     strategy:
       matrix:
-        language: ["rust", "python"]
+        language: ["rust"]
     uses: ./.github/workflows/copy-processor-images-to-dockerhub.yaml
     with:
       processor_language: ${{ matrix.language }}

.github/workflows/copy-processor-images-to-dockerhub.yaml

@@ -69,6 +69,6 @@ jobs:
         env:
           FORCE_COLOR: 3 # Force color output as per https://github.com/google/zx#using-github-actions
           GIT_SHA: ${{ inputs.GIT_SHA }}
-          GCP_DOCKER_ARTIFACT_PROCESSOR_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}
         run: pnpm release-processor-images --language=${{ inputs.processor_language }} --version-tag=${{ inputs.version_tag }} --wait-for-image-seconds=3600
         working-directory: scripts

.github/workflows/lint.yaml

@@ -31,10 +31,19 @@ jobs:
       - uses: actions/checkout@v3
       - name: Install deps and run linter
         run: |
+          sudo apt update && sudo apt install libdw-dev
           cargo install cargo-sort
           rustup update
           rustup toolchain install nightly
           rustup component add clippy --toolchain nightly
           rustup component add rustfmt --toolchain nightly
           scripts/rust_lint.sh --check
         working-directory: rust
+      - run: bash scripts/check_banned_deps.sh
+        working-directory: rust
+      - name: Ensure the --no-default-features build passes too
+        run: cargo build --no-default-features
+        working-directory: rust
+      - name: Ensure tests pass
+        run: cargo test
+        working-directory: rust

.github/workflows/nightly-check-protobuf-compatibility.yaml

@@ -0,0 +1,39 @@
+name: "Nightly Check Protobuf Compatibility"
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 9 * * *"
+
+# cancel redundant builds
+concurrency:
+  # for push and workflow_dispatch events we use `github.sha` in the concurrency group and don't really cancel each other out/limit concurrency
+  # for pull_request events newer jobs cancel earlier jobs to save on CI etc.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha || github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  CheckProtobufCompatibilityAgainstTestnet:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get current version
+        id: get_upstream_commit_version
+        run: |
+          echo "::set-output name=upstream_commit_version::$(curl -s https://api.testnet.aptoslabs.com/v1 | jq -r .git_hash)"
+
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/protobuf-compatibility-check/
+        with:
+          upstream_commit_version: ${{ steps.get_upstream_commit_version.outputs.upstream_commit_version }}
+
+  CheckProtobufCompatibilityAgainstMainnet:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get current version
+        id: get_upstream_commit_version
+        run: |
+          echo "::set-output name=upstream_commit_version::$(curl -s https://api.mainnet.aptoslabs.com/v1 | jq -r .git_hash)"
+
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/protobuf-compatibility-check/
+        with:
+          upstream_commit_version: ${{ steps.get_upstream_commit_version.outputs.upstream_commit_version }}

README.md

@@ -24,3 +24,6 @@ This guide will get you started with creating an Aptos indexer with custom parsi
 
 ## [Aptos Indexer GRPC Release Notes](https://github.com/aptos-labs/aptos-core/blob/main/ecosystem/indexer-grpc/release_notes.md)
 
+
+> [!WARNING]  
+> The typescript implementation is known to get stuck when there are lots of data to process. The issue is with the GRPC client and we haven't had a chance to optimize. Please proceed with caution.