Substrate support preview

.github/workflows/docs.yml

@@ -18,7 +18,8 @@ on:
       - '.github/workflows/installer.yml'
 
 env:
-  FRONTEND_BRANCH: dev
+  # FIXME: substrate preview
+  FRONTEND_BRANCH: feat/substrate
   GITHUB_TOKEN: ${{ secrets.DOCS_GITHUB_TOKEN }}
 
 jobs:
@@ -64,7 +65,8 @@ jobs:
 
       # NOTE: Installer is tested in `installer` workflow
       - name: Commit and push frontend
-        if: github.ref_type == 'tag'
+        # FIXME: substrate preview
+        # if: github.ref_type == 'tag'
         run: |
           cp src/dipdup/install.py ../interface/public/install.py
           cd ../interface

.gitignore

@@ -12,6 +12,7 @@
 !**/pdm.lock
 !**/README.md
 !**/.keep
+!**/py.typed
 
 # Add Python code
 !**/*.py

.vscode/launch.json

@@ -33,6 +33,38 @@
         "DIPDUP_DEBUG": "1"
       }
     },
+    {
+      "name": "demo_substrate_events: run",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "dipdup",
+      "args": [
+        "-e",
+        ".env",
+        "run"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/src/demo_substrate_events",
+      "env": {
+        "DIPDUP_DEBUG": "1"
+      }
+    },
+    {
+      "name": "demo_substrate_events: init",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "dipdup",
+      "args": [
+        "-e",
+        ".env",
+        "init"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/src/demo_substrate_events",
+      "env": {
+        "DIPDUP_DEBUG": "1"
+      }
+    },
     {
       "name": "demo_evm_events: run",
       "type": "debugpy",

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic
 
 Releases prior to 7.0 has been removed from this file to declutter search results; see the [archived copy](https://github.com/dipdup-io/dipdup/blob/8.0.0b5/CHANGELOG.md) for the full list.
 
+## [Unreleased]
+
+### Fixed
+
+- config: Fixed setting default loglevels when `logging` is a dict.
+
 ## [8.1.1] - 2024-10-17
 
 ### Fixed

Makefile

@@ -13,9 +13,17 @@ FRONTEND_PATH=../interface
 help:           ## Show this help (default)
 	@grep -Fh "##" $(MAKEFILE_LIST) | grep -Fv grep -F | sed -e 's/\\$$//' | sed -e 's/##//'
 
-install:
+##
+##-- Dependencies
+##
+
+install:        ## Install dependencies
 	pdm sync --clean
 
+update:         ## Update dependencies and dump requirements.txt
+	pdm update
+	pdm export --without-hashes -f requirements --prod -o requirements.txt
+
 ##
 ##-- CI
 ##
@@ -53,13 +61,14 @@ mypy:           ## Lint with mypy
 docs_build: docs
 docs:           ## Build docs
 	python scripts/docs.py check-links --source docs
-	python scripts/docs.py dump-references
+	# FIXME
+	# python scripts/docs.py dump-references  || true
 	python scripts/docs.py dump-demos
 	python scripts/docs.py dump-metrics
 	python scripts/docs.py dump-jsonschema
 	python scripts/docs.py merge-changelog
 	python scripts/docs.py markdownlint
-	python scripts/docs.py build --source docs --destination ${FRONTEND_PATH}/content/docs
+	# python scripts/docs.py build --source docs --destination ${FRONTEND_PATH}/content/docs
 
 docs_serve:     ## Build docs and start frontend server
 	python scripts/docs.py build --source docs --destination ${FRONTEND_PATH}/content/docs --watch --serve
@@ -83,10 +92,6 @@ typeignore:     ## Find type:ignore comments
 ##-- Release
 ##
 
-update:         ## Update dependencies and dump requirements.txt
-	pdm update
-	pdm export --without-hashes -f requirements --prod -o requirements.txt
-
 demos:          ## Recreate demo projects from templates
 	python scripts/demos.py render ${DEMO}
 	python scripts/demos.py init ${DEMO}

docs/0.quickstart-substrate.md

@@ -0,0 +1,178 @@
+---
+title: "Quickstart (Substrate)"
+description: "This page will guide you through the steps to get your first selective indexer up and running in a few minutes without getting too deep into the details."
+navigation.icon: "stars"
+network: "ethereum"
+---
+
+# Quickstart
+
+::banner{type="warning"}
+Substrate support is in early preview stage. API and features may change in the future.
+::
+
+This page will guide you through the steps to get your first selective indexer up and running in a few minutes without getting too deep into the details.
+
+Let's create an indexer for the balance transfers in AssetHub network. Our goal is to save all transfers to the database and then calculate some statistics of its holders' activity.
+
+## Install DipDup
+
+A modern Linux/macOS distribution with Python 3.12 installed is required to run DipDup.
+
+The recommended way to install DipDup CLI is [pipx](https://pipx.pypa.io/stable/). We also provide a convenient helper script that installs all necessary tools. Run the following command in your terminal:
+
+{{ #include _curl-spell.md }}
+
+See the [Installation](../docs/1.getting-started/1.installation.md) page for all options.
+
+After installation, run the following command to switch to the preview branch:
+
+```shell [Terminal]
+dipdup self install -f -r feat/substrate
+```
+
+## Create a project
+
+DipDup CLI has a built-in project generator. Run the following command in your terminal:
+
+```shell [Terminal]
+dipdup new
+```
+
+Choose `Substrate` network and `demo_substrate_events` template.
+
+::banner{type="note"}
+Want to skip a tutorial and start from scratch? Choose `[none]` and `demo_blank` instead and proceed to the [Config](../docs/1.getting-started/3.config.md) section.
+::
+
+Follow the instructions; the project will be created in the new directory.
+
+## Write a configuration file
+
+In the project root, you'll find a file named `dipdup.yaml`. It's the main configuration file of your indexer. We will discuss it in detail in the [Config](../docs/1.getting-started/3.config.md) section; now it has the following content:
+
+```yaml [dipdup.yaml]
+{{ #include ../src/demo_substrate_events/dipdup.yaml }}
+```
+
+## Generate types and stubs
+
+Now it's time to generate typeclasses and callback stubs based on definitions from config. Examples below use `demo_substrate_events` as a package name; yours may differ.
+
+Run the following command:
+
+```shell [Terminal]
+dipdup init
+```
+
+DipDup will create a Python package `demo_substrate_events` with everything you need to start writing your indexer. Use `package tree` command to see the generated structure:
+
+```shell [Terminal]
+$ dipdup package tree
+demo_substrate_events [/home/droserasprout/git/dipdup/src/demo_substrate_events]
+├── abi
+│   ├── assethub/v1000000.json
+│   ├── assethub/v1001002.json
+│   ├── ...
+│   └── assethub/v9430.json
+├── configs
+│   ├── dipdup.compose.yaml
+│   ├── dipdup.sqlite.yaml
+│   ├── dipdup.swarm.yaml
+│   └── replay.yaml
+├── deploy
+│   ├── .env.default
+│   ├── Dockerfile
+│   ├── compose.sqlite.yaml
+│   ├── compose.swarm.yaml
+│   ├── compose.yaml
+│   ├── sqlite.env.default
+│   └── swarm.env.default
+├── graphql
+├── handlers
+│   ├── batch.py
+│   └── on_transfer.py
+├── hasura
+├── hooks
+│   ├── on_index_rollback.py
+│   ├── on_reindex.py
+│   ├── on_restart.py
+│   └── on_synchronized.py
+├── models
+│   └── __init__.py
+├── sql
+├── types
+│   ├── assethub/substrate_events/assets_transferred/__init__.py
+│   ├── assethub/substrate_events/assets_transferred/v601.py
+│   └── assethub/substrate_events/assets_transferred/v700.py
+└── py.typed
+```
+
+That's a lot of files and directories! But don't worry, we will need only `models` and `handlers` sections in this guide.
+
+## Define data models
+
+DipDup supports storing data in SQLite, PostgreSQL and TimescaleDB databases. We use custom ORM based on Tortoise ORM as an abstraction layer.
+
+First, you need to define a model class. Our schema will consist of a single model `Holder` with the following fields:
+
+|             |                                     |
+| ----------- | ----------------------------------- |
+| `address`   | account address                     |
+| `balance`   | token amount held by the account    |
+| `turnover`  | total amount of transfer/mint calls |
+| `tx_count`  | number of transfers/mints           |
+| `last_seen` | time of the last transfer/mint      |
+
+Here's how to define this model in DipDup:
+
+```python [models/__init__.py]
+{{ #include ../src/demo_substrate_events/models/__init__.py }}
+```
+
+## Implement handlers
+
+Everything's ready to implement an actual indexer logic.
+
+Our task is to index all the balance updates. Put some code to the `on_transfer` handler callback to process matched logs:
+
+```python [handlers/on_transfer.py]
+{{ #include ../src/demo_substrate_events/handlers/on_transfer.py }}
+```
+
+And that's all! We can run the indexer now.
+
+## Next steps
+
+Run the indexer in memory:
+
+```shell
+dipdup run
+```
+
+Store data in SQLite database:
+
+```shell
+dipdup -c . -c configs/dipdup.sqlite.yaml run
+```
+
+Or spawn a Compose stack with PostgreSQL and Hasura:
+
+```shell
+cd deploy
+cp .env.default .env
+# Edit .env file before running
+docker-compose up
+```
+
+DipDup will fetch all the historical data and then switch to realtime updates. You can check the progress in the logs.
+
+If you use SQLite, run this query to check the data:
+
+```bash
+sqlite3 demo_substrate_events.sqlite 'SELECT * FROM holder LIMIT 10'
+```
+
+If you run a Compose stack, open `http://127.0.0.1:8080` in your browser to see the Hasura console (an exposed port may differ). You can use it to explore the database and build GraphQL queries.
+
+Congratulations! You've just created your first DipDup indexer. Proceed to the Getting Started section to learn more about DipDup configuration and features.

docs/7.references/1.cli.md

@@ -512,4 +512,4 @@ Discord:               <a class="reference external" href="https://discord.gg/aG
 
 </section>
 </section>
-</section>
+</section>