alternative demo

.vscode/launch.json

@@ -65,6 +65,38 @@
         "DIPDUP_DEBUG": "1"
       }
     },
+    {
+      "name": "demo_substrate_events_orm: run",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "dipdup",
+      "args": [
+        "-e",
+        ".env",
+        "run"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/src/demo_substrate_events_orm",
+      "env": {
+        "DIPDUP_DEBUG": "1"
+      }
+    },
+    {
+      "name": "demo_substrate_events_orm: init",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "dipdup",
+      "args": [
+        "-e",
+        ".env",
+        "init"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/src/demo_substrate_events_orm",
+      "env": {
+        "DIPDUP_DEBUG": "1"
+      }
+    },
     {
       "name": "demo_evm_events: run",
       "type": "debugpy",

CHANGELOG.md

@@ -10,6 +10,7 @@ Releases prior to 7.0 has been removed from this file to declutter search result
 
 ### Fixed
 
+- cli: Fixed `--template` option being ignored when `--quiet` flag is set.
 - config: Fixed setting default loglevels when `logging` is a dict.
 
 ## [8.1.1] - 2024-10-17

docs/7.references/2.config.md

@@ -1216,4 +1216,4 @@ description: "Config file reference"
 </ul>
 </dd>
 </dl>
-</dd></dl>
+</dd></dl>

docs/7.references/3.context.md

@@ -135,13 +135,15 @@ description: "Context reference"
 
 ## dipdup.context.DipDupContext.execute_sql_query
 
-<em class="property"><span class="pre">async</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">DipDupContext.</span></span><span class="sig-name descname"><span class="pre">execute_sql_query</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">values</span></span></em><span class="sig-paren">)</span></dt>
+<em class="property"><span class="pre">async</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">DipDupContext.</span></span><span class="sig-name descname"><span class="pre">execute_sql_query</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span></dt>
 <dd><p>Executes SQL query with given name included with the project</p>
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>name</strong> (<em>str</em>) – SQL query name within <cite>sql</cite> directory</p></li>
-<li><p><strong>values</strong> (<em>Any</em>) – Values to pass to the query</p></li>
+<li><p><strong>values</strong> – Values to pass to the query</p></li>
+<li><p><strong>args</strong> (<em>Any</em>)</p></li>
+<li><p><strong>kwargs</strong> (<em>Any</em>)</p></li>
 </ul>
 </dd>
 <dt class="field-even" style="color: var(--txt-primary);">Return type<span class="colon">:</span></dt>
@@ -391,4 +393,4 @@ to provide a generic metadata interface (see docs).</p>
 <dd class="field-even"><p>None</p>
 </dd>
 </dl>
-</dd></dl>
+</dd></dl>

docs/7.references/4.models.md

@@ -859,4 +859,4 @@ description: "Models reference"
 </dd></dl>
 
 </section>
-</section>
+</section>

docs/8.examples/_demos_table.md

@@ -3,6 +3,7 @@
 |-|-|-|-|
 | demo_blank |  | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_blank) |
 | demo_substrate_events |  | Substrate balance transfers [PREVIEW] | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_substrate_events) |
+| demo_substrate_events_orm |  | Substrate balance transfers (ORM) [PREVIEW] | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_substrate_events_orm) |
 | demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_evm_events) |
 | demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_evm_transactions) |
 | demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.1/src/demo_evm_uniswap) |

docs/9.release-notes/_8.0_changelog.md

@@ -25,6 +25,7 @@
 - cli: Don't save reports for successful test runs.
 - cli: Don't update existing installation in `self install` command unless asked to.
 - cli: Fixed `--pre` installer flag.
+- cli: Fixed `--template` option being ignored when `--quiet` flag is set.
 - cli: Fixed env files not being loaded in some commands.
 - cli: Fixed errors raised when the project package is invalid.
 - cli: Fixed progress estimation when there are indexes with `last_level` option set.

src/demo_blank/deploy/Dockerfile

@@ -1,6 +1,6 @@
 FROM dipdup/dipdup:8
 # FROM ghcr.io/dipdup-io/dipdup:8
-FROM ghcr.io/dipdup-io/dipdup:feat-substrate
+# FROM ghcr.io/dipdup-io/dipdup:next
 
 # COPY --chown=dipdup pyproject.toml README.md .
 # RUN pip install .

src/demo_blank/replay.yaml

@@ -0,0 +1,5 @@
+spec_version: 3.0
+replay:
+  description: Empty config for a fresh start
+  package: demo_blank
+  template: demo_blank

src/demo_evm_events/replay.yaml

@@ -0,0 +1,5 @@
+spec_version: 3.0
+replay:
+  description: ERC-20 token transfers (from event logs)
+  package: demo_evm_events
+  template: demo_evm_events

src/demo_evm_transactions/replay.yaml

@@ -0,0 +1,5 @@
+spec_version: 3.0
+replay:
+  description: ERC-20 token transfers (from transactions)
+  package: demo_evm_transactions
+  template: demo_evm_transactions

src/demo_evm_uniswap/replay.yaml

@@ -0,0 +1,7 @@
+spec_version: 3.0
+replay:
+  description: Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB)
+  package: demo_evm_uniswap
+  template: demo_evm_uniswap
+  postgres_image: timescale/timescaledb-ha:pg15
+  postgres_data_path: /home/postgres/pgdata/data

src/demo_starknet_events/replay.yaml

@@ -0,0 +1,5 @@
+spec_version: 3.0
+replay:
+  description: ERC-20 token transfers (from events)
+  package: demo_starknet_events
+  template: demo_starknet_events

src/demo_substrate_events/handlers/batch.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable
+
 from dipdup.context import HandlerContext
 from dipdup.index import MatchedHandler
 
@@ -8,4 +9,4 @@ async def batch(
     handlers: Iterable[MatchedHandler],
 ) -> None:
     for handler in handlers:
-        await ctx.fire_matched_handler(handler)
+        await ctx.fire_matched_handler(handler)

src/demo_substrate_events/handlers/on_transfer.py

@@ -1,7 +1,3 @@
-from decimal import Decimal
-
-from tortoise.exceptions import DoesNotExist
-
 from demo_substrate_events import models as models
 from demo_substrate_events.types.assethub.substrate_events.assets_transferred import AssetsTransferredPayload
 from dipdup.context import HandlerContext
@@ -12,42 +8,20 @@ async def on_transfer(
     ctx: HandlerContext,
     event: SubstrateEvent[AssetsTransferredPayload],
 ) -> None:
-    amount = Decimal(event.payload.get('amount') or event.payload['value'])
+    amount = event.payload.get('amount') or event.payload['value']
     if not amount:
         return
 
-    address_from = event.payload['from']
-    await on_balance_update(
-        address=address_from,
-        balance_update=-amount,
-        level=event.data.level,
-    )
-    address_to = event.payload['to']
-    await on_balance_update(
-        address=address_to,
-        balance_update=amount,
-        level=event.data.level,
+    await ctx.execute_sql_query(
+        'update_balance',
+        event.payload['from'],
+        '-' + amount,
+        event.data.level,
     )
 
-
-async def on_balance_update(
-    address: str,
-    balance_update: Decimal,
-    level: int,
-) -> None:
-    try:
-        holder = await models.Holder.cached_get(pk=address)
-    except DoesNotExist:
-        holder = models.Holder(
-            address=address,
-            balance=0,
-            turnover=0,
-            tx_count=0,
-            last_seen=None,
-        )
-        holder.cache()
-    holder.balance += balance_update
-    holder.turnover += abs(balance_update)
-    holder.tx_count += 1
-    holder.last_seen = level
-    await holder.save()
+    await ctx.execute_sql_query(
+        'update_balance',
+        event.payload['to'],
+        amount,
+        event.data.level,
+    )

src/demo_substrate_events/hooks/on_index_rollback.py

@@ -13,4 +13,4 @@ async def on_index_rollback(
         index=index.name,
         from_level=from_level,
         to_level=to_level,
-    )
+    )

src/demo_substrate_events/hooks/on_reindex.py

@@ -4,4 +4,4 @@
 async def on_reindex(
     ctx: HookContext,
 ) -> None:
-    await ctx.execute_sql('on_reindex')
+    await ctx.execute_sql('on_reindex')

src/demo_substrate_events/hooks/on_restart.py

@@ -4,4 +4,4 @@
 async def on_restart(
     ctx: HookContext,
 ) -> None:
-    await ctx.execute_sql('on_restart')
+    await ctx.execute_sql('on_restart')

src/demo_substrate_events/hooks/on_synchronized.py

@@ -4,4 +4,4 @@
 async def on_synchronized(
     ctx: HookContext,
 ) -> None:
-    await ctx.execute_sql('on_synchronized')
+    await ctx.execute_sql('on_synchronized')

src/demo_substrate_events/models/__init__.py

@@ -1,13 +1,10 @@
 from dipdup import fields
-from dipdup.models import CachedModel
+from dipdup.models import Model
 
 
-class Holder(CachedModel):
+class Holder(Model):
     address = fields.TextField(primary_key=True)
     balance = fields.DecimalField(decimal_places=6, max_digits=40, default=0)
     turnover = fields.DecimalField(decimal_places=6, max_digits=40, default=0)
     tx_count = fields.BigIntField(default=0)
     last_seen = fields.BigIntField(null=True)
-
-    class Meta:
-        maxsize = 2 ** 18

src/demo_substrate_events/sql/update_balance.sql

@@ -0,0 +1,16 @@
+insert into
+    holder (
+        [address],
+        balance,
+        turnover,
+        tx_count,
+        last_seen
+    )
+values
+    (:address, :balance, abs(:balance), 1, :level) on conflict ([address]) do
+update
+set
+    balance = balance + :balance,
+    turnover = turnover + abs(:balance),
+    tx_count = tx_count + 1,
+    last_seen = :level;

src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v601.py

@@ -2,7 +2,8 @@
 
 from __future__ import annotations
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
+from pydantic import Field
 
 
 class V601(BaseModel):

src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v700.py

@@ -2,7 +2,8 @@
 
 from __future__ import annotations
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
+from pydantic import Field
 
 
 class V700(BaseModel):

src/demo_substrate_events_orm/.dockerignore

@@ -0,0 +1,22 @@
+# Ignore all
+*
+
+# Add metadata and build files
+!demo_substrate_events_orm
+!pyproject.toml
+!*.lock
+!README.md
+
+# Add Python code
+!**/*.py
+**/.*_cache
+**/__pycache__
+
+# Add configs and scripts (but not env!)
+!**/*.graphql
+!**/*.json
+!**/*.sql
+!**/*.yaml
+!**/*.yml
+!**/*.j2
+!**/.keep

src/demo_substrate_events_orm/.gitignore

@@ -0,0 +1,29 @@
+# Ignore all
+*
+!*/
+
+# Add metadata and build files
+!demo_substrate_events_orm
+!.gitignore
+!.dockerignore
+!py.typed
+!**/Dockerfile
+!**/Makefile
+!**/pyproject.toml
+!**/*.lock
+!**/README.md
+!**/.keep
+
+# Add Python code
+!**/*.py
+**/.*_cache
+**/__pycache__
+
+# Add configs and scripts (but not env!)
+!**/*.graphql
+!**/*.json
+!**/*.sql
+!**/*.yaml
+!**/*.yml
+!**/*.j2
+!**/*.env.default