WIP T 16269 broken erc721 balances tables (#1525)

* opposite amounts and incremental filters * remove wallet address from tests erc721 * fix send and receive, amount signs * T 16269 possible fix (#1530) * fix ERC721 balances * revert column refs in schemas Co-authored-by: 0xRob <[email protected]>
duneanalytics · Sep 8, 2022 · 3e08848 · 3e08848
1 parent a3dfe87
commit 3e08848
Show file tree

Hide file tree

Showing 12 changed files with 51 additions and 46 deletions.
diff --git a/models/balances/ethereum/balances_ethereum_schema.yml b/models/balances/ethereum/balances_ethereum_schema.yml
@@ -110,7 +110,7 @@ models:
       blockchain: ethereum
       sector: balances
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['balances', 'ethereum', 'erc721', 'day', 'soispoke', 'dot2dotseurat', 'hildobby']
     description: >
@@ -140,7 +140,7 @@ models:
       blockchain: ethereum
       sector: balances
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['balances', 'ethereum', 'erc721', 'hour', 'soispoke', 'dot2dotseurat', 'hildobby']
     description: >
@@ -160,7 +160,7 @@ models:
       blockchain: ethereum
       sector: balances
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['balances', 'ethereum', 'erc721', 'latest', 'soispoke', 'dot2dotseurat', 'hildobby']
     description: >
@@ -169,8 +169,8 @@ models:
     columns:
       - *blockchain
       - *last_updated
-      - *wallet_address
-      - *token_address
+      - *wallet_address_erc721
+      - *token_address_erc721
       - *tokenId
       - *collection
       - *category
@@ -191,10 +191,10 @@ models:
       - *day
       - &wallet_address_erc1155
         name: wallet_address
-        description: "Address of the wallet holding this ERC721 token"
+        description: "Address of the wallet holding this ERC1155 tokens"
       - &token_address_erc1155
         name: token_address
-        description: "Contract address for the ERC721 token"
+        description: "Contract address for the ERC1155 tokens"
       - &tokenId_erc1155
         name: tokenId
         description: "ID of this ERC721 token"
@@ -239,9 +239,9 @@ models:
     columns:
       - *blockchain
       - *last_updated
-      - *wallet_address
-      - *token_address
+      - *wallet_address_erc1155
+      - *token_address_erc1155
       - *tokenId_erc1155
       - *amount_erc1155
       - *collection
-      - *category
+      - *category
diff --git a/models/balances/ethereum/erc721/balances_ethereum_erc721_day.sql b/models/balances/ethereum/erc721/balances_ethereum_erc721_day.sql
@@ -19,7 +19,7 @@ with
     token_address,
     tokenId,
     day,
-    lead(day, 1, now()) OVER (PARTITION BY token_address, wallet_address ORDER BY day) AS next_day
+    lead(day, 1, now()) OVER (PARTITION BY token_address, tokenId ORDER BY day) AS next_day
     FROM {{ ref('transfers_ethereum_erc721_rolling_day') }})
 
 SELECT distinct

diff --git a/models/balances/ethereum/erc721/balances_ethereum_erc721_hour.sql b/models/balances/ethereum/erc721/balances_ethereum_erc721_hour.sql
@@ -19,7 +19,7 @@ with
     token_address,
     tokenId,
     hour,
-    lead(hour, 1, now()) OVER (PARTITION BY token_address, wallet_address ORDER BY hour) AS next_hour
+    lead(hour, 1, now()) OVER (PARTITION BY token_address, tokenId ORDER BY hour) AS next_hour
     FROM {{ ref('transfers_ethereum_erc721_rolling_hour') }})
 
 SELECT distinct

diff --git a/models/balances/ethereum/erc721/balances_ethereum_erc721_latest.sql b/models/balances/ethereum/erc721/balances_ethereum_erc721_latest.sql
@@ -8,6 +8,7 @@ SELECT distinct
     tokenId,
     nft_tokens.name as collection,
     updated_at
-FROM {{ ref('transfers_ethereum_erc721_rolling_hour') }}
+FROM {{ ref('transfers_ethereum_erc721_rolling_day') }}
 LEFT JOIN {{ ref('tokens_nft') }} nft_tokens ON nft_tokens.contract_address = token_address
 AND nft_tokens.blockchain = 'ethereum'
+WHERE recency_index = 1
diff --git a/models/transfers/ethereum/erc721/transfers_ethereum_erc721.sql b/models/transfers/ethereum/erc721/transfers_ethereum_erc721.sql
@@ -1,8 +1,8 @@
 {{ config(materialized='view', alias='erc721') }}
 
 with
-    sent_transfers as (
-        select 'send' || '-' ||  evt_tx_hash || '-' || evt_index || '-' || `to` as unique_tx_id,
+    received_transfers as (
+        select 'receive' || '-' ||  evt_tx_hash || '-' || evt_index || '-' || `to` as unique_tx_id,
             to as wallet_address,
             contract_address as token_address,
             evt_block_time,
@@ -13,8 +13,8 @@ with
     )
 
     ,
-    received_transfers as (
-        select 'receive' || '-' || evt_tx_hash || '-' || evt_index || '-' || `from` as unique_tx_id,
+    sent_transfers as (
+        select 'send' || '-' || evt_tx_hash || '-' || evt_index || '-' || `from` as unique_tx_id,
             from as wallet_address,
             contract_address as token_address,
             evt_block_time,
@@ -25,7 +25,7 @@ with
     )
 
 select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, tokenId, amount, unique_tx_id
-from sent_transfers
+from received_transfers
 union
 select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, tokenId, amount, unique_tx_id
-from received_transfers
+from sent_transfers
diff --git a/models/transfers/ethereum/erc721/transfers_ethereum_erc721_agg_day.sql b/models/transfers/ethereum/erc721/transfers_ethereum_erc721_agg_day.sql
@@ -1,5 +1,5 @@
 {{ config(
-        alias ='erc721_agg_day', 
+        alias ='erc721_agg_day',
         materialized ='incremental',
         file_format ='delta',
         incremental_strategy='merge',
@@ -13,12 +13,11 @@ select
     wallet_address,
     token_address,
     tokenId,
-    unique_tx_id || '-' || wallet_address || '-' || token_address || tokenId as unique_transfer_id
+    date_trunc('day', evt_block_time) || '-' || token_address || '-' || tokenId as unique_transfer_id
 from {{ ref('transfers_ethereum_erc721') }}
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-where date_trunc('day', evt_block_time) > now() - interval 2 days
+where evt_block_time >= date_trunc('hour', now() - interval '1 week')
 {% endif %}
-group by
-    date_trunc('day', evt_block_time), wallet_address, token_address, tokenId,unique_tx_id
-having sum(amount) = 1
+group by 1,2,3,4,5
+having sum(amount) = 1
diff --git a/models/transfers/ethereum/erc721/transfers_ethereum_erc721_agg_hour.sql b/models/transfers/ethereum/erc721/transfers_ethereum_erc721_agg_hour.sql
@@ -1,5 +1,5 @@
 {{ config(
-        alias ='erc721_agg_hour', 
+        alias ='erc721_agg_hour',
         materialized ='incremental',
         file_format ='delta',
         incremental_strategy='merge',
@@ -13,12 +13,11 @@ select
     wallet_address,
     token_address,
     tokenId,
-    unique_tx_id || '-' || wallet_address || '-' || token_address || tokenId as unique_transfer_id
+    date_trunc('hour', evt_block_time) || '-' || token_address || '-'  || tokenId as unique_transfer_id
 from {{ ref('transfers_ethereum_erc721') }}
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-where date_trunc('hour', evt_block_time) > now() - interval 2 days
+where evt_block_time >= date_trunc('day', now() - interval '1 week')
 {% endif %}
-group by
-    date_trunc('hour', evt_block_time), wallet_address, token_address, tokenId,unique_tx_id
-having sum(amount) = 1
+group by 1,2,3,4,5
+having sum(amount) = 1
diff --git a/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_day.sql b/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_day.sql
@@ -9,5 +9,5 @@
             token_address,
             tokenId,
             current_timestamp() as updated_at,
-            row_number() over (partition by token_address, tokenId, wallet_address order by day desc) as recency_index
-        from {{ ref('transfers_ethereum_erc721_agg_day') }}
+            row_number() over (partition by token_address, tokenId order by day desc) as recency_index
+        from {{ ref('transfers_ethereum_erc721_agg_day') }}
diff --git a/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_hour.sql b/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_hour.sql
@@ -9,5 +9,5 @@
             token_address,
             tokenId,
             current_timestamp() as updated_at,
-            row_number() over (partition by token_address, tokenId, wallet_address order by hour desc) as recency_index
-        from {{ ref('transfers_ethereum_erc721_agg_hour') }}
+            row_number() over (partition by token_address, tokenId order by hour desc) as recency_index
+        from {{ ref('transfers_ethereum_erc721_agg_hour') }}
diff --git a/models/transfers/ethereum/transfers_ethereum_schema.yml b/models/transfers/ethereum/transfers_ethereum_schema.yml
@@ -130,7 +130,7 @@ models:
       blockchain: ethereum
       sector: transfers
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['transfers', 'ethereum', 'erc721', 'soispoke', 'dot2dotseurat', 'hildobby']
     description: "ERC721 Token Transfers on Ethereum. This table is updated every 30 minutes."
@@ -142,7 +142,7 @@ models:
         name: wallet_address
         description: "Wallet address of sender or receiver."
       - &token_address_erc721
-        name: token_address_erc721
+        name: token_address
       - name: symbol
         description: "ERC721 token symbol"
       - *evt_block_time
@@ -157,7 +157,7 @@ models:
       blockchain: ethereum
       sector: transfers
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['transfers', 'ethereum', 'erc721', 'agg_hour', 'soispoke', 'dot2dotseurat', 'hildobby']
     columns:
@@ -166,13 +166,17 @@ models:
       - *wallet_address_erc721
       - *token_address_erc721
       - *tokenId
+      - name: unique_transfer_id
+        description: "Unique ID, used in order to apply the merge into incremental strategy"
+        tests:
+          - unique
 
   - name: transfers_ethereum_erc721_rolling_hour
     meta:
       blockchain: ethereum
       sector: transfers
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['transfers', 'ethereum', 'erc721', 'rolling_hour', 'soispoke', 'dot2dotseurat', 'hildobby']
     columns:
@@ -191,7 +195,7 @@ models:
       blockchain: ethereum
       sector: transfers
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['transfers', 'ethereum', 'erc721', 'agg_hour', 'soispoke', 'dot2dotseurat', 'hildobby']
     columns:
@@ -200,13 +204,17 @@ models:
       - *wallet_address_erc721
       - *token_address_erc721
       - *tokenId
+      - name: unique_transfer_id
+        description: "Unique ID, used in order to apply the merge into incremental strategy"
+        tests:
+          - unique
 
   - name: transfers_ethereum_erc721_rolling_day
     meta:
       blockchain: ethereum
       sector: transfers
       project: erc721
-      contibutors: soispoke, dot2dotseurat, hildobby
+      contibutors: soispoke, dot2dotseurat, hildobby, 0xRob
     config:
       tags: ['transfers', 'ethereum', 'erc721', 'rolling_day', 'soispoke', 'dot2dotseurat', 'hildobby']
     columns:
@@ -317,4 +325,4 @@ models:
       - name: updated_at
         description: "UTC timestamp when table was last updated"
       - name: recency_index
-        description: "Index of most recent balance ascending. recency_index=1 is the wallet/contract pair's most recent balance"
+        description: "Index of most recent balance ascending. recency_index=1 is the wallet/contract pair's most recent balance"
diff --git a/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql b/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql
@@ -2,11 +2,10 @@
 
 select blockchain,
     day,
-    wallet_address,
     token_address,
     tokenId,
     count(*)
 from {{ ref('balances_ethereum_erc721_day') }}
 where day >= now() - interval '2 days'
-group by blockchain, day, wallet_address, token_address, tokenId
+group by blockchain, day, token_address, tokenId
 having count(*) > 1
diff --git a/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql b/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql
@@ -2,11 +2,10 @@
 
 select blockchain,
     hour,
-    wallet_address,
     token_address,
     tokenId,
     count(*)
 from {{ ref('balances_ethereum_erc721_hour') }}
 where hour >= now() - interval '2 hours'
-group by blockchain, hour, wallet_address, token_address, tokenId
+group by blockchain, hour, token_address, tokenId
 having count(*) > 1