Merge branch 'blockchain-etl:master' into master

Gauntlet-xyz · Apr 22, 2024 · ad9ae3a · ad9ae3a
2 parents eb1dcd8 + bcfc20b
commit ad9ae3a
Show file tree

Hide file tree

Showing 201 changed files with 9,711 additions and 9 deletions.
diff --git a/dags/ethereumetl_airflow/build_partition_dag.py b/dags/ethereumetl_airflow/build_partition_dag.py
@@ -191,6 +191,15 @@ def enrich_task(ds, **kwargs):
     where true
     and date(block_timestamp) > '{ds}'
     union all
+    -- blob transaction fees credits
+    select
+        from_address as address,
+        -(cast(receipt_blob_gas_used as numeric) * cast(receipt_blob_gas_price as numeric)) as value
+    from `{public_project_id}.{public_dataset_name}.transactions`
+    where true
+    and receipt_blob_gas_used > 0
+    and date(block_timestamp) > '{ds}'
+    union all
     -- withdrawals
     select
         withdrawal.address as address,

diff --git a/dags/resources/stages/enrich/schemas/blocks.json b/dags/resources/stages/enrich/schemas/blocks.json
@@ -130,5 +130,15 @@
         ],
         "mode": "REPEATED",
         "description": "Validator withdrawals"
+    },
+    {
+        "name": "blob_gas_used",
+        "type": "INT64",
+        "description": "The total amount of blob gas consumed by transactions in the block"
+    },
+    {
+        "name": "excess_blob_gas",
+        "type": "INT64",
+        "description": "A running total of blob gas consumed in excess of the target, prior to the block. This is used to set blob gas pricing"
     }
 ]
diff --git a/dags/resources/stages/enrich/schemas/transactions.json b/dags/resources/stages/enrich/schemas/transactions.json
@@ -110,5 +110,26 @@
         "name": "receipt_effective_gas_price",
         "type": "INT64",
         "description": "The actual value per gas deducted from the senders account. Replacement of gas_price after EIP-1559"
+    },
+    {
+        "name": "max_fee_per_blob_gas",
+        "type": "INT64",
+        "description": "The maximum fee a user is willing to pay per blob gas"
+    },
+    {
+        "name": "blob_versioned_hashes",
+        "type": "STRING",
+        "mode": "REPEATED",
+        "description": "A list of hashed outputs from kzg_to_versioned_hash"
+    },
+    {
+        "name": "receipt_blob_gas_price",
+        "type": "INT64",
+        "description": "Blob gas price"
+    },
+    {
+        "name": "receipt_blob_gas_used",
+        "type": "INT64",
+        "description": "Blob gas used"
     }
 ]
diff --git a/dags/resources/stages/enrich/sqls/balances.sql b/dags/resources/stages/enrich/sqls/balances.sql
@@ -35,6 +35,15 @@ with double_entry_book as (
     where true
     and date(block_timestamp) <= '{{ds}}'
     union all
+    -- blob transaction fees credits
+    select
+        from_address as address,
+        -(cast(receipt_blob_gas_used as numeric) * cast(receipt_blob_gas_price as numeric)) as value
+    from `{{params.destination_dataset_project_id}}.{{params.dataset_name}}.transactions`
+    where true
+    and receipt_blob_gas_used > 0
+    and date(block_timestamp) <= '{{ds}}'
+    union all
     -- withdrawals
     select
         withdrawal.address as address,

diff --git a/dags/resources/stages/enrich/sqls/blocks.sql b/dags/resources/stages/enrich/sqls/blocks.sql
@@ -20,6 +20,8 @@ SELECT
     blocks.base_fee_per_gas,
     blocks.withdrawals_root,
     blocks.withdrawals,
+    blocks.blob_gas_used,
+    blocks.excess_blob_gas,
 FROM {{params.dataset_name_raw}}.blocks AS blocks
 where true
     {% if not params.load_all_partitions %}

diff --git a/dags/resources/stages/enrich/sqls/merge/merge_blocks.sql b/dags/resources/stages/enrich/sqls/merge/merge_blocks.sql
@@ -23,7 +23,9 @@ insert (
     transaction_count,
     base_fee_per_gas,
     withdrawals_root,
-    withdrawals
+    withdrawals,
+    blob_gas_used,
+    excess_blob_gas
 ) values (
     timestamp,
     number,
@@ -45,7 +47,9 @@ insert (
     transaction_count,
     base_fee_per_gas,
     withdrawals_root,
-    withdrawals
+    withdrawals,
+    blob_gas_used,
+    excess_blob_gas
 )
 when not matched by source and date(timestamp) = '{{ds}}' then
 delete
diff --git a/dags/resources/stages/enrich/sqls/merge/merge_transactions.sql b/dags/resources/stages/enrich/sqls/merge/merge_transactions.sql
@@ -23,7 +23,11 @@ insert (
     max_fee_per_gas,
     max_priority_fee_per_gas,
     transaction_type,
-    receipt_effective_gas_price
+    receipt_effective_gas_price,
+    max_fee_per_blob_gas,
+    blob_versioned_hashes,
+    receipt_blob_gas_price,
+    receipt_blob_gas_used
 ) values (
     `hash`,
     nonce,
@@ -45,7 +49,11 @@ insert (
     max_fee_per_gas,
     max_priority_fee_per_gas,
     transaction_type,
-    receipt_effective_gas_price
+    receipt_effective_gas_price,
+    max_fee_per_blob_gas,
+    blob_versioned_hashes,
+    receipt_blob_gas_price,
+    receipt_blob_gas_used
 )
 when not matched by source and date(block_timestamp) = '{{ds}}' then
 delete
diff --git a/dags/resources/stages/enrich/sqls/transactions.sql b/dags/resources/stages/enrich/sqls/transactions.sql
@@ -19,7 +19,11 @@ SELECT
     transactions.max_fee_per_gas,
     transactions.max_priority_fee_per_gas,
     transactions.transaction_type,
-    receipts.effective_gas_price as receipt_effective_gas_price
+    receipts.effective_gas_price as receipt_effective_gas_price,
+    transactions.max_fee_per_blob_gas as max_fee_per_blob_gas,
+    transactions.blob_versioned_hashes as blob_versioned_hashes,
+    receipts.blob_gas_price as receipt_blob_gas_price,
+    receipts.blob_gas_used as receipt_blob_gas_used,
 FROM {{params.dataset_name_raw}}.blocks AS blocks
     JOIN {{params.dataset_name_raw}}.transactions AS transactions ON blocks.number = transactions.block_number
     JOIN {{params.dataset_name_raw}}.receipts AS receipts ON transactions.hash = receipts.transaction_hash

diff --git a/...stages/parse/table_definitions/chainlink/view_AccessControlledOffchainAggregator_info.sql b/...stages/parse/table_definitions/chainlink/view_AccessControlledOffchainAggregator_info.sql
@@ -30,7 +30,7 @@ STRUCT('0x478238a1c8b862498c74d0647329aef9ea6819ed' as contract_address, 8 as de
 STRUCT('0xdeaa4288c85e7e0be40bce49e76d4e321d20fc36' as contract_address, 18 as decimals, '0xf05d9b6c08757eacb1fbec18e36a1b7566a13deb' as proxy_address, 'Calculated XSUSHI / ETH' as description, 4 as version),
 STRUCT('0x5df960959de45a2ba9dc11e6fd6f77107f43256c' as contract_address, 8 as decimals, '0x4e155ed98afe9034b7a5962f6c84c86d869daa9d' as proxy_address, 'RPL / USD' as description, 4 as version),
 STRUCT('0x3cf055335b521863a62fb4374972560e3e55a193' as contract_address, 8 as decimals, '0xcc70f09a6cc17553b2e31954cd36e4a2d89501f7' as proxy_address, 'SUSHI / USD' as description, 4 as version),
-STRUCT('0x2abfc56aaa39be7a946ec39aac5d452e30614df1' as contract_address, 8 as decimals, '0x483d36f6a1d063d580c7a24f9a42b346f3a69fbb' as proxy_address, 'RAI / USD' as description, 4 as version),
+-- STRUCT('0x2abfc56aaa39be7a946ec39aac5d452e30614df1' as contract_address, 8 as decimals, '0x483d36f6a1d063d580c7a24f9a42b346f3a69fbb' as proxy_address, 'RAI / USD' as description, 4 as version),
 STRUCT('0xd74ff3f1b565597e59d44320f53a5c5c8ba85f7b' as contract_address, 18 as decimals, '0xf017fcb346a1885194689ba23eff2fe6fa5c483b' as proxy_address, 'CBETH / ETH' as description, 4 as version),
 -- STRUCT('0xbbc9ab3f66d0b934089fdcceda3615a6bb879633' as contract_address, 8 as decimals, '0x7eed379bf00005cfed29fed4009669de9bcc21ce' as proxy_address, 'ANKR / USD' as description, 4 as version),
 STRUCT('0x9cb248e68fb81d0cfe7d6b3265fe6bf123a71fe0' as contract_address, 18 as decimals, '0x536218f9e9eb48863970252233c8f271f554c2d0' as proxy_address, 'RETH / ETH' as description, 4 as version),

diff --git a/dags/resources/stages/parse/table_definitions/cseth/CSETHToken_event_Approval.json b/dags/resources/stages/parse/table_definitions/cseth/CSETHToken_event_Approval.json
@@ -0,0 +1,54 @@
+{
+    "parser": {
+        "abi": {
+            "anonymous": false,
+            "inputs": [
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "owner",
+                    "type": "address"
+                },
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "spender",
+                    "type": "address"
+                },
+                {
+                    "indexed": false,
+                    "internalType": "uint256",
+                    "name": "value",
+                    "type": "uint256"
+                }
+            ],
+            "name": "Approval",
+            "type": "event"
+        },
+        "contract_address": "0x5d74468b69073f809d4fae90afec439e69bf6263",
+        "field_mapping": {},
+        "type": "log"
+    },
+    "table": {
+        "dataset_name": "cseth",
+        "schema": [
+            {
+                "description": "",
+                "name": "owner",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "spender",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "value",
+                "type": "STRING"
+            }
+        ],
+        "table_description": "",
+        "table_name": "CSETHToken_event_Approval"
+    }
+}
diff --git a/dags/resources/stages/parse/table_definitions/cseth/CSETHToken_event_Transfer.json b/dags/resources/stages/parse/table_definitions/cseth/CSETHToken_event_Transfer.json
@@ -0,0 +1,54 @@
+{
+    "parser": {
+        "abi": {
+            "anonymous": false,
+            "inputs": [
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "from",
+                    "type": "address"
+                },
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "to",
+                    "type": "address"
+                },
+                {
+                    "indexed": false,
+                    "internalType": "uint256",
+                    "name": "value",
+                    "type": "uint256"
+                }
+            ],
+            "name": "Transfer",
+            "type": "event"
+        },
+        "contract_address": "0x5d74468b69073f809d4fae90afec439e69bf6263",
+        "field_mapping": {},
+        "type": "log"
+    },
+    "table": {
+        "dataset_name": "cseth",
+        "schema": [
+            {
+                "description": "",
+                "name": "from",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "to",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "value",
+                "type": "STRING"
+            }
+        ],
+        "table_description": "",
+        "table_name": "CSETHToken_event_Transfer"
+    }
+}
diff --git a/...rces/stages/parse/table_definitions/curve/CurveStableswapFactoryNG_call_deploy_gauge.json b/...rces/stages/parse/table_definitions/curve/CurveStableswapFactoryNG_call_deploy_gauge.json
@@ -0,0 +1,36 @@
+{
+    "parser": {
+        "abi": {
+            "inputs": [
+                {
+                    "name": "_pool",
+                    "type": "address"
+                }
+            ],
+            "name": "deploy_gauge",
+            "outputs": [
+                {
+                    "name": "",
+                    "type": "address"
+                }
+            ],
+            "stateMutability": "nonpayable",
+            "type": "function"
+        },
+        "contract_address": "0x6a8cbed756804b16e05e741edabd5cb544ae21bf",
+        "field_mapping": {},
+        "type": "trace"
+    },
+    "table": {
+        "dataset_name": "curve",
+        "schema": [
+            {
+                "description": "",
+                "name": "_pool",
+                "type": "STRING"
+            }
+        ],
+        "table_description": "",
+        "table_name": "CurveStableswapFactoryNG_call_deploy_gauge"
+    }
+}