From afc48f18f0e656c37db3d798e019a8d07d13341f Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 28 Nov 2023 18:21:51 -0500 Subject: [PATCH] add preview script --- README.md | 28 +++++++++++++++----------- queries/query_3237721.sql | 41 --------------------------------------- queries/query_3237723.sql | 12 ------------ queries/query_3237726.sql | 39 ------------------------------------- queries/query_3237745.sql | 33 ------------------------------- requirements.txt | 3 ++- scripts/preview_query.py | 30 ++++++++++++++++++++++++++++ 7 files changed, 49 insertions(+), 137 deletions(-) create mode 100644 scripts/preview_query.py diff --git a/README.md b/README.md index 4bc6685..acd5f02 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,28 @@ A template for creating repos to manage your Dune queries using the CRUD API. Th 2. Copy and paste that list into the `queries.yml` file (or any list of query ids, doesn't have to be linked to a dashboard). -3. Install the python requirements and run the `pull_from_dune.py` script. You can input the following lines into a terminal/CLI: - ``` - pip install -r requirements.txt - python scripts/pull_from_dune.py - ``` - This will bring in your query ids into `/query_{id}.sql` files within the `/queries` folder. You can run `pull_from_dune.py` anytime you need to update your queries in this repo with edits from the Dune app. - -4. Make any changes you need to directly in the repo, and any time you push a commit `push_to_dune.py` will run and save your changes into Dune directly. You can also run the script manually: - ``` - python scripts/push_to_dune.py - ``` +3. Then, run `pull_from_dune.py` to bring in all queries into `/query_{id}.sql` files within the `/queries` folder. Directions are below. + +4. Make any changes you need to directly in the repo. Any time you push a commit `push_to_dune.py` will run and save your changes into Dune directly. 💡: We use the [Dune CRUD API](https://dune.com/docs/api/api-reference/edit-queries/) to manage queries - this does not change how your queries behave in app. --- +### Scripts + +You'll need python installed to run the script commands. Install the required packages first: + +``` +pip install -r requirements.txt +``` + +| Script | Action | Command | +|---|---|---| +| `pull_from_dune.py` | updates/adds queries to your repo based on ids in queries.yml | `python scripts/pull_from_dune.py` | +| `push_to_dune.py` | updates queries to Dune based on files in your `queries` folder | `python scripts/push_to_dune.py` | +| `preview_query.py` | gives you the first 20 rows of results by running a query from your `queries` folder. Specify the id. | `python scripts/preview_query.py 3237723` | + ### For Contributors I've set up four types of issues right now: diff --git a/queries/query_3237721.sql b/queries/query_3237721.sql index fceebb5..7a075d5 100644 --- a/queries/query_3237721.sql +++ b/queries/query_3237721.sql @@ -3,47 +3,6 @@ -- query link: https://dune.com/queries/3237721 -WITH - seven_day_volume AS ( - SELECT - project AS "Project", - SUM(CAST(amount_usd AS DOUBLE)) AS usd_volume - FROM - dex."trades" AS t - WHERE - block_time > NOW() - INTERVAL '7' day - GROUP BY - 1 - ), - one_day_volume AS ( - SELECT - project AS "Project", - SUM(CAST(amount_usd AS DOUBLE)) AS usd_volume - FROM - dex."trades" AS t - WHERE - block_time > NOW() - INTERVAL '1' day - GROUP BY - 1 - ) -SELECT - ROW_NUMBER() OVER ( - ORDER BY - SUM(seven.usd_volume) DESC NULLS FIRST - ) AS "Rank", - seven."Project", - SUM(seven.usd_volume) AS "7 Days Volume", - SUM(one.usd_volume) AS "24 Hours Volume" -FROM - seven_day_volume AS seven - LEFT JOIN one_day_volume AS one ON seven."Project" = one."Project" -WHERE - NOT seven.usd_volume IS NULL -GROUP BY - 2 -ORDER BY - 3 DESC NULLS FIRST --- DEX by volume 🏦 WITH seven_day_volume AS ( SELECT diff --git a/queries/query_3237723.sql b/queries/query_3237723.sql index 6f0b19c..f80fa25 100644 --- a/queries/query_3237723.sql +++ b/queries/query_3237723.sql @@ -3,18 +3,6 @@ -- query link: https://dune.com/queries/3237723 -SELECT - blockchain, - DATE_TRUNC('week', block_time), - SUM(CAST(amount_usd AS DOUBLE)) AS usd_volume -FROM - dex."trades" AS t /* AND block_time < date_trunc('week', Now()) -- Add this line to see stats from current week */ -WHERE - block_time > NOW() - INTERVAL '365' day -GROUP BY - 1, - 2 --- Weekly DEX volume by chain SELECT blockchain, DATE_TRUNC('week', block_time), diff --git a/queries/query_3237726.sql b/queries/query_3237726.sql index c0e5b4d..9377470 100644 --- a/queries/query_3237726.sql +++ b/queries/query_3237726.sql @@ -3,45 +3,6 @@ -- query link: https://dune.com/queries/3237726 -WITH - seven_day_volume AS ( - SELECT - project AS "Project", - SUM(CAST(amount_usd AS DOUBLE)) AS usd_volume - FROM - dex_aggregator.trades AS t - WHERE - block_time > CURRENT_TIMESTAMP - INTERVAL '7' day - GROUP BY - 1 - ), - one_day_volume AS ( - SELECT - project AS "Project", - SUM(CAST(amount_usd AS DOUBLE)) AS usd_volume - FROM - dex_aggregator.trades AS t - WHERE - block_time > CURRENT_TIMESTAMP - INTERVAL '1' day - GROUP BY - 1 - ) -SELECT - ROW_NUMBER() OVER ( - ORDER BY - SUM(seven.usd_volume) DESC NULLS FIRST - ) AS "Rank", - seven."Project", - SUM(seven.usd_volume) AS "7 Days Volume", - SUM(one.usd_volume) AS "24 Hours Volume" -FROM - seven_day_volume AS seven - LEFT JOIN one_day_volume AS one ON seven."Project" = one."Project" -GROUP BY - 2 -ORDER BY - 3 DESC NULLS FIRST --- Aggregator by volume 📢 WITH seven_day_volume AS ( SELECT diff --git a/queries/query_3237745.sql b/queries/query_3237745.sql index 4cd793a..1693129 100644 --- a/queries/query_3237745.sql +++ b/queries/query_3237745.sql @@ -3,39 +3,6 @@ -- query link: https://dune.com/queries/3237745 -with - all_traders as ( - SELECT - date_trunc('week',block_time) as week - , tx_from - , sum(amount_usd) as volume - FROM (SELECT - block_time - , tx_hash - , tx_from - , max(amount_usd) as amount_usd - FROM dex.trades - group by 1,2,3 - ) - WHERE amount_usd is not null - group by 1,2 - ) - -SELECT -week -, case - when volume < 1e2 then '< $100' - when volume >= 1e2 and volume < 1e3 then '< $1,000' - when volume >= 1e3 and volume < 1e4 then '< $10,000' - when volume >= 1e4 and volume < 1e5 then '< $100,000' - when volume >= 1e5 and volume < 1e6 then '< $1,000,000' - when volume >= 1e6 then '$1m+' -end as trader_bucket -, count(*) -FROM all_traders -WHERE week >= NOW() - INTERVAL '365' day -group by 1,2 --- EVM DEX Traders by Bucket with all_traders as ( SELECT diff --git a/requirements.txt b/requirements.txt index 16bde8f..ad76048 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ dune-client==1.3.0 pyyaml -python-dotenv \ No newline at end of file +python-dotenv +pandas \ No newline at end of file diff --git a/scripts/preview_query.py b/scripts/preview_query.py new file mode 100644 index 0000000..4137632 --- /dev/null +++ b/scripts/preview_query.py @@ -0,0 +1,30 @@ +import os +from dune_client.client import DuneClient +from dotenv import load_dotenv +import sys +import pandas as pd + +dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env') +load_dotenv(dotenv_path) + +dune = DuneClient.from_env() + +#get id passed in python script invoke +id = sys.argv[1] + +query_file = os.path.join(os.path.dirname(__file__), '..', 'queries', f'query_{id}.sql') + +print('getting 20 line preview for query {}...'.format(id)) + +with open(query_file, 'r', encoding='utf-8') as file: + query_text = file.read() + +results = dune.run_sql(query_text + '\n limit 20') +# print(results.result.rows) +results = pd.DataFrame(data=results.result.rows) +print('\n') +print(results) +print('\n') +print(results.describe()) +print('\n') +print(results.info()) \ No newline at end of file