Complete candidate route test suite

Makefile

@@ -180,9 +180,9 @@ test-prices-mainnet:
 
 ### E2E Test
 
-# Run E2E tests in verbose mode (-s)
+# Run E2E tests in verbose mode (-s) -n 4 concurrent workers
 e2e-run-dev:
-	pytest -s
+	pytest -s -n 4
 
 #### E2E Python Setup
 

scripts/quote.sh

@@ -1,15 +1,17 @@
 #!/bin/bash
 
-url=$1
+sqs_address=$1
+node_address=$2
 
 # This script compares single hop quotes by running them against SQS and chain directly.
+# ./scripts/quote.sh https://sqs.osmosis.zone https://rpc.osmosis.zone:443
 
-chain_amount_out=$(osmosisd q poolmanager estimate-swap-exact-amount-in 1436 100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc --swap-route-pool-ids 1436 --swap-route-denoms ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4 --node $url:26657)
+chain_amount_out=$(osmosisd q poolmanager estimate-swap-exact-amount-in 1436 100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc --swap-route-pool-ids 1436 --swap-route-denoms ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4 --node $node_address)
 
-sqs_custom_res=$(curl "$url:9092/router/custom-direct-quote?tokenIn=100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc&tokenOutDenom=ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4&poolID=1436")
+sqs_custom_res=$(curl "$sqs_address/router/custom-direct-quote?tokenIn=100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc&tokenOutDenom=ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4&poolID=1436")
 sqs_custom_amount_out=$(echo $sqs_custom_res | jq .amount_out)
 
-sqs_optimal_res=$(curl "$url:9092/router/quote?tokenIn=100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc&tokenOutDenom=ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4")
+sqs_optimal_res=$(curl "$sqs_address/router/quote?tokenIn=100000000factory/osmo1z0qrq605sjgcqpylfl4aa6s90x738j7m58wyatt0tdzflg2ha26q67k743/wbtc&tokenOutDenom=ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4")
 sqs_optimal_amount_out=$(echo $sqs_optimal_res | jq .amount_out)
 
 echo "chain_amount_out: $chain_amount_out"

tests/conftest.py

@@ -1,11 +1,19 @@
 import pytest
 from sqs_service import *
 
+SERVICE_SQS_STAGE = SQSService(SQS_STAGE)
+SERVICE_SQS_PROD = SQSService(SQS_PROD)
+
 # Define the environment URLs
 # All tests will be run against these URLs
 @pytest.fixture(params=[
     SQS_STAGE,
 ])
 
 def environment_url(request):
-    return request.param
+    return request.param
+
+SERVICE_MAP = {
+    SQS_STAGE: SERVICE_SQS_STAGE,
+    SQS_PROD: SERVICE_SQS_PROD
+}

tests/constants.py

@@ -2,4 +2,4 @@
 
 # Misc constants
 UOSMO = "uosmo"
-USDC = setup.display_to_data_map["usdc"]["denom"]
+USDC = 'ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4'

tests/requirements.txt

@@ -1,11 +1,13 @@
 certifi==2024.2.2
 charset-normalizer==3.3.2
 exceptiongroup==1.2.1
+execnet==2.1.1
 idna==3.7
 iniconfig==2.0.0
 packaging==24.0
 pluggy==1.5.0
 pytest==8.2.0
+pytest-xdist==3.6.1
 requests==2.31.0
 tomli==2.0.1
 urllib3==2.2.1

tests/setup.py

@@ -1,5 +1,7 @@
 import copy
+import itertools
 from data_service import all_tokens_data, all_pools_data
+from conftest import SERVICE_SQS_PROD
 from enum import Enum, IntEnum
 
 # Numia pool type constants using an Enum
@@ -29,6 +31,16 @@ class E2EPoolType(IntEnum):
     NumiaPoolType.CONCENTRATED.value: E2EPoolType.CONCENTRATED
 }
 
+# This is the number of tokens we allow being skipped due to being unlisted
+# or not having liquidity. This number is hand-picked arbitrarily. We have around ~350 tokens
+# at the time of writing this test and we leave a small buffer.
+ALLOWED_NUM_TOKENS_USDC_PAIR_SKIPPED = 50
+# This is the minimum number of misc token pairs
+# that we expect to construct in setup.
+# Since our test setup is dynamic, it is important to
+# validate we do not get false positve passes due to small
+# number of pairs constructed.
+MIN_NUM_MISC_TOKEN_PAIRS = 10
 
 def get_e2e_pool_type_from_numia_pool(pool):
     """Gets an e2e pool type from a Numia pool."""
@@ -71,22 +83,36 @@ def get_denoms_from_pool_tokens(pool_tokens):
     return denoms
 
 
-def map_pool_type_to_pool_data(pool_data):
+def create_pool_data_maps(pool_data):
     """
-    Returns a dictionary mapping each pool type to associated ID, liquidity, and tokens.
+    Creates and returns pool data dictionaries for testing.
+
+    Returns two dictionaries:
+
+    1. A dictionary mapping each pool type to associated ID, liquidity, and tokens.
 
     Example output:
     {
         "e2e_pool_type_1": [[pool_id, liquidity, [denoms]]],
         "e2e_pool_type_2": [[pool_id, liquidity, [denoms]], ...],
         ...
     }
+
+    2. A dictionary mapping each denom to the pool with highest liquidity
+
+    Example output:
+    {
+        "uosmo": { "pool_liquidity": 1000000, "pool_id": 1 },
+        ...
+    }
     """
     if not pool_data:
         return {}
 
     pool_type_to_data = {}
 
+    denom_top_liquidity_pool_map = {}
+
     for pool in pool_data:
         # Convert Numia pool type to e2e pool type
         e2e_pool_type = get_e2e_pool_type_from_numia_pool(pool)
@@ -106,7 +132,19 @@ def map_pool_type_to_pool_data(pool_data):
         # Append the pool data to the list for this pool type
         pool_type_to_data[e2e_pool_type].append([pool_id, liquidity, denoms])
 
-    return pool_type_to_data
+        for denom in denoms:
+            denom_pool_data = denom_top_liquidity_pool_map.get(denom)
+
+            if denom_pool_data:
+                # Update the pool ID if the liquidity is higher
+                if liquidity > denom_pool_data['pool_liquidity']:
+                    denom_pool_data['pool_liquidity'] = liquidity
+                    denom_pool_data['pool_id'] = pool_id
+            else:
+                # Create first mapping for this denom
+                denom_top_liquidity_pool_map[denom] = {'pool_liquidity': liquidity, 'pool_id': pool_id}
+
+    return pool_type_to_data, denom_top_liquidity_pool_map
 
 
 def create_field_to_data_map(tokens_data, key_field):
@@ -221,6 +259,47 @@ def choose_pcl_pool_tokens_by_liq_asc(num_pairs=1, min_liq=0, max_liq=float('inf
     Returns [pool ID, [tokens]]"""
     return choose_pool_type_tokens_by_liq_asc(E2EPoolType.COSMWASM_ASTROPORT, num_pairs, min_liq, max_liq, asc)
 
+def choose_valid_listed_tokens():
+    """
+    Returns all listed tokens from the asset list that have at least one pool with liquidity.
+
+    Queries production SQS for the tokens asset list metadata.
+
+    Queries Numia for the pool liquidity data.
+    """
+
+    # We rely on SQS itself for getting the tokens metadata for configuring tests.
+    # While it is not the best practice, we make an exception since this is the most reliable way to get
+    # The asset list data. In the future, we can implement custom test parsing to replace relying on SQS
+    # in test setup.
+    tokens_metadata = SERVICE_SQS_PROD.get_tokens_metadata()
+
+    if len(tokens_metadata) == 0:
+        raise ValueError("Error: no tokens metadata retrieved from SQS during tokens setup")
+
+    valid_listed_tokens = []
+
+    for denom, metadata in tokens_metadata.items():
+        # Skip unlisted tokens as they should be unsupported
+        # in SQS.
+        if metadata['is_unlisted']:
+            [print(f"Denom {denom} is unlisted")]
+            continue
+
+        # Skip tokens with no pools with liquidity as we cannot find routes in-between them.
+        top_liquidity_pool = denom_top_liquidity_pool_map.get(denom)
+        if top_liquidity_pool is None:
+            print(f"Denom {denom} has no pool with liquidity")
+            continue
+
+        valid_listed_tokens.append(denom)
+
+    skipped_token_count = len(tokens_metadata) - len(valid_listed_tokens)
+    if skipped_token_count > ALLOWED_NUM_TOKENS_USDC_PAIR_SKIPPED:
+        raise ValueError(f"Too many tokens {skipped_token_count} from the metadata were untested, allowed {ALLOWED_NUM_TOKENS_USDC_PAIR_SKIPPED} tokens to be skipped")
+
+    return valid_listed_tokens
+
 
 def chain_denom_to_display(chain_denom):
     """Function to map chain denom to display."""
@@ -237,8 +316,64 @@ def chain_denoms_to_display(chain_denoms):
 # Create a map of chain denom to token data
 chain_denom_to_data_map = create_chain_denom_to_data_map(all_tokens_data)
 
-# Create a map of pool type to pool data
-pool_type_to_denoms = map_pool_type_to_pool_data(all_pools_data)
+# Create two maps:
+# 1. A map of pool type to pool data
+# 2. A map of denom to top liquidity pool
+pool_type_to_denoms, denom_top_liquidity_pool_map = create_pool_data_maps(all_pools_data)
 
 # Create a map of pool ID to pool data
 pool_by_id_map = {pool.get('pool_id'): pool for pool in all_pools_data}
+
+# Listed tokens that have at least one pool with liquidity
+valid_listed_tokens = choose_valid_listed_tokens() 
+
+# One Transmuter token pair [[pool_id, ['denom0', 'denom1']]]
+transmuter_token_pairs = choose_transmuter_pool_tokens_by_liq_asc(1)
+
+# One Astroport token pair [[pool_id, ['denom0', 'denom1']]]
+astroport_token_pair = choose_pcl_pool_tokens_by_liq_asc(1)
+
+def create_token_pairs():
+    """
+    Selects the following groups of tokens:
+    1. Top 5 by-liquidity
+    2. Top 5 by-volume
+    3. Five low liquidity (between 5000 and 10000 USD)
+    4. Five low volume (between 5000 and 10000 USD)
+
+    Then,
+    - Puts them all in a set
+    - Constructs combinations between each.
+
+    Returns combinations in the following format:
+    [['denom0', 'denom1']]
+    """
+
+    # Five top by-liquidity tokens
+    top_five_liquidity_tokens = choose_tokens_liq_range(5)
+
+    # Five top by-volume tokens
+    top_five_volume_tokens = choose_tokens_volume_range(5)
+
+    # Five low liquidity tokens
+    five_low_liquidity_tokens = choose_tokens_liq_range(5, 5000, 10000)
+
+    # Five low volume tokens
+    five_low_volume_tokens = choose_tokens_volume_range(5, 5000, 10000)
+
+    # Put all tokens in a set to ensure uniqueness
+    all_tokens = set(top_five_liquidity_tokens + top_five_volume_tokens +
+                     five_low_liquidity_tokens + five_low_volume_tokens)
+
+    # Construct all unique combinations of token pairs
+    token_pairs = list(itertools.combinations(all_tokens, 2))
+
+    # Format pairs for return
+    formatted_pairs = [[token1, token2] for token1, token2 in token_pairs]
+
+    if len(formatted_pairs) > MIN_NUM_MISC_TOKEN_PAIRS:
+        ValueError(f"Constructeed {len(formatted_pairs)}, min expected {MIN_NUM_MISC_TOKEN_PAIRS}")
+
+    return formatted_pairs
+
+misc_token_pairs = create_token_pairs()

tests/sqs_service.py

@@ -1,6 +1,7 @@
 import requests
 
 SQS_STAGE = "https://sqs.stage.osmosis.zone"
+SQS_PROD = "https://sqs.osmosis.zone"
 
 ROUTER_ROUTES_URL = "/router/routes"