Issue3 - Some epochs might not get truevals fix this (#67)

* Add trueval_sign function

* Update trueval

* Add trueval subgraph module

* Update trueval main

* Formatting

* Add models module

* Fixes

* Fixes

* Fixes

* First pass without threading

* Fixes

* Threading first pass

* To be continued

* To be continued

* Almost there

* Done

* Update max threads default

* Linter fixes

* Fixes

* Remove cache

* No more threads!

* Increase sleep time

* Fix merge

* Formatting

* Fixes

* Fixz

* Remove fl

* Remove trueval sign

* Add try-except

* Improvements

* Formatting

* Remove models.py

* Add test files

* Remove unused class

* move trueval subgraph to util

* Fix

* Add contract_data test

* Add slot test

* Improvements

* Fix

* Add quote and base as property

* Update data test

* Fix

* Add trueval tests

* Formatting

* Fixes

* Fix

* Fix

* Fix symbol interpret

* Add live tests

* Improve main

* Formatting

* Fixes

* Remove redundant var

* Order import

* Formatting

* Tests

* Add test mode

* Main test

* Fix and looks

* Formatting

* Disable

* Fix default

* Use filters

* Formatting

* improvements

* Add subgraph test

* Formatting

pdr_backend/models/contract_data.py

@@ -0,0 +1,32 @@
+class ContractData:
+    def __init__(
+        self,
+        name: str,
+        address: str,
+        symbol: str,
+        seconds_per_epoch: int,
+        seconds_per_subscription: int,
+        trueval_submit_timeout: int,
+        owner: str,
+        pair: str,
+        timeframe: str,
+        source: str,
+    ):
+        self.name = name
+        self.address = address
+        self.symbol = symbol
+        self.seconds_per_epoch = seconds_per_epoch
+        self.seconds_per_subscription = seconds_per_subscription
+        self.trueval_submit_timeout = trueval_submit_timeout
+        self.owner = owner
+        self.pair = pair
+        self.timeframe = timeframe
+        self.source = source
+
+    @property
+    def quote(self):
+        return self.pair.split("-")[1]
+
+    @property
+    def base(self):
+        return self.pair.split("-")[0]

pdr_backend/models/slot.py

@@ -0,0 +1,7 @@
+from pdr_backend.models.contract_data import ContractData
+
+
+class Slot:
+    def __init__(self, slot: int, contract: ContractData):
+        self.slot = slot
+        self.contract = contract

pdr_backend/models/test/test_contract_data.py

@@ -0,0 +1,29 @@
+from pdr_backend.models.contract_data import ContractData
+
+
+def test_contract_data_initialization():
+    contract = ContractData(
+        "Contract Name",
+        "0x12345",
+        "test",
+        300,
+        60,
+        15,
+        "0xowner",
+        "BTC-ETH",
+        "1h",
+        "binance",
+    )
+
+    assert contract.name == "Contract Name"
+    assert contract.address == "0x12345"
+    assert contract.symbol == "test"
+    assert contract.seconds_per_epoch == 300
+    assert contract.seconds_per_subscription == 60
+    assert contract.trueval_submit_timeout == 15
+    assert contract.owner == "0xowner"
+    assert contract.pair == "BTC-ETH"
+    assert contract.timeframe == "1h"
+    assert contract.source == "binance"
+    assert contract.quote == "ETH"
+    assert contract.base == "BTC"

pdr_backend/models/test/test_slot.py

@@ -0,0 +1,24 @@
+from pdr_backend.models.slot import Slot
+from pdr_backend.models.contract_data import ContractData
+
+
+def test_slot_initialization():
+    contract = ContractData(
+        "Contract Name",
+        "0x12345",
+        "test",
+        300,
+        60,
+        15,
+        "0xowner",
+        "BTC/ETH",
+        "1h",
+        "binance",
+    )
+
+    slot_number = 5
+    slot = Slot(slot_number, contract)
+
+    assert slot.slot == slot_number
+    assert slot.contract == contract
+    assert isinstance(slot.contract, ContractData)

pdr_backend/trueval/main.py

@@ -1,111 +1,114 @@
-from os import getenv
-from threading import Thread
 import time
+from os import getenv
 from typing import Dict
 
 from pdr_backend.models.predictoor_contract import PredictoorContract
 from pdr_backend.util.env import getenv_or_exit
-from pdr_backend.util.subgraph import query_predictContracts
 from pdr_backend.trueval.trueval import get_true_val
 from pdr_backend.util.web3_config import Web3Config
+from pdr_backend.util.subgraph import get_pending_slots
+from pdr_backend.models.slot import Slot
 
-rpc_url = getenv_or_exit("RPC_URL")
-subgraph_url = getenv_or_exit("SUBGRAPH_URL")
-private_key = getenv_or_exit("PRIVATE_KEY")
-pair_filters = getenv("PAIR_FILTER")
-timeframe_filter = getenv("TIMEFRAME_FILTER")
-source_filter = getenv("SOURCE_FILTER")
-owner_addresses = getenv("OWNER_ADDRS")
-
-web3_config = Web3Config(rpc_url, private_key)
-owner = web3_config.owner
 
-""" Get all intresting topics that we can submit trueval """
-topics: Dict[str, dict] = {}
+contract_cache: Dict[str, tuple] = {}
 
 
-class NewTrueVal(Thread):
-    def __init__(self, topic, predictoor_contract, current_ts, epoch):
-        Thread.__init__(self)
-
-        # set a default value
-        self.values = {
-            "last_submited_epoch": epoch,
-            "contract_address": predictoor_contract.contract_address,
-        }
-        self.topic = topic
-        self.epoch = epoch
+class NewTrueVal:
+    def __init__(
+        self,
+        slot: Slot,
+        predictoor_contract: PredictoorContract,
+        seconds_per_epoch,
+    ):
+        self.slot = slot
         self.predictoor_contract = predictoor_contract
-        self.current_ts = current_ts
+        self.seconds_per_epoch = seconds_per_epoch
 
-    def run(self):
+    def run(self) -> dict:
         """
         Get timestamp of previous epoch-2 , get the price
         Get timestamp of previous epoch-1, get the price
         Compare and submit trueval
         """
-        seconds_per_epoch = self.predictoor_contract.get_secondsPerEpoch()
-        initial_ts = (self.epoch - 2) * seconds_per_epoch
-
-        end_ts = (self.epoch - 1) * seconds_per_epoch
+        self.slot.slot = int(self.slot.slot)
+        initial_ts = self.slot.slot - self.seconds_per_epoch
+        end_ts = self.slot.slot
 
-        slot = (self.epoch - 1) * seconds_per_epoch
+        (true_val, error) = get_true_val(self.slot.contract, initial_ts, end_ts)
+        if error:
+            raise Exception(
+                f"Error getting trueval for {self.slot.contract.pair} and slot {self.slot.slot}"
+            )
 
-        (true_val, cancel_round) = get_true_val(self.topic, initial_ts, end_ts)
+        # pylint: disable=line-too-long
         print(
-            f"Contract:{self.predictoor_contract.contract_address} - "
-            f"Submitting true_val {true_val} for slot:{slot}"
+            f"Contract:{self.predictoor_contract.contract_address} - Submitting true_val {true_val} and slot:{self.slot.slot}"
+        )
+
+        tx = self.predictoor_contract.submit_trueval(
+            true_val, self.slot.slot, False, True
+        )
+
+        return tx
+
+
+def process_slot(slot: Slot, web3_config: Web3Config) -> dict:
+    contract_address = slot.contract.address
+    if contract_address in contract_cache:
+        predictoor_contract, seconds_per_epoch = contract_cache[contract_address]
+    else:
+        predictoor_contract = PredictoorContract(web3_config, contract_address)
+        seconds_per_epoch = predictoor_contract.get_secondsPerEpoch()
+        contract_cache[contract_address] = (
+            predictoor_contract,
+            seconds_per_epoch,
         )
-        try:
-            self.predictoor_contract.submit_trueval(true_val, slot, cancel_round)
-        except Exception as e:
-            print(e)
+    trueval = NewTrueVal(slot, predictoor_contract, seconds_per_epoch)
+    return trueval.run()
+
+
+def main(testing=False):
+    rpc_url = getenv_or_exit("RPC_URL")
+    subgraph_url = getenv_or_exit("SUBGRAPH_URL")
+    private_key = getenv_or_exit("PRIVATE_KEY")
+    pair_filter = getenv("PAIR_FILTER")
+    timeframe_filter = getenv("TIMEFRAME_FILTER")
+    source_filter = getenv("SOURCE_FILTER")
+    owner_addresses = getenv("OWNER_ADDRS")
+    sleep_time = int(getenv("SLEEP_TIME", "30"))
+    batch_size = int(getenv("BATCH_SIZE", "50"))
 
+    web3_config = Web3Config(rpc_url, private_key)
 
-def process_block(block):
-    """Process each contract and see if we need to submit"""
-    global topics
-    if not topics:
-        topics = query_predictContracts(
+    while True:
+        timestamp = web3_config.w3.eth.get_block("latest")["timestamp"]
+        pending_slots = get_pending_slots(
             subgraph_url,
-            pair_filters,
+            timestamp,
+            owner_addresses,
+            pair_filter,
             timeframe_filter,
             source_filter,
-            owner_addresses,
         )
-    print(f"Got new block: {block['number']} with {len(topics)} topics")
-    for address in topics:
-        topic = topics[address]
-        predictoor_contract = PredictoorContract(web3_config, address)
-        epoch = predictoor_contract.get_current_epoch()
-        seconds_per_epoch = predictoor_contract.get_secondsPerEpoch()
-        seconds_till_epoch_end = (
-            epoch * seconds_per_epoch + seconds_per_epoch - block["timestamp"]
-        )
-        print(
-            f"\t{topic['name']} (at address {topic['address']} "
-            f"is at epoch {epoch}, seconds_per_epoch: {seconds_per_epoch}"
-            f", seconds_till_epoch_end: {seconds_till_epoch_end}"
-        )
-        if epoch > topic["last_submited_epoch"] and epoch > 1:
-            # Let's make a prediction & claim rewards
-            thr = NewTrueVal(topic, predictoor_contract, block["timestamp"], epoch)
-            thr.run()
-            address = thr.values["contract_address"].lower()
-            new_epoch = thr.values["last_submited_epoch"]
-            topics[address]["last_submited_epoch"] = new_epoch
-
-
-def main():
-    print("Starting main loop...")
-    lastblock = 0
-    while True:
-        block = web3_config.w3.eth.block_number
-        if block > lastblock:
-            lastblock = block
-            process_block(web3_config.w3.eth.get_block(block, full_transactions=False))
-        else:
-            time.sleep(1)
+        print(f"Found {len(pending_slots)} pending slots, processing {batch_size}")
+        pending_slots = pending_slots[:batch_size]
+
+        if len(pending_slots) == 0:
+            print(f"No pending slots, sleeping for {sleep_time} seconds...")
+            time.sleep(sleep_time)
+            continue
+
+        for slot in pending_slots:
+            print("-" * 30)
+            print(f"Processing slot {slot.slot} for contract {slot.contract.address}")
+            try:
+                process_slot(slot, web3_config)
+            except Exception as e:
+                print("An error occured", e)
+        if testing:
+            break
+        print(f"Done processing, sleeping for {sleep_time} seconds...")
+        time.sleep(sleep_time)
 
 
 if __name__ == "__main__":

pdr_backend/trueval/test/test_trueval.py

@@ -1,5 +1,91 @@
+from unittest.mock import patch
 from pdr_backend.trueval.trueval import get_true_val
+from pdr_backend.models.contract_data import ContractData
 
 
-def test_get_true_val():
-    pass
+def mock_fetch_ohlcv(*args, **kwargs):
+    since = kwargs.get("since")
+    if since == 1:
+        return [[None, 100]]
+    elif since == 2:
+        return [[None, 200]]
+    else:
+        raise ValueError("Invalid timestamp")
+
+
+def mock_fetch_ohlcv_fail(*args, **kwargs):
+    return []
+
+
+def test_get_trueval_success():
+    contract = ContractData(
+        name="ETH-USDT",
+        address="0x1",
+        symbol="ETH-USDT",
+        seconds_per_epoch=60,
+        seconds_per_subscription=500,
+        pair="eth-usdt",
+        source="kraken",
+        timeframe="5m",
+        trueval_submit_timeout=100,
+        owner="0xowner",
+    )
+
+    with patch("ccxt.kraken.fetch_ohlcv", mock_fetch_ohlcv):
+        result = get_true_val(contract, 1, 2)
+        assert result == (True, False)  # 1st True because 200 > 100
+
+
+def test_get_trueval_live_lowercase_slash():
+    contract = ContractData(
+        name="ETH-USDT",
+        address="0x1",
+        symbol="ETH-USDT",
+        seconds_per_epoch=60,
+        seconds_per_subscription=500,
+        pair="btc/usdt",
+        source="kraken",
+        timeframe="5m",
+        trueval_submit_timeout=100,
+        owner="0xowner",
+    )
+
+    result = get_true_val(contract, 1692943200, 1692943500)
+    assert result == (True, False)
+
+
+def test_get_trueval_live_lowercase_dash():
+    contract = ContractData(
+        name="ETH-USDT",
+        address="0x1",
+        symbol="ETH-USDT",
+        seconds_per_epoch=60,
+        seconds_per_subscription=500,
+        pair="btc-usdt",
+        source="kraken",
+        timeframe="5m",
+        trueval_submit_timeout=100,
+        owner="0xowner",
+    )
+
+    result = get_true_val(contract, 1692943200, 1692943500)
+    assert result == (True, False)
+
+
+def test_get_trueval_fail():
+    contract = ContractData(
+        name="ETH-USDT",
+        address="0x1",
+        symbol="ETH-USDT",
+        seconds_per_epoch=60,
+        seconds_per_subscription=500,
+        pair="eth-usdt",
+        source="kraken",
+        timeframe="5m",
+        trueval_submit_timeout=100,
+        owner="0xowner",
+    )
+
+    with patch("ccxt.kraken.fetch_ohlcv", mock_fetch_ohlcv_fail):
+        result = get_true_val(contract, 1, 2)
+        assert result == (False, True)  # 2nd True because failed