Cleanup rewrite (#2)

* cache time,  replace_rpc_text func, get settings endpoint

* moves to its own config

* cleanup

* auto set RPC from receiving domain base URL

* better cache_times json

* Disable endpoint,

* REST to config

* rest helper funcs, no RPC favicon

* remove old script

* cleanup rest setup guide

* cleanup comments

* before_first_request for wsgi

* .env

* cleanup cache times

.env

@@ -1,5 +1,8 @@
-# .env file
-# 
+# === APPLICATION PORTS ===
+# you also need to update run_rest.sh and run_rpc.sh to these values if you change :)
+REST_PORT=5000
+RPC_PORT=5001
+
 # === REDIS ===
 # change 127.0.0.1 to the address or name of docker service
 REDIS_URL=redis://127.0.0.1:6379/0
@@ -13,35 +16,20 @@ INCREASE_COUNTER_EVERY=25
 # ====================================
 # = RPCS & REST ENDPOINTS (TO QUERY) =
 # ====================================
-RPC_URL="http://localhost:26657"
+# Note: These can be localhost if you wish to run on the node itself
+RPC_URL="http://15.204.143.232:26657"
 BACKUP_RPC_URL="https://rpc.juno.strange.love"
 
-# REST_URL="http://15.204.143.232:1317"
-REST_URL="http://localhost:1317"
-BACKUP_REST_URL="https://api.juno.strange.love" # TODO
-
 
-# ====================================
-# = RPCS & REST ENDPOINTS (COSMETIC) =
-# ====================================
-# Basically remove non required ports (80/443) & no http(s)://
-# This repalces the RPCs HTML to match out endpoints on click
-BASE_RPC="localhost:26657" 
-BACKUP_BASE_RPC="rpc.juno.strange.love"
+REST_URL="http://15.204.143.232:1317"
+BACKUP_REST_URL="https://api.juno.strange.love"
 
-# What your A record + domain is ( where the user goes without and ports or http(s):// )
-RPC_DOMAIN="juno-rpc.reece.sh"
+# === WEBSOCKET ===
+RPC_WEBSOCKET="15.204.143.232:26657"
 
-# === WEBSOCKETS ===
-WEBSOCKET_ADDR="15.204.143.232:26657"
-# BACKUP_WEBSOCKET_ADDR="rpc.juno.strange.love:443" ??
 
-# === APPLICATION PORTS ===
-# you also need to update run_rest.sh and run_rpc.sh to these values if you change :)
-REST_PORT=5000
-RPC_PORT=5001
 
 # === Cosmetic ===
-API_TITLE="Juno Network API"
 RPC_TITLE="Juno Network RPC"
+API_TITLE="Juno Network REST API"
 RPC_CUSTOM_TEXT='<a href="https://twitter.com/Reecepbcups_/status/1617396571188133888?s=20&t=OKi00IkStINFqYVweZXlaw">Custom caching solution active for {RPC_DOMAIN}</a><br><a href="https://juno-api.reece.sh">My Juno REST API</a><br>'

CONFIG.py

@@ -0,0 +1,77 @@
+import json
+import os
+import re
+from os import getenv
+
+import redis
+from dotenv import load_dotenv
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+load_dotenv(os.path.join(CURRENT_DIR, ".env"))
+
+# =============
+# === REDIS ===
+# =============
+REDIS_URL = getenv("REDIS_URL", "redis://127.0.0.1:6379/0")
+REDIS_DB = redis.Redis.from_url(REDIS_URL)
+
+ENABLE_COUNTER = getenv("ENABLE_COUNTER", "true").lower().startswith("t")
+INC_EVERY = int(getenv("INCREASE_COUNTER_EVERY", 10))
+
+# ===========
+# === RPC ===
+# ===========
+RPC_PORT = int(getenv("RPC_PORT", 5001))
+RPC_PREFIX = getenv("REDIS_RPC_PREFIX", "junorpc")
+RPC_URL = getenv("RPC_URL", "https://juno-rpc.reece.sh:443")
+
+BACKUP_RPC_URL = getenv("BACKUP_RPC_URL", "https://rpc.juno.strange.love:443")
+
+RPC_WEBSOCKET = f'ws://{getenv("WEBSOCKET_ADDR", "15.204.143.232:26657")}/websocket'
+
+RPC_DOMAIN = getenv("RPC_DOMAIN", "localhost:5001")
+
+# ============
+# === REST ===
+# ============
+REST_PORT = int(getenv("REST_PORT", 5000))
+
+API_TITLE = getenv("API_TITLE", "Swagger API")
+REST_PREFIX = getenv("REDIS_REST_PREFIX", "junorest")
+
+REST_URL = getenv("REST_URL", "https://juno-rest.reece.sh")
+BACKUP_REST_URL = getenv("BACKUP_REST_URL", f"https://api.juno.strange.love")
+
+OPEN_API = f"{REST_URL}/static/openapi.yml"
+
+# === Cache Times ===
+cache_times: dict = {}
+DEFAULT_CACHE_SECONDS: int = 6
+RPC_ENDPOINTS: dict = {}
+REST_ENDPOINTS: dict = {}
+
+# === CACHE HELPER ===
+def update_cache_times():
+    """
+    Updates any config variables which can be changed without restarting the server.
+    Useful for the /cache_info endpoint & actually applying said cache changes at any time
+    """
+    global cache_times, DEFAULT_CACHE_SECONDS, RPC_ENDPOINTS, REST_ENDPOINTS
+
+    with open(os.path.join(CURRENT_DIR, "cache_times.json"), "r") as f:
+        cache_times = json.loads(f.read())
+
+    DEFAULT_CACHE_SECONDS = cache_times.get("DEFAULT", 6)
+    RPC_ENDPOINTS = cache_times.get("rpc", {})
+    REST_ENDPOINTS = cache_times.get("rest", {})
+
+
+def get_cache_time_seconds(path: str, is_rpc: bool) -> int:
+    endpoints = RPC_ENDPOINTS if is_rpc else REST_ENDPOINTS
+
+    for k, seconds in endpoints.items():
+        if re.match(k, path):
+            return seconds
+
+    return DEFAULT_CACHE_SECONDS

HELPERS.py

@@ -0,0 +1,93 @@
+import re
+from os import getenv
+
+import requests
+
+import CONFIG
+from CONFIG import REDIS_DB
+
+total_calls = {
+    # RPC:
+    "total_cache;get_rpc_endpoint": 0,
+    "total_outbound;get_rpc_endpoint": 0,
+    # RPC Cache:
+    "total_cache;post_endpoint": 0,
+    "total_outbound;post_endpoint": 0,
+    # REST:
+    "total_cache;get_all_rest": 0,
+    "total_outbound;get_all_rest": 0,
+}
+
+
+def increment_call_value(key):
+    global total_calls
+
+    if CONFIG.ENABLE_COUNTER == False:
+        return
+
+    if key not in total_calls:
+        total_calls[key] = 0
+
+    if total_calls[key] >= CONFIG.INC_EVERY:
+        REDIS_DB.incr(f"{CONFIG.RPC_PREFIX};{key}", amount=total_calls[key])
+        total_calls[key] = 0
+    else:
+        total_calls[key] += 1
+
+
+def download_openapi_locally():
+    r = requests.get(CONFIG.OPEN_API)
+    file_loc = f"{CONFIG.CURRENT_DIR}/static/openapi.yml"
+    with open(file_loc, "w") as f:
+        f.write(r.text)
+
+
+def get_swagger_code_from_source():
+    req = requests.get(f"{CONFIG.REST_URL}")
+
+    html = req.text.replace(
+        "//unpkg.com/[email protected]/favicon-16x16.png",
+        "/static/rest-favicon.png",
+    )
+    html = re.sub(r"<title>.*</title>", f"<title>{CONFIG.API_TITLE}</title>", html)
+    return html
+
+
+def replace_rpc_text() -> str:
+    # we replace after on requests of the user, then repalce this text to our cache endpoint at time of requests to root endpoint
+    try:
+        RPC_ROOT_HTML = requests.get(f"{CONFIG.RPC_URL}/").text
+    except:
+        RPC_ROOT_HTML = requests.get(f"{CONFIG.BACKUP_RPC_URL}/").text
+
+    RPC_TITLE = getenv("RPC_TITLE", "")
+    if len(RPC_TITLE) > 0:
+        RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
+            "<html><body>",
+            f"<html><head><title>{RPC_TITLE}</title></head><body>",
+        )
+
+    # Puts text at the bottom, maybe put at the top in the future?
+    RPC_CUSTOM_TEXT = getenv("RPC_CUSTOM_TEXT", "").replace(
+        "{RPC_DOMAIN}", f"{CONFIG.RPC_DOMAIN}"
+    )
+    if len(RPC_CUSTOM_TEXT) > 0:
+        RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
+            "Available endpoints:<br><br>",
+            f"{RPC_CUSTOM_TEXT}<br>Available endpoints:<br><br>",
+        )
+
+    # add cache_info endpoint. THIS REMOVES BLANK 'Available endpoints:<br><br>'
+    RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
+        "Available endpoints:<br><br>",
+        f'<a href="//{{BASE_URL}}/cache_info">//{{BASE_URL}}/cache_info</a><br><br>',
+        # we replace the BASE_URL on the call to the root endpoint
+    )
+
+    # Set RPC favicon to nothing
+    RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
+        "<head>",
+        f'<head><link rel="icon" href="data:,">',
+    )
+
+    return RPC_ROOT_HTML

README.md

@@ -59,7 +59,12 @@ docs here...
 ### Variable Length Cache
 
 In the `cache_times.json` file, you can specify specific endpoints and how long said queries should persist in the cache.
-This is useful for large queries such as /validators which may return 100+ validators. This data does not change all the often, making it useful for caching for longer periods of time.
+This is useful for large queries such as /validators which may return 100+ validators. This data does not change often, making it useful for caching for longer periods of time.
+
+If you wish to disable the cache, you can set the value to 0 for said endpoint. If you wish to disable the endpoint query entirely, set to a value less than 0 (such as -1).
+By default the cosmos/auth/v1beta1/accounts endpoint is disabled, as it temporarily halts the node.
 
 This file uses regex pattern matching as keys, with values as the number of seconds to cache once it has been called.
 For python strings, you must prefix any `*` you find with a `.`. So to match "random" in "my 8 random11 string", you would do `.*random.*` to match all before and after.
+
+This is ONLY the path, which means it does not start with a `/`.

cache_times.json

@@ -1,23 +1,26 @@
 {
     "DEFAULT": 6,
     "rpc": {        
-        ".*genesis.*": 3600,
-        ".*block?height=.*": 3600,
-        ".*block_results?height=.*": 3600,
-        ".*unconfirmed_txs.*": 1
+        "genesis": 86400,
+        "genesis.*": 86400,
+        "block?height=.*": 21600,
+        "block_results?height=.*": 21600,
+        "unconfirmed_txs": 1
     },
     "rest": {        
-        ".*/delegations": 300,
-        ".*bank/v1beta1/supply.*": 300,
-        ".*/params": 300,
-        ".*/slashes": 30,
-        ".*/commission": 30,
-        ".*/outstanding_rewards": 30,
-        ".*/proposals": 60,
-        ".*/historical_info/.*": 3600,
-        ".*cosmos/staking/v1beta1/pool": 30,
-        ".*cosmos/staking/v1beta1/validators": 120,
-        ".*ibc/apps/transfer/v1/denom_traces": 30,
-        ".*cosmos/base/tendermint/v1beta1/node_info": 60
+        "cosmos\/auth\/v1beta1\/accounts": -1,
+
+        ".*\/params": 300,
+        ".*delegations": 300,
+        ".*slashes": 30,
+        ".*commission": 30,
+        ".*outstanding_rewards": 30,
+        "cosmos\/gov\/v1beta1\/proposals.*": 60,
+        "cosmos\/staking\/v1beta1\/historical_info.*": 3600,
+        "cosmos\/bank\/v1beta1\/supply": 60,
+        "cosmos\/staking\/v1beta1\/pool": 30,
+        "cosmos\/staking\/v1beta1\/validators": 120,
+        "ibc\/apps\/transfer\/v1\/denom_traces": 30,
+        "tendermint\/v1beta1\/node_info": 60
     }
 }

rest.py

@@ -0,0 +1,86 @@
+# Reece Williams | https://reece.sh | Jan 2023
+# ----------------------------------------------
+# pip install Flask redis flask_caching requests
+# pip install --upgrade urllib3
+# ----------------------------------------------
+
+import json
+import re
+
+import requests
+from flask import Flask, jsonify, request
+from flask_cors import CORS, cross_origin
+
+import CONFIG
+from CONFIG import REDIS_DB
+from HELPERS import (
+    download_openapi_locally,
+    get_swagger_code_from_source,
+    increment_call_value,
+)
+
+app = Flask(__name__)
+cors = CORS(app, resources={r"/*": {"origins": "*"}})
+
+
+REST_SWAGGER_HTML = ""
+
+
+@app.before_first_request
+def before_first_request():
+    CONFIG.update_cache_times()
+    download_openapi_locally()
+
+
+# if route is just /, return the openapi swagger ui
+@app.route("/", methods=["GET"])
+@cross_origin()
+def root():
+    global REST_SWAGGER_HTML
+
+    if len(REST_SWAGGER_HTML) > 0:
+        return REST_SWAGGER_HTML
+
+    REST_SWAGGER_HTML = get_swagger_code_from_source()
+    return REST_SWAGGER_HTML
+
+
+# return all RPC queries
+@app.route("/<path:path>", methods=["GET"])
+@cross_origin()
+def get_all_rest(path):
+    url = f"{CONFIG.REST_URL}/{path}"
+    args = request.args
+
+    cache_seconds = CONFIG.get_cache_time_seconds(path, is_rpc=False)
+    if cache_seconds < 0:
+        return jsonify(
+            {
+                "error": f"cosmos endpoint cache: The path '{path}' is disabled on this node..."
+            }
+        )
+
+    key = f"{CONFIG.REST_PREFIX};{url};{args}"
+
+    v = REDIS_DB.get(key)
+    if v:
+        increment_call_value("total_cache;get_all_rest")
+        return jsonify(json.loads(v))
+
+    try:
+        req = requests.get(url, params=args)
+    except:
+        req = requests.get(f"{CONFIG.BACKUP_REST_URL}/{path}", params=args)
+
+    if req.status_code != 200:
+        return jsonify(req.json())
+
+    REDIS_DB.setex(key, cache_seconds, json.dumps(req.json()))
+    increment_call_value("total_outbound;get_all_rest")
+
+    return req.json()
+
+
+if __name__ == "__main__":
+    before_first_request()
+    app.run(debug=True, host="0.0.0.0", port=CONFIG.REST_PORT)