Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup rewrite #2

Merged
merged 14 commits into from
Jan 26, 2023
36 changes: 12 additions & 24 deletions .env
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# .env file
#
# === APPLICATION PORTS ===
# you also need to update run_rest.sh and run_rpc.sh to these values if you change :)
REST_PORT=5000
RPC_PORT=5001

# === REDIS ===
# change 127.0.0.1 to the address or name of docker service
REDIS_URL=redis://127.0.0.1:6379/0
Expand All @@ -13,35 +16,20 @@ INCREASE_COUNTER_EVERY=25
# ====================================
# = RPCS & REST ENDPOINTS (TO QUERY) =
# ====================================
RPC_URL="http://localhost:26657"
# Note: These can be localhost if you wish to run on the node itself
RPC_URL="http://15.204.143.232:26657"
BACKUP_RPC_URL="https://rpc.juno.strange.love"

# REST_URL="http://15.204.143.232:1317"
REST_URL="http://localhost:1317"
BACKUP_REST_URL="https://api.juno.strange.love" # TODO


# ====================================
# = RPCS & REST ENDPOINTS (COSMETIC) =
# ====================================
# Basically remove non required ports (80/443) & no http(s)://
# This repalces the RPCs HTML to match out endpoints on click
BASE_RPC="localhost:26657"
BACKUP_BASE_RPC="rpc.juno.strange.love"
REST_URL="http://15.204.143.232:1317"
BACKUP_REST_URL="https://api.juno.strange.love"

# What your A record + domain is ( where the user goes without and ports or http(s):// )
RPC_DOMAIN="juno-rpc.reece.sh"
# === WEBSOCKET ===
RPC_WEBSOCKET="15.204.143.232:26657"

# === WEBSOCKETS ===
WEBSOCKET_ADDR="15.204.143.232:26657"
# BACKUP_WEBSOCKET_ADDR="rpc.juno.strange.love:443" ??

# === APPLICATION PORTS ===
# you also need to update run_rest.sh and run_rpc.sh to these values if you change :)
REST_PORT=5000
RPC_PORT=5001

# === Cosmetic ===
API_TITLE="Juno Network API"
RPC_TITLE="Juno Network RPC"
API_TITLE="Juno Network REST API"
RPC_CUSTOM_TEXT='<a href="https://twitter.com/Reecepbcups_/status/1617396571188133888?s=20&t=OKi00IkStINFqYVweZXlaw">Custom caching solution active for {RPC_DOMAIN}</a><br><a href="https://juno-api.reece.sh">My Juno REST API</a><br>'
77 changes: 77 additions & 0 deletions CONFIG.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import json
import os
import re
from os import getenv

import redis
from dotenv import load_dotenv

CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))

load_dotenv(os.path.join(CURRENT_DIR, ".env"))

# =============
# === REDIS ===
# =============
REDIS_URL = getenv("REDIS_URL", "redis://127.0.0.1:6379/0")
REDIS_DB = redis.Redis.from_url(REDIS_URL)

ENABLE_COUNTER = getenv("ENABLE_COUNTER", "true").lower().startswith("t")
INC_EVERY = int(getenv("INCREASE_COUNTER_EVERY", 10))

# ===========
# === RPC ===
# ===========
RPC_PORT = int(getenv("RPC_PORT", 5001))
RPC_PREFIX = getenv("REDIS_RPC_PREFIX", "junorpc")
RPC_URL = getenv("RPC_URL", "https://juno-rpc.reece.sh:443")

BACKUP_RPC_URL = getenv("BACKUP_RPC_URL", "https://rpc.juno.strange.love:443")

RPC_WEBSOCKET = f'ws://{getenv("WEBSOCKET_ADDR", "15.204.143.232:26657")}/websocket'

RPC_DOMAIN = getenv("RPC_DOMAIN", "localhost:5001")

# ============
# === REST ===
# ============
REST_PORT = int(getenv("REST_PORT", 5000))

API_TITLE = getenv("API_TITLE", "Swagger API")
REST_PREFIX = getenv("REDIS_REST_PREFIX", "junorest")

REST_URL = getenv("REST_URL", "https://juno-rest.reece.sh")
BACKUP_REST_URL = getenv("BACKUP_REST_URL", f"https://api.juno.strange.love")

OPEN_API = f"{REST_URL}/static/openapi.yml"

# === Cache Times ===
cache_times: dict = {}
DEFAULT_CACHE_SECONDS: int = 6
RPC_ENDPOINTS: dict = {}
REST_ENDPOINTS: dict = {}

# === CACHE HELPER ===
def update_cache_times():
"""
Updates any config variables which can be changed without restarting the server.
Useful for the /cache_info endpoint & actually applying said cache changes at any time
"""
global cache_times, DEFAULT_CACHE_SECONDS, RPC_ENDPOINTS, REST_ENDPOINTS

with open(os.path.join(CURRENT_DIR, "cache_times.json"), "r") as f:
cache_times = json.loads(f.read())

DEFAULT_CACHE_SECONDS = cache_times.get("DEFAULT", 6)
RPC_ENDPOINTS = cache_times.get("rpc", {})
REST_ENDPOINTS = cache_times.get("rest", {})


def get_cache_time_seconds(path: str, is_rpc: bool) -> int:
endpoints = RPC_ENDPOINTS if is_rpc else REST_ENDPOINTS

for k, seconds in endpoints.items():
if re.match(k, path):
return seconds

return DEFAULT_CACHE_SECONDS
93 changes: 93 additions & 0 deletions HELPERS.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import re
from os import getenv

import requests

import CONFIG
from CONFIG import REDIS_DB

total_calls = {
# RPC:
"total_cache;get_rpc_endpoint": 0,
"total_outbound;get_rpc_endpoint": 0,
# RPC Cache:
"total_cache;post_endpoint": 0,
"total_outbound;post_endpoint": 0,
# REST:
"total_cache;get_all_rest": 0,
"total_outbound;get_all_rest": 0,
}


def increment_call_value(key):
global total_calls

if CONFIG.ENABLE_COUNTER == False:
return

if key not in total_calls:
total_calls[key] = 0

if total_calls[key] >= CONFIG.INC_EVERY:
REDIS_DB.incr(f"{CONFIG.RPC_PREFIX};{key}", amount=total_calls[key])
total_calls[key] = 0
else:
total_calls[key] += 1


def download_openapi_locally():
r = requests.get(CONFIG.OPEN_API)
file_loc = f"{CONFIG.CURRENT_DIR}/static/openapi.yml"
with open(file_loc, "w") as f:
f.write(r.text)


def get_swagger_code_from_source():
req = requests.get(f"{CONFIG.REST_URL}")

html = req.text.replace(
"//unpkg.com/[email protected]/favicon-16x16.png",
"/static/rest-favicon.png",
)
html = re.sub(r"<title>.*</title>", f"<title>{CONFIG.API_TITLE}</title>", html)
return html


def replace_rpc_text() -> str:
# we replace after on requests of the user, then repalce this text to our cache endpoint at time of requests to root endpoint
try:
RPC_ROOT_HTML = requests.get(f"{CONFIG.RPC_URL}/").text
except:
RPC_ROOT_HTML = requests.get(f"{CONFIG.BACKUP_RPC_URL}/").text

RPC_TITLE = getenv("RPC_TITLE", "")
if len(RPC_TITLE) > 0:
RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
"<html><body>",
f"<html><head><title>{RPC_TITLE}</title></head><body>",
)

# Puts text at the bottom, maybe put at the top in the future?
RPC_CUSTOM_TEXT = getenv("RPC_CUSTOM_TEXT", "").replace(
"{RPC_DOMAIN}", f"{CONFIG.RPC_DOMAIN}"
)
if len(RPC_CUSTOM_TEXT) > 0:
RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
"Available endpoints:<br><br>",
f"{RPC_CUSTOM_TEXT}<br>Available endpoints:<br><br>",
)

# add cache_info endpoint. THIS REMOVES BLANK 'Available endpoints:<br><br>'
RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
"Available endpoints:<br><br>",
f'<a href="//{{BASE_URL}}/cache_info">//{{BASE_URL}}/cache_info</a><br><br>',
# we replace the BASE_URL on the call to the root endpoint
)

# Set RPC favicon to nothing
RPC_ROOT_HTML = RPC_ROOT_HTML.replace(
"<head>",
f'<head><link rel="icon" href="data:,">',
)

return RPC_ROOT_HTML
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,12 @@ docs here...
### Variable Length Cache

In the `cache_times.json` file, you can specify specific endpoints and how long said queries should persist in the cache.
This is useful for large queries such as /validators which may return 100+ validators. This data does not change all the often, making it useful for caching for longer periods of time.
This is useful for large queries such as /validators which may return 100+ validators. This data does not change often, making it useful for caching for longer periods of time.

If you wish to disable the cache, you can set the value to 0 for said endpoint. If you wish to disable the endpoint query entirely, set to a value less than 0 (such as -1).
By default the cosmos/auth/v1beta1/accounts endpoint is disabled, as it temporarily halts the node.

This file uses regex pattern matching as keys, with values as the number of seconds to cache once it has been called.
For python strings, you must prefix any `*` you find with a `.`. So to match "random" in "my 8 random11 string", you would do `.*random.*` to match all before and after.

This is ONLY the path, which means it does not start with a `/`.
35 changes: 19 additions & 16 deletions cache_times.json
Original file line number Diff line number Diff line change
@@ -1,23 +1,26 @@
{
"DEFAULT": 6,
"rpc": {
".*genesis.*": 3600,
".*block?height=.*": 3600,
".*block_results?height=.*": 3600,
".*unconfirmed_txs.*": 1
"genesis": 86400,
"genesis.*": 86400,
"block?height=.*": 21600,
"block_results?height=.*": 21600,
"unconfirmed_txs": 1
},
"rest": {
".*/delegations": 300,
".*bank/v1beta1/supply.*": 300,
".*/params": 300,
".*/slashes": 30,
".*/commission": 30,
".*/outstanding_rewards": 30,
".*/proposals": 60,
".*/historical_info/.*": 3600,
".*cosmos/staking/v1beta1/pool": 30,
".*cosmos/staking/v1beta1/validators": 120,
".*ibc/apps/transfer/v1/denom_traces": 30,
".*cosmos/base/tendermint/v1beta1/node_info": 60
"cosmos\/auth\/v1beta1\/accounts": -1,

".*\/params": 300,
".*delegations": 300,
".*slashes": 30,
".*commission": 30,
".*outstanding_rewards": 30,
"cosmos\/gov\/v1beta1\/proposals.*": 60,
"cosmos\/staking\/v1beta1\/historical_info.*": 3600,
"cosmos\/bank\/v1beta1\/supply": 60,
"cosmos\/staking\/v1beta1\/pool": 30,
"cosmos\/staking\/v1beta1\/validators": 120,
"ibc\/apps\/transfer\/v1\/denom_traces": 30,
"tendermint\/v1beta1\/node_info": 60
}
}
86 changes: 86 additions & 0 deletions rest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Reece Williams | https://reece.sh | Jan 2023
# ----------------------------------------------
# pip install Flask redis flask_caching requests
# pip install --upgrade urllib3
# ----------------------------------------------

import json
import re

import requests
from flask import Flask, jsonify, request
from flask_cors import CORS, cross_origin

import CONFIG
from CONFIG import REDIS_DB
from HELPERS import (
download_openapi_locally,
get_swagger_code_from_source,
increment_call_value,
)

app = Flask(__name__)
cors = CORS(app, resources={r"/*": {"origins": "*"}})


REST_SWAGGER_HTML = ""


@app.before_first_request
def before_first_request():
CONFIG.update_cache_times()
download_openapi_locally()


# if route is just /, return the openapi swagger ui
@app.route("/", methods=["GET"])
@cross_origin()
def root():
global REST_SWAGGER_HTML

if len(REST_SWAGGER_HTML) > 0:
return REST_SWAGGER_HTML

REST_SWAGGER_HTML = get_swagger_code_from_source()
return REST_SWAGGER_HTML


# return all RPC queries
@app.route("/<path:path>", methods=["GET"])
@cross_origin()
def get_all_rest(path):
url = f"{CONFIG.REST_URL}/{path}"
args = request.args

cache_seconds = CONFIG.get_cache_time_seconds(path, is_rpc=False)
if cache_seconds < 0:
return jsonify(
{
"error": f"cosmos endpoint cache: The path '{path}' is disabled on this node..."
}
)

key = f"{CONFIG.REST_PREFIX};{url};{args}"

v = REDIS_DB.get(key)
if v:
increment_call_value("total_cache;get_all_rest")
return jsonify(json.loads(v))

try:
req = requests.get(url, params=args)
except:
req = requests.get(f"{CONFIG.BACKUP_REST_URL}/{path}", params=args)

if req.status_code != 200:
return jsonify(req.json())

REDIS_DB.setex(key, cache_seconds, json.dumps(req.json()))
increment_call_value("total_outbound;get_all_rest")

return req.json()


if __name__ == "__main__":
before_first_request()
app.run(debug=True, host="0.0.0.0", port=CONFIG.REST_PORT)
Loading