Add functionality to fetch guest accounts on demand

nitter.example.conf

@@ -26,18 +26,14 @@ enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.accounts)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
-tokenCount = 10
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
-# the limits reset every 15 minutes, and the pool is filled up so there's
-# always at least `tokenCount` usable tokens. only increase this if you receive
-# major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
 # Instead of guest_accounts.json, fetch guest accounts from an external URL.
 # Nitter will re-fetch accounts if it runs out of valid ones.
-guestAccountsUrl = ""    # https://example.com/download
-guestAccountsHost = ""   # defaults to nitter's hostname
-guestAccountsKey = ""    # a random string that will be used as a secret to authenticate against the URL
+[GuestAccounts]
+usePool = false  # enable fetching accounts from external pool.
+poolUrl = ""     # https://example.com/download
+poolId = ""      # defaults to nitter's hostname
+poolAuth = ""    # random secret string for authentication
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/auth.nim

@@ -25,11 +25,6 @@ const
   }.toTable
 
 var
-  pool: HttpPool
-  guestAccountsUrl = ""
-  guestAccountsHost = ""
-  guestAccountsKey = ""
-  guestAccountsUrlLastFetched = 0
   accountPool: seq[GuestAccount]
   enableLogging = false
 
@@ -164,27 +159,6 @@ proc release*(account: GuestAccount) =
   dec account.pending
 
 proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
-  let now = epochTime().int
-
-  if accountPool.len == 0 and guestAccountsUrl != "" and guestAccountsUrlLastFetched < now - 3600:
-    once:
-      pool = HttpPool()
-
-    guestAccountsUrlLastFetched = now
-
-    log "fetching more accounts from service"
-    pool.use(newHttpHeaders()):
-      let resp = await c.get("$1?host=$2&key=$3" % [guestAccountsUrl, guestAccountsHost, guestAccountsKey])
-      let guestAccounts = await resp.body
-
-      log "status code from service: ", resp.status
-
-      for line in guestAccounts.splitLines:
-        if line != "":
-          accountPool.add parseGuestAccount(line)
-
-      accountPool.keepItIf(not it.hasExpired)
-
   for i in 0 ..< accountPool.len:
     if result.isReady(api): break
     result = accountPool.sample()
@@ -214,9 +188,6 @@ proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
 
 proc initAccountPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug
-  guestAccountsUrl = cfg.guestAccountsUrl
-  guestAccountsHost = cfg.guestAccountsHost
-  guestAccountsKey = cfg.guestAccountsKey
 
   let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path
 
@@ -227,20 +198,45 @@ proc initAccountPool*(cfg: Config; path: string) =
   elif fileExists(path):
     log "Parsing JSON guest accounts file: ", path
     accountPool = parseGuestAccounts(path)
-  elif guestAccountsUrl == "":
-    echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
+  elif not cfg.guestAccountsUsePool:
+    echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests. Alternatively, configure the guest account pool in nitter.conf"
     quit 1
 
-  let accountsPrePurge = accountPool.len
   accountPool.keepItIf(not it.hasExpired)
 
   log "Successfully added ", accountPool.len, " valid accounts."
 
+proc updateAccountPool*(cfg: Config) {.async.} =
+  if not cfg.guestAccountsUsePool:
+    return
+
+  while true:
+    if accountPool.len == 0:
+      let pool = HttpPool()
+
+      log "fetching more accounts from service"
+      pool.use(newHttpHeaders()):
+        let resp = await c.get("$1?id=$2&auth=$3" % [cfg.guestAccountsPoolUrl, cfg.guestAccountsPoolId, cfg.guestAccountsPoolAuth])
+        let guestAccounts = await resp.body
+
+        log "status code from service: ", resp.status
+
+        for line in guestAccounts.splitLines:
+          if line != "":
+            accountPool.add parseGuestAccount(line)
+
+        accountPool.keepItIf(not it.hasExpired)
+
+    await sleepAsync(3600)
+
 proc getAuthHash*(cfg: Config): string =
-  if cfg.guestAccountsKey == "":
-    log "guestAccountsKey is set to bogus value, responding with empty string"
+  if cfg.guestAccountsPoolAuth == "":
+    # If somebody turns on pool auth and provides a dummy key, we should
+    # prevent third parties from using that mis-configured auth and impersonate
+    # this instance
+    log "poolAuth is set to bogus value, responding with empty string"
     return ""
 
-  let hashStr = $sha_256.digest(cfg.guestAccountsKey)
+  let hashStr = $sha_256.digest(cfg.guestAccountsPoolAuth)
 
   return hashStr.toLowerAscii

src/config.nim

@@ -36,14 +36,16 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     # Config
     hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
     base64Media: cfg.get("Config", "base64Media", false),
-    minTokens: cfg.get("Config", "tokenCount", 10),
     enableRss: cfg.get("Config", "enableRSS", true),
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
     proxyAuth: cfg.get("Config", "proxyAuth", ""),
-    guestAccountsUrl: cfg.get("Config", "guestAccountsUrl", ""),
-    guestAccountsKey: cfg.get("Config", "guestAccountsKey", ""),
-    guestAccountsHost: cfg.get("Config", "guestAccountsHost", cfg.get("Server", "hostname", ""))
+
+    # GuestAccounts
+    guestAccountsUsePool: cfg.get("GuestAccounts", "usePool", false),
+    guestAccountsPoolUrl: cfg.get("GuestAccounts", "poolUrl", ""),
+    guestAccountsPoolAuth: cfg.get("GuestAccounts", "poolAuth", ""),
+    guestAccountsPoolId: cfg.get("GuestAccounts", "poolId", cfg.get("Server", "hostname", ""))
   )
 
   return (conf, cfg)

src/nitter.nim

@@ -22,6 +22,7 @@ let
   accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
 
 initAccountPool(cfg, accountsPath)
+asyncCheck updateAccountPool(cfg)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning

src/routes/auth.nim

@@ -7,6 +7,6 @@ import ".."/[types, auth]
 
 proc createAuthRouter*(cfg: Config) =
   router auth:
-    get "/.well-known/nitter-auth-sha256":
-      cond cfg.guestAccountsUrl != ""
+    get "/.well-known/nitter-request-auth":
+      cond cfg.guestAccountsUsePool
       resp Http200, {"content-type": "text/plain"}, getAuthHash(cfg)

src/types.nim

@@ -254,18 +254,19 @@ type
     title*: string
     hostname*: string
     staticDir*: string
-    guestAccountsUrl*: string
-    guestAccountsKey*: string
-    guestAccountsHost*: string
 
     hmacKey*: string
     base64Media*: bool
-    minTokens*: int
     enableRss*: bool
     enableDebug*: bool
     proxy*: string
     proxyAuth*: string
 
+    guestAccountsUsePool*: bool
+    guestAccountsPoolUrl*: string
+    guestAccountsPoolId*: string
+    guestAccountsPoolAuth*: string
+
     rssCacheTime*: int
     listCacheTime*: int