Add configurable retry logic

Fixes #1382

nitter.example.conf

@@ -34,6 +34,8 @@ proxyAuth = ""
 apiProxy = ""                # nitter-proxy host, e.g. localhost:7000
 disableTid = false           # enable this if cookie-based auth is failing
 maxConcurrentReqs = 2        # max requests at a time per session to avoid race conditions
+maxRetries = 1               # max number of retries on rate limit errors
+retryDelayMs = 150           # delay in ms between retries
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/apiutils.nim

@@ -10,14 +10,22 @@ const
   rlLimit = "x-rate-limit-limit"
   errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
-var 
+var
   pool: HttpPool
   disableTid: bool
   apiProxy: string
+  maxRetries: int
+  retryDelayMs: int
 
 proc setDisableTid*(disable: bool) =
   disableTid = disable
 
+proc setMaxRetries*(n: int) =
+  maxRetries = n
+
+proc setRetryDelayMs*(ms: int) =
+  retryDelayMs = ms
+
 proc setApiProxy*(url: string) =
   apiProxy = ""
   if url.len > 0:
@@ -120,6 +128,10 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         badClient = true
         raise newException(BadClientError, "Bad client")
 
+      if resp.status == $Http404 and result.len == 0:
+        echo "[sessions] transient 404 (empty body), retrying: ", url.path
+        raise rateLimitError()
+
     if resp.headers.hasKey(rlRemaining):
       let
         remaining = parseInt(resp.headers[rlRemaining])
@@ -165,11 +177,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     release(session)
 
 template retry(bod) =
-  try:
-    bod
-  except RateLimitError:
-    echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
-    bod
+  for i in 0 ..< maxRetries:
+    try:
+      bod
+      break
+    except RateLimitError:
+      echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
+           " request (", i, "/", maxRetries, ")..."
+      if retryDelayMs > 0:
+        await sleepAsync(retryDelayMs)
 
 proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:

src/config.nim

@@ -49,7 +49,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     proxyAuth: cfg.get("Config", "proxyAuth", ""),
     apiProxy: cfg.get("Config", "apiProxy", ""),
     disableTid: cfg.get("Config", "disableTid", false),
-    maxConcurrentReqs: cfg.get("Config", "maxConcurrentReqs", 2)
+    maxConcurrentReqs: cfg.get("Config", "maxConcurrentReqs", 2),
+    maxRetries: cfg.get("Config", "maxRetries", 1),
+    retryDelayMs: cfg.get("Config", "retryDelayMs", 150)
   )
 
   return (conf, cfg)

src/nitter.nim

@@ -40,6 +40,8 @@ setHttpProxy(cfg.proxy, cfg.proxyAuth)
 setApiProxy(cfg.apiProxy)
 setDisableTid(cfg.disableTid)
 setMaxConcurrentReqs(cfg.maxConcurrentReqs)
+setMaxRetries(cfg.maxRetries)
+setRetryDelayMs(cfg.retryDelayMs)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)

src/types.nim

@@ -310,6 +310,8 @@ type
     apiProxy*: string
     disableTid*: bool
     maxConcurrentReqs*: int
+    maxRetries*: int
+    retryDelayMs*: int
 
     rssCacheTime*: int
     listCacheTime*: int