diff --git a/nitter.example.conf b/nitter.example.conf index 5263ebf..a58c63c 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -34,6 +34,8 @@ proxyAuth = "" apiProxy = "" # nitter-proxy host, e.g. localhost:7000 disableTid = false # enable this if cookie-based auth is failing maxConcurrentReqs = 2 # max requests at a time per session to avoid race conditions +maxRetries = 1 # max number of retries on rate limit errors +retryDelayMs = 150 # delay in ms between retries # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/apiutils.nim b/src/apiutils.nim index 42664cf..fe0479a 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -10,14 +10,22 @@ const rlLimit = "x-rate-limit-limit" errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest} -var +var pool: HttpPool disableTid: bool apiProxy: string + maxRetries: int + retryDelayMs: int proc setDisableTid*(disable: bool) = disableTid = disable +proc setMaxRetries*(n: int) = + maxRetries = n + +proc setRetryDelayMs*(ms: int) = + retryDelayMs = ms + proc setApiProxy*(url: string) = apiProxy = "" if url.len > 0: @@ -120,6 +128,10 @@ template fetchImpl(result, fetchBody) {.dirty.} = badClient = true raise newException(BadClientError, "Bad client") + if resp.status == $Http404 and result.len == 0: + echo "[sessions] transient 404 (empty body), retrying: ", url.path + raise rateLimitError() + if resp.headers.hasKey(rlRemaining): let remaining = parseInt(resp.headers[rlRemaining]) @@ -165,11 +177,15 @@ template fetchImpl(result, fetchBody) {.dirty.} = release(session) template retry(bod) = - try: - bod - except RateLimitError: - echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..." - bod + for i in 0 ..< maxRetries: + try: + bod + break + except RateLimitError: + echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, + " request (", i, "/", maxRetries, ")..." + if retryDelayMs > 0: + await sleepAsync(retryDelayMs) proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = retry: diff --git a/src/config.nim b/src/config.nim index 343b301..800224f 100644 --- a/src/config.nim +++ b/src/config.nim @@ -49,7 +49,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = proxyAuth: cfg.get("Config", "proxyAuth", ""), apiProxy: cfg.get("Config", "apiProxy", ""), disableTid: cfg.get("Config", "disableTid", false), - maxConcurrentReqs: cfg.get("Config", "maxConcurrentReqs", 2) + maxConcurrentReqs: cfg.get("Config", "maxConcurrentReqs", 2), + maxRetries: cfg.get("Config", "maxRetries", 1), + retryDelayMs: cfg.get("Config", "retryDelayMs", 150) ) return (conf, cfg) diff --git a/src/nitter.nim b/src/nitter.nim index ec2decf..ba72818 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -40,6 +40,8 @@ setHttpProxy(cfg.proxy, cfg.proxyAuth) setApiProxy(cfg.apiProxy) setDisableTid(cfg.disableTid) setMaxConcurrentReqs(cfg.maxConcurrentReqs) +setMaxRetries(cfg.maxRetries) +setRetryDelayMs(cfg.retryDelayMs) initAboutPage(cfg.staticDir) waitFor initRedisPool(cfg) diff --git a/src/types.nim b/src/types.nim index c2aca50..7f98b9d 100644 --- a/src/types.nim +++ b/src/types.nim @@ -310,6 +310,8 @@ type apiProxy*: string disableTid*: bool maxConcurrentReqs*: int + maxRetries*: int + retryDelayMs*: int rssCacheTime*: int listCacheTime*: int