diff --git a/.gitignore b/.gitignore index ea520dc..dbd2f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ nitter /public/md/*.html nitter.conf guest_accounts.json* +sessions.json* dump.rdb diff --git a/nitter.example.conf b/nitter.example.conf index f0b4214..360e07b 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -23,7 +23,7 @@ redisMaxConnections = 30 hmacKey = "secretkey" # random key for cryptographic signing of video urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # set this to false to disable RSS feeds -enableDebug = false # enable request logs and debug endpoints (/.accounts) +enableDebug = false # enable request logs and debug endpoints (/.sessions) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" tokenCount = 10 diff --git a/src/apiutils.nim b/src/apiutils.nim index 0a6e0d2..3a0d12c 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -46,14 +46,14 @@ template fetchImpl(result, fetchBody) {.dirty.} = once: pool = HttpPool() - var account = await getGuestAccount(api) - if account.oauthToken.len == 0: - echo "[accounts] Empty oauth token, account: ", account.id + var session = await getSession(api) + if session.oauthToken.len == 0: + echo "[sessions] Empty oauth token, session: ", session.id raise rateLimitError() try: var resp: AsyncResponse - pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)): + pool.use(genHeaders($url, session.oauthToken, session.oauthSecret)): template getContent = resp = await c.get($url) result = await resp.body @@ -68,7 +68,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) - account.setRateLimit(api, remaining, reset) + session.setRateLimit(api, remaining, reset) if result.len > 0: if resp.headers.getOrDefault("content-encoding") == "gzip": @@ -78,15 +78,15 @@ template fetchImpl(result, fetchBody) {.dirty.} = let errors = result.fromJson(Errors) echo "Fetch error, API: ", api, ", errors: ", errors if errors in {expiredToken, badToken, locked}: - invalidate(account) + invalidate(session) raise rateLimitError() elif errors in {rateLimited}: # rate limit hit, resets after 24 hours - setLimited(account, api) + setLimited(session, api) raise rateLimitError() elif result.startsWith("429 Too Many Requests"): - echo "[accounts] 429 error, API: ", api, ", account: ", account.id - account.apis[api].remaining = 0 + echo "[sessions] 429 error, API: ", api, ", session: ", session.id + session.apis[api].remaining = 0 # rate limit hit, resets after the 15 minute window raise rateLimitError() @@ -102,17 +102,17 @@ template fetchImpl(result, fetchBody) {.dirty.} = except OSError as e: raise e except Exception as e: - let id = if account.isNil: "null" else: $account.id - echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url + let id = if session.isNil: "null" else: $session.id + echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url raise rateLimitError() finally: - release(account) + release(session) template retry(bod) = try: bod except RateLimitError: - echo "[accounts] Rate limited, retrying ", api, " request..." + echo "[sessions] Rate limited, retrying ", api, " request..." bod proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = @@ -129,7 +129,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = if error != null: echo "Fetch error, API: ", api, ", error: ", error if error in {expiredToken, badToken, locked}: - invalidate(account) + invalidate(session) raise rateLimitError() proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = diff --git a/src/auth.nim b/src/auth.nim index 85fe4a7..c57008b 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -1,9 +1,9 @@ #SPDX-License-Identifier: AGPL-3.0-only import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import types -import experimental/parser/guestaccount +import experimental/parser/session -# max requests at a time per account to avoid race conditions +# max requests at a time per session to avoid race conditions const maxConcurrentReqs = 3 dayInSeconds = 24 * 60 * 60 @@ -23,16 +23,16 @@ const }.toTable var - accountPool: seq[GuestAccount] + sessionPool: seq[Session] enableLogging = false template log(str: varargs[string, `$`]) = - if enableLogging: echo "[accounts] ", str.join("") + if enableLogging: echo "[sessions] ", str.join("") proc snowflakeToEpoch(flake: int64): int64 = int64(((flake shr 22) + 1288834974657) div 1000) -proc getAccountPoolHealth*(): JsonNode = +proc getSessionPoolHealth*(): JsonNode = let now = epochTime().int var @@ -43,38 +43,38 @@ proc getAccountPoolHealth*(): JsonNode = newest = 0'i64 average = 0'i64 - for account in accountPool: - let created = snowflakeToEpoch(account.id) + for session in sessionPool: + let created = snowflakeToEpoch(session.id) if created > newest: newest = created if created < oldest: oldest = created average += created - if account.limited: - limited.incl account.id + if session.limited: + limited.incl session.id - for api in account.apis.keys: + for api in session.apis.keys: let - apiStatus = account.apis[api] + apiStatus = session.apis[api] reqs = apiMaxReqs[api] - apiStatus.remaining - # no requests made with this account and endpoint since the limit reset + # no requests made with this session and endpoint since the limit reset if apiStatus.reset < now: continue reqsPerApi.mgetOrPut($api, 0).inc reqs totalReqs.inc reqs - if accountPool.len > 0: - average = average div accountPool.len + if sessionPool.len > 0: + average = average div sessionPool.len else: oldest = 0 average = 0 return %*{ - "accounts": %*{ - "total": accountPool.len, + "sessions": %*{ + "total": sessionPool.len, "limited": limited.card, "oldest": $fromUnix(oldest), "newest": $fromUnix(newest), @@ -86,22 +86,22 @@ proc getAccountPoolHealth*(): JsonNode = } } -proc getAccountPoolDebug*(): JsonNode = +proc getSessionPoolDebug*(): JsonNode = let now = epochTime().int var list = newJObject() - for account in accountPool: - let accountJson = %*{ + for session in sessionPool: + let sessionJson = %*{ "apis": newJObject(), - "pending": account.pending, + "pending": session.pending, } - if account.limited: - accountJson["limited"] = %true + if session.limited: + sessionJson["limited"] = %true - for api in account.apis.keys: + for api in session.apis.keys: let - apiStatus = account.apis[api] + apiStatus = session.apis[api] obj = %*{} if apiStatus.reset > now.int: @@ -111,92 +111,91 @@ proc getAccountPoolDebug*(): JsonNode = if "remaining" notin obj: continue - accountJson{"apis", $api} = obj - list[$account.id] = accountJson + sessionJson{"apis", $api} = obj + list[$session.id] = sessionJson return %list proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") -proc noAccountsError*(): ref NoAccountsError = - newException(NoAccountsError, "no accounts available") +proc noSessionsError*(): ref NoSessionsError = + newException(NoSessionsError, "no sessions available") -proc isLimited(account: GuestAccount; api: Api): bool = - if account.isNil: +proc isLimited(session: Session; api: Api): bool = + if session.isNil: return true - if account.limited and api != Api.userTweets: - if (epochTime().int - account.limitedAt) > dayInSeconds: - account.limited = false - log "resetting limit: ", account.id + if session.limited and api != Api.userTweets: + if (epochTime().int - session.limitedAt) > dayInSeconds: + session.limited = false + log "resetting limit: ", session.id else: return false - if api in account.apis: - let limit = account.apis[api] + if api in session.apis: + let limit = session.apis[api] return limit.remaining <= 10 and limit.reset > epochTime().int else: return false -proc isReady(account: GuestAccount; api: Api): bool = - not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api)) +proc isReady(session: Session; api: Api): bool = + not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api)) -proc invalidate*(account: var GuestAccount) = - if account.isNil: return - log "invalidating: ", account.id +proc invalidate*(session: var Session) = + if session.isNil: return + log "invalidating: ", session.id # TODO: This isn't sufficient, but it works for now - let idx = accountPool.find(account) - if idx > -1: accountPool.delete(idx) - account = nil + let idx = sessionPool.find(session) + if idx > -1: sessionPool.delete(idx) + session = nil -proc release*(account: GuestAccount) = - if account.isNil: return - dec account.pending +proc release*(session: Session) = + if session.isNil: return + dec session.pending -proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = - for i in 0 ..< accountPool.len: +proc getSession*(api: Api): Future[Session] {.async.} = + for i in 0 ..< sessionPool.len: if result.isReady(api): break - result = accountPool.sample() + result = sessionPool.sample() if not result.isNil and result.isReady(api): inc result.pending else: - log "no accounts available for API: ", api - raise noAccountsError() + log "no sessions available for API: ", api + raise noSessionsError() -proc setLimited*(account: GuestAccount; api: Api) = - account.limited = true - account.limitedAt = epochTime().int - log "rate limited by api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id +proc setLimited*(session: Session; api: Api) = + session.limited = true + session.limitedAt = epochTime().int + log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id -proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = +proc setRateLimit*(session: Session; api: Api; remaining, reset: int) = # avoid undefined behavior in race conditions - if api in account.apis: - let limit = account.apis[api] + if api in session.apis: + let limit = session.apis[api] if limit.reset >= reset and limit.remaining < remaining: return if limit.reset == reset and limit.remaining >= remaining: - account.apis[api].remaining = remaining + session.apis[api].remaining = remaining return - account.apis[api] = RateLimit(remaining: remaining, reset: reset) + session.apis[api] = RateLimit(remaining: remaining, reset: reset) -proc initAccountPool*(cfg: Config; path: string) = +proc initSessionPool*(cfg: Config; path: string) = enableLogging = cfg.enableDebug - let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path - - if fileExists(jsonlPath): - log "Parsing JSONL guest accounts file: ", jsonlPath - for line in jsonlPath.lines: - accountPool.add parseGuestAccount(line) - elif fileExists(path): - log "Parsing JSON guest accounts file: ", path - accountPool = parseGuestAccounts(path) - else: - echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests." + if path.endsWith(".json"): + echo "[sessions] ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl" quit 1 - log "Successfully added ", accountPool.len, " valid accounts." + if not fileExists(path): + echo "[sessions] ERROR: ", path, " not found. This file is required to authenticate API requests." + quit 1 + + log "Parsing JSONL account sessions file: ", path + for line in path.lines: + sessionPool.add parseSession(line) + + log "Successfully added ", sessionPool.len, " valid account sessions." diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim deleted file mode 100644 index f7e6d34..0000000 --- a/src/experimental/parser/guestaccount.nim +++ /dev/null @@ -1,21 +0,0 @@ -import std/strutils -import jsony -import ../types/guestaccount -from ../../types import GuestAccount - -proc toGuestAccount(account: RawAccount): GuestAccount = - let id = account.oauthToken[0 ..< account.oauthToken.find('-')] - result = GuestAccount( - id: parseBiggestInt(id), - oauthToken: account.oauthToken, - oauthSecret: account.oauthTokenSecret - ) - -proc parseGuestAccount*(raw: string): GuestAccount = - let rawAccount = raw.fromJson(RawAccount) - result = rawAccount.toGuestAccount - -proc parseGuestAccounts*(path: string): seq[GuestAccount] = - let rawAccounts = readFile(path).fromJson(seq[RawAccount]) - for account in rawAccounts: - result.add account.toGuestAccount diff --git a/src/experimental/parser/session.nim b/src/experimental/parser/session.nim new file mode 100644 index 0000000..ee9c93e --- /dev/null +++ b/src/experimental/parser/session.nim @@ -0,0 +1,15 @@ +import std/strutils +import jsony +import ../types/session +from ../../types import Session + +proc parseSession*(raw: string): Session = + let + session = raw.fromJson(RawSession) + id = session.oauthToken[0 ..< session.oauthToken.find('-')] + + result = Session( + id: parseBiggestInt(id), + oauthToken: session.oauthToken, + oauthSecret: session.oauthTokenSecret + ) diff --git a/src/experimental/types/guestaccount.nim b/src/experimental/types/session.nim similarity index 71% rename from src/experimental/types/guestaccount.nim rename to src/experimental/types/session.nim index 244edb3..4165204 100644 --- a/src/experimental/types/guestaccount.nim +++ b/src/experimental/types/session.nim @@ -1,4 +1,4 @@ type - RawAccount* = object + RawSession* = object oauthToken*: string oauthTokenSecret*: string diff --git a/src/nitter.nim b/src/nitter.nim index 958dc0b..f81dc1c 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -19,9 +19,9 @@ let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") (cfg, fullCfg) = getConfig(configPath) - accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") + sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl") -initAccountPool(cfg, accountsPath) +initSessionPool(cfg, sessionsPath) if not cfg.enableDebug: # Silence Jester's query warning @@ -97,10 +97,10 @@ routes: resp Http429, showError( &"Instance has been rate limited.
Use {link} or try again later.", cfg) - error NoAccountsError: + error NoSessionsError: const link = a("another instance", href = instancesUrl) resp Http429, showError( - &"Instance has no available accounts.
Use {link} or try again later.", cfg) + &"Instance has no auth tokens, or is fully rate limited.
Use {link} or try again later.", cfg) extend rss, "" extend status, "" diff --git a/src/routes/debug.nim b/src/routes/debug.nim index 895a285..97c5bef 100644 --- a/src/routes/debug.nim +++ b/src/routes/debug.nim @@ -6,8 +6,8 @@ import ".."/[auth, types] proc createDebugRouter*(cfg: Config) = router debug: get "/.health": - respJson getAccountPoolHealth() + respJson getSessionPoolHealth() - get "/.accounts": + get "/.sessions": cond cfg.enableDebug - respJson getAccountPoolDebug() + respJson getSessionPoolDebug() diff --git a/src/types.nim b/src/types.nim index 685fbad..6c1f1b6 100644 --- a/src/types.nim +++ b/src/types.nim @@ -6,7 +6,7 @@ genPrefsType() type RateLimitError* = object of CatchableError - NoAccountsError* = object of CatchableError + NoSessionsError* = object of CatchableError InternalError* = object of CatchableError BadClientError* = object of CatchableError @@ -31,7 +31,7 @@ type remaining*: int reset*: int - GuestAccount* = ref object + Session* = ref object id*: int64 oauthToken*: string oauthSecret*: string