diff --git a/.gitignore b/.gitignore
index ea520dc..dbd2f6b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,5 @@ nitter
/public/md/*.html
nitter.conf
guest_accounts.json*
+sessions.json*
dump.rdb
diff --git a/nitter.example.conf b/nitter.example.conf
index f0b4214..360e07b 100644
--- a/nitter.example.conf
+++ b/nitter.example.conf
@@ -23,7 +23,7 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds
-enableDebug = false # enable request logs and debug endpoints (/.accounts)
+enableDebug = false # enable request logs and debug endpoints (/.sessions)
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
tokenCount = 10
diff --git a/src/apiutils.nim b/src/apiutils.nim
index 0a6e0d2..3a0d12c 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -46,14 +46,14 @@ template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
- var account = await getGuestAccount(api)
- if account.oauthToken.len == 0:
- echo "[accounts] Empty oauth token, account: ", account.id
+ var session = await getSession(api)
+ if session.oauthToken.len == 0:
+ echo "[sessions] Empty oauth token, session: ", session.id
raise rateLimitError()
try:
var resp: AsyncResponse
- pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)):
+ pool.use(genHeaders($url, session.oauthToken, session.oauthSecret)):
template getContent =
resp = await c.get($url)
result = await resp.body
@@ -68,7 +68,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
- account.setRateLimit(api, remaining, reset)
+ session.setRateLimit(api, remaining, reset)
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -78,15 +78,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
let errors = result.fromJson(Errors)
echo "Fetch error, API: ", api, ", errors: ", errors
if errors in {expiredToken, badToken, locked}:
- invalidate(account)
+ invalidate(session)
raise rateLimitError()
elif errors in {rateLimited}:
# rate limit hit, resets after 24 hours
- setLimited(account, api)
+ setLimited(session, api)
raise rateLimitError()
elif result.startsWith("429 Too Many Requests"):
- echo "[accounts] 429 error, API: ", api, ", account: ", account.id
- account.apis[api].remaining = 0
+ echo "[sessions] 429 error, API: ", api, ", session: ", session.id
+ session.apis[api].remaining = 0
# rate limit hit, resets after the 15 minute window
raise rateLimitError()
@@ -102,17 +102,17 @@ template fetchImpl(result, fetchBody) {.dirty.} =
except OSError as e:
raise e
except Exception as e:
- let id = if account.isNil: "null" else: $account.id
- echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
+ let id = if session.isNil: "null" else: $session.id
+ echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
raise rateLimitError()
finally:
- release(account)
+ release(session)
template retry(bod) =
try:
bod
except RateLimitError:
- echo "[accounts] Rate limited, retrying ", api, " request..."
+ echo "[sessions] Rate limited, retrying ", api, " request..."
bod
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
@@ -129,7 +129,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if error != null:
echo "Fetch error, API: ", api, ", error: ", error
if error in {expiredToken, badToken, locked}:
- invalidate(account)
+ invalidate(session)
raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
diff --git a/src/auth.nim b/src/auth.nim
index 85fe4a7..c57008b 100644
--- a/src/auth.nim
+++ b/src/auth.nim
@@ -1,9 +1,9 @@
#SPDX-License-Identifier: AGPL-3.0-only
import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
import types
-import experimental/parser/guestaccount
+import experimental/parser/session
-# max requests at a time per account to avoid race conditions
+# max requests at a time per session to avoid race conditions
const
maxConcurrentReqs = 3
dayInSeconds = 24 * 60 * 60
@@ -23,16 +23,16 @@ const
}.toTable
var
- accountPool: seq[GuestAccount]
+ sessionPool: seq[Session]
enableLogging = false
template log(str: varargs[string, `$`]) =
- if enableLogging: echo "[accounts] ", str.join("")
+ if enableLogging: echo "[sessions] ", str.join("")
proc snowflakeToEpoch(flake: int64): int64 =
int64(((flake shr 22) + 1288834974657) div 1000)
-proc getAccountPoolHealth*(): JsonNode =
+proc getSessionPoolHealth*(): JsonNode =
let now = epochTime().int
var
@@ -43,38 +43,38 @@ proc getAccountPoolHealth*(): JsonNode =
newest = 0'i64
average = 0'i64
- for account in accountPool:
- let created = snowflakeToEpoch(account.id)
+ for session in sessionPool:
+ let created = snowflakeToEpoch(session.id)
if created > newest:
newest = created
if created < oldest:
oldest = created
average += created
- if account.limited:
- limited.incl account.id
+ if session.limited:
+ limited.incl session.id
- for api in account.apis.keys:
+ for api in session.apis.keys:
let
- apiStatus = account.apis[api]
+ apiStatus = session.apis[api]
reqs = apiMaxReqs[api] - apiStatus.remaining
- # no requests made with this account and endpoint since the limit reset
+ # no requests made with this session and endpoint since the limit reset
if apiStatus.reset < now:
continue
reqsPerApi.mgetOrPut($api, 0).inc reqs
totalReqs.inc reqs
- if accountPool.len > 0:
- average = average div accountPool.len
+ if sessionPool.len > 0:
+ average = average div sessionPool.len
else:
oldest = 0
average = 0
return %*{
- "accounts": %*{
- "total": accountPool.len,
+ "sessions": %*{
+ "total": sessionPool.len,
"limited": limited.card,
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
@@ -86,22 +86,22 @@ proc getAccountPoolHealth*(): JsonNode =
}
}
-proc getAccountPoolDebug*(): JsonNode =
+proc getSessionPoolDebug*(): JsonNode =
let now = epochTime().int
var list = newJObject()
- for account in accountPool:
- let accountJson = %*{
+ for session in sessionPool:
+ let sessionJson = %*{
"apis": newJObject(),
- "pending": account.pending,
+ "pending": session.pending,
}
- if account.limited:
- accountJson["limited"] = %true
+ if session.limited:
+ sessionJson["limited"] = %true
- for api in account.apis.keys:
+ for api in session.apis.keys:
let
- apiStatus = account.apis[api]
+ apiStatus = session.apis[api]
obj = %*{}
if apiStatus.reset > now.int:
@@ -111,92 +111,91 @@ proc getAccountPoolDebug*(): JsonNode =
if "remaining" notin obj:
continue
- accountJson{"apis", $api} = obj
- list[$account.id] = accountJson
+ sessionJson{"apis", $api} = obj
+ list[$session.id] = sessionJson
return %list
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
-proc noAccountsError*(): ref NoAccountsError =
- newException(NoAccountsError, "no accounts available")
+proc noSessionsError*(): ref NoSessionsError =
+ newException(NoSessionsError, "no sessions available")
-proc isLimited(account: GuestAccount; api: Api): bool =
- if account.isNil:
+proc isLimited(session: Session; api: Api): bool =
+ if session.isNil:
return true
- if account.limited and api != Api.userTweets:
- if (epochTime().int - account.limitedAt) > dayInSeconds:
- account.limited = false
- log "resetting limit: ", account.id
+ if session.limited and api != Api.userTweets:
+ if (epochTime().int - session.limitedAt) > dayInSeconds:
+ session.limited = false
+ log "resetting limit: ", session.id
else:
return false
- if api in account.apis:
- let limit = account.apis[api]
+ if api in session.apis:
+ let limit = session.apis[api]
return limit.remaining <= 10 and limit.reset > epochTime().int
else:
return false
-proc isReady(account: GuestAccount; api: Api): bool =
- not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api))
+proc isReady(session: Session; api: Api): bool =
+ not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
-proc invalidate*(account: var GuestAccount) =
- if account.isNil: return
- log "invalidating: ", account.id
+proc invalidate*(session: var Session) =
+ if session.isNil: return
+ log "invalidating: ", session.id
# TODO: This isn't sufficient, but it works for now
- let idx = accountPool.find(account)
- if idx > -1: accountPool.delete(idx)
- account = nil
+ let idx = sessionPool.find(session)
+ if idx > -1: sessionPool.delete(idx)
+ session = nil
-proc release*(account: GuestAccount) =
- if account.isNil: return
- dec account.pending
+proc release*(session: Session) =
+ if session.isNil: return
+ dec session.pending
-proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
- for i in 0 ..< accountPool.len:
+proc getSession*(api: Api): Future[Session] {.async.} =
+ for i in 0 ..< sessionPool.len:
if result.isReady(api): break
- result = accountPool.sample()
+ result = sessionPool.sample()
if not result.isNil and result.isReady(api):
inc result.pending
else:
- log "no accounts available for API: ", api
- raise noAccountsError()
+ log "no sessions available for API: ", api
+ raise noSessionsError()
-proc setLimited*(account: GuestAccount; api: Api) =
- account.limited = true
- account.limitedAt = epochTime().int
- log "rate limited by api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
+proc setLimited*(session: Session; api: Api) =
+ session.limited = true
+ session.limitedAt = epochTime().int
+ log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
-proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
+proc setRateLimit*(session: Session; api: Api; remaining, reset: int) =
# avoid undefined behavior in race conditions
- if api in account.apis:
- let limit = account.apis[api]
+ if api in session.apis:
+ let limit = session.apis[api]
if limit.reset >= reset and limit.remaining < remaining:
return
if limit.reset == reset and limit.remaining >= remaining:
- account.apis[api].remaining = remaining
+ session.apis[api].remaining = remaining
return
- account.apis[api] = RateLimit(remaining: remaining, reset: reset)
+ session.apis[api] = RateLimit(remaining: remaining, reset: reset)
-proc initAccountPool*(cfg: Config; path: string) =
+proc initSessionPool*(cfg: Config; path: string) =
enableLogging = cfg.enableDebug
- let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path
-
- if fileExists(jsonlPath):
- log "Parsing JSONL guest accounts file: ", jsonlPath
- for line in jsonlPath.lines:
- accountPool.add parseGuestAccount(line)
- elif fileExists(path):
- log "Parsing JSON guest accounts file: ", path
- accountPool = parseGuestAccounts(path)
- else:
- echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
+ if path.endsWith(".json"):
+ echo "[sessions] ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
quit 1
- log "Successfully added ", accountPool.len, " valid accounts."
+ if not fileExists(path):
+ echo "[sessions] ERROR: ", path, " not found. This file is required to authenticate API requests."
+ quit 1
+
+ log "Parsing JSONL account sessions file: ", path
+ for line in path.lines:
+ sessionPool.add parseSession(line)
+
+ log "Successfully added ", sessionPool.len, " valid account sessions."
diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim
deleted file mode 100644
index f7e6d34..0000000
--- a/src/experimental/parser/guestaccount.nim
+++ /dev/null
@@ -1,21 +0,0 @@
-import std/strutils
-import jsony
-import ../types/guestaccount
-from ../../types import GuestAccount
-
-proc toGuestAccount(account: RawAccount): GuestAccount =
- let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
- result = GuestAccount(
- id: parseBiggestInt(id),
- oauthToken: account.oauthToken,
- oauthSecret: account.oauthTokenSecret
- )
-
-proc parseGuestAccount*(raw: string): GuestAccount =
- let rawAccount = raw.fromJson(RawAccount)
- result = rawAccount.toGuestAccount
-
-proc parseGuestAccounts*(path: string): seq[GuestAccount] =
- let rawAccounts = readFile(path).fromJson(seq[RawAccount])
- for account in rawAccounts:
- result.add account.toGuestAccount
diff --git a/src/experimental/parser/session.nim b/src/experimental/parser/session.nim
new file mode 100644
index 0000000..ee9c93e
--- /dev/null
+++ b/src/experimental/parser/session.nim
@@ -0,0 +1,15 @@
+import std/strutils
+import jsony
+import ../types/session
+from ../../types import Session
+
+proc parseSession*(raw: string): Session =
+ let
+ session = raw.fromJson(RawSession)
+ id = session.oauthToken[0 ..< session.oauthToken.find('-')]
+
+ result = Session(
+ id: parseBiggestInt(id),
+ oauthToken: session.oauthToken,
+ oauthSecret: session.oauthTokenSecret
+ )
diff --git a/src/experimental/types/guestaccount.nim b/src/experimental/types/session.nim
similarity index 71%
rename from src/experimental/types/guestaccount.nim
rename to src/experimental/types/session.nim
index 244edb3..4165204 100644
--- a/src/experimental/types/guestaccount.nim
+++ b/src/experimental/types/session.nim
@@ -1,4 +1,4 @@
type
- RawAccount* = object
+ RawSession* = object
oauthToken*: string
oauthTokenSecret*: string
diff --git a/src/nitter.nim b/src/nitter.nim
index 958dc0b..f81dc1c 100644
--- a/src/nitter.nim
+++ b/src/nitter.nim
@@ -19,9 +19,9 @@ let
configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
(cfg, fullCfg) = getConfig(configPath)
- accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
+ sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
-initAccountPool(cfg, accountsPath)
+initSessionPool(cfg, sessionsPath)
if not cfg.enableDebug:
# Silence Jester's query warning
@@ -97,10 +97,10 @@ routes:
resp Http429, showError(
&"Instance has been rate limited.
Use {link} or try again later.", cfg)
- error NoAccountsError:
+ error NoSessionsError:
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
- &"Instance has no available accounts.
Use {link} or try again later.", cfg)
+ &"Instance has no auth tokens, or is fully rate limited.
Use {link} or try again later.", cfg)
extend rss, ""
extend status, ""
diff --git a/src/routes/debug.nim b/src/routes/debug.nim
index 895a285..97c5bef 100644
--- a/src/routes/debug.nim
+++ b/src/routes/debug.nim
@@ -6,8 +6,8 @@ import ".."/[auth, types]
proc createDebugRouter*(cfg: Config) =
router debug:
get "/.health":
- respJson getAccountPoolHealth()
+ respJson getSessionPoolHealth()
- get "/.accounts":
+ get "/.sessions":
cond cfg.enableDebug
- respJson getAccountPoolDebug()
+ respJson getSessionPoolDebug()
diff --git a/src/types.nim b/src/types.nim
index 685fbad..6c1f1b6 100644
--- a/src/types.nim
+++ b/src/types.nim
@@ -6,7 +6,7 @@ genPrefsType()
type
RateLimitError* = object of CatchableError
- NoAccountsError* = object of CatchableError
+ NoSessionsError* = object of CatchableError
InternalError* = object of CatchableError
BadClientError* = object of CatchableError
@@ -31,7 +31,7 @@ type
remaining*: int
reset*: int
- GuestAccount* = ref object
+ Session* = ref object
id*: int64
oauthToken*: string
oauthSecret*: string