Salastil/nitter
1
0
Fork 0
mirror of https://github.com/zedeus/nitter.git synced 2025-12-06 03:55:36 -05:00
Code Activity

Rename accounts/guest accounts to sessions

Browse Source 
The new file loaded by default is now ./sessions.jsonl
JSONL is also required, .json support dropped.
This commit is contained in:
Zed
2025-02-05 04:09:36 +01:00
parent afad55749b
commit 6fcd849eff
10 changed files with 114 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -11,4 +11,5 @@ nitter
/public/md/*.html /public/md/*.html
nitter.conf nitter.conf
guest_accounts.json* guest_accounts.json*
sessions.json*
dump.rdb dump.rdb

2
nitter.example.conf
View File

@@ -23,7 +23,7 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.accounts) enableDebug = false # enable request logs and debug endpoints (/.sessions)
proxy = "" # http/https url, SOCKS proxies are not supported proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = "" proxyAuth = ""
tokenCount = 10 tokenCount = 10

28
src/apiutils.nim
View File

@@ -46,14 +46,14 @@ template fetchImpl(result, fetchBody) {.dirty.} =
once: once:
pool = HttpPool() pool = HttpPool()
var account = await getGuestAccount(api) var session = await getSession(api)
if account.oauthToken.len == 0: if session.oauthToken.len == 0:
echo "[accounts] Empty oauth token, account: ", account.id echo "[sessions] Empty oauth token, session: ", session.id
raise rateLimitError() raise rateLimitError()
try: try:
var resp: AsyncResponse var resp: AsyncResponse
pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)): pool.use(genHeaders($url, session.oauthToken, session.oauthSecret)):
template getContent = template getContent =
resp = await c.get($url) resp = await c.get($url)
result = await resp.body result = await resp.body
@@ -68,7 +68,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
let let
remaining = parseInt(resp.headers[rlRemaining]) remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset]) reset = parseInt(resp.headers[rlReset])
account.setRateLimit(api, remaining, reset) session.setRateLimit(api, remaining, reset)
if result.len > 0: if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip": if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -78,15 +78,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
let errors = result.fromJson(Errors) let errors = result.fromJson(Errors)
echo "Fetch error, API: ", api, ", errors: ", errors echo "Fetch error, API: ", api, ", errors: ", errors
if errors in {expiredToken, badToken, locked}: if errors in {expiredToken, badToken, locked}:
invalidate(account) invalidate(session)
raise rateLimitError() raise rateLimitError()
elif errors in {rateLimited}: elif errors in {rateLimited}:
# rate limit hit, resets after 24 hours # rate limit hit, resets after 24 hours
setLimited(account, api) setLimited(session, api)
raise rateLimitError() raise rateLimitError()
elif result.startsWith("429 Too Many Requests"): elif result.startsWith("429 Too Many Requests"):
echo "[accounts] 429 error, API: ", api, ", account: ", account.id echo "[sessions] 429 error, API: ", api, ", session: ", session.id
account.apis[api].remaining = 0 session.apis[api].remaining = 0
# rate limit hit, resets after the 15 minute window # rate limit hit, resets after the 15 minute window
raise rateLimitError() raise rateLimitError()
@@ -102,17 +102,17 @@ template fetchImpl(result, fetchBody) {.dirty.} =
except OSError as e: except OSError as e:
raise e raise e
except Exception as e: except Exception as e:
let id = if account.isNil: "null" else: $account.id let id = if session.isNil: "null" else: $session.id
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
raise rateLimitError() raise rateLimitError()
finally: finally:
release(account) release(session)
template retry(bod) = template retry(bod) =
try: try:
bod bod
except RateLimitError: except RateLimitError:
echo "[accounts] Rate limited, retrying ", api, " request..." echo "[sessions] Rate limited, retrying ", api, " request..."
bod bod
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
@@ -129,7 +129,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if error != null: if error != null:
echo "Fetch error, API: ", api, ", error: ", error echo "Fetch error, API: ", api, ", error: ", error
if error in {expiredToken, badToken, locked}: if error in {expiredToken, badToken, locked}:
invalidate(account) invalidate(session)
raise rateLimitError() raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =

147
src/auth.nim
View File

@@ -1,9 +1,9 @@
#SPDX-License-Identifier: AGPL-3.0-only #SPDX-License-Identifier: AGPL-3.0-only
import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
import types import types
import experimental/parser/guestaccount import experimental/parser/session
# max requests at a time per account to avoid race conditions # max requests at a time per session to avoid race conditions
const const
maxConcurrentReqs = 3 maxConcurrentReqs = 3
dayInSeconds = 24 * 60 * 60 dayInSeconds = 24 * 60 * 60
@@ -23,16 +23,16 @@ const
}.toTable }.toTable
var var
accountPool: seq[GuestAccount] sessionPool: seq[Session]
enableLogging = false enableLogging = false
template log(str: varargs[string, `$`]) = template log(str: varargs[string, `$`]) =
if enableLogging: echo "[accounts] ", str.join("") if enableLogging: echo "[sessions] ", str.join("")
proc snowflakeToEpoch(flake: int64): int64 = proc snowflakeToEpoch(flake: int64): int64 =
int64(((flake shr 22) + 1288834974657) div 1000) int64(((flake shr 22) + 1288834974657) div 1000)
proc getAccountPoolHealth*(): JsonNode = proc getSessionPoolHealth*(): JsonNode =
let now = epochTime().int let now = epochTime().int
var var
@@ -43,38 +43,38 @@ proc getAccountPoolHealth*(): JsonNode =
newest = 0'i64 newest = 0'i64
average = 0'i64 average = 0'i64
for account in accountPool: for session in sessionPool:
let created = snowflakeToEpoch(account.id) let created = snowflakeToEpoch(session.id)
if created > newest: if created > newest:
newest = created newest = created
if created < oldest: if created < oldest:
oldest = created oldest = created
average += created average += created
if account.limited: if session.limited:
limited.incl account.id limited.incl session.id
for api in account.apis.keys: for api in session.apis.keys:
let let
apiStatus = account.apis[api] apiStatus = session.apis[api]
reqs = apiMaxReqs[api] - apiStatus.remaining reqs = apiMaxReqs[api] - apiStatus.remaining
# no requests made with this account and endpoint since the limit reset # no requests made with this session and endpoint since the limit reset
if apiStatus.reset < now: if apiStatus.reset < now:
continue continue
reqsPerApi.mgetOrPut($api, 0).inc reqs reqsPerApi.mgetOrPut($api, 0).inc reqs
totalReqs.inc reqs totalReqs.inc reqs
if accountPool.len > 0: if sessionPool.len > 0:
average = average div accountPool.len average = average div sessionPool.len
else: else:
oldest = 0 oldest = 0
average = 0 average = 0
return %*{ return %*{
"accounts": %*{ "sessions": %*{
"total": accountPool.len, "total": sessionPool.len,
"limited": limited.card, "limited": limited.card,
"oldest": $fromUnix(oldest), "oldest": $fromUnix(oldest),
"newest": $fromUnix(newest), "newest": $fromUnix(newest),
@@ -86,22 +86,22 @@ proc getAccountPoolHealth*(): JsonNode =
} }
} }
proc getAccountPoolDebug*(): JsonNode = proc getSessionPoolDebug*(): JsonNode =
let now = epochTime().int let now = epochTime().int
var list = newJObject() var list = newJObject()
for account in accountPool: for session in sessionPool:
let accountJson = %*{ let sessionJson = %*{
"apis": newJObject(), "apis": newJObject(),
"pending": account.pending, "pending": session.pending,
} }
if account.limited: if session.limited:
accountJson["limited"] = %true sessionJson["limited"] = %true
for api in account.apis.keys: for api in session.apis.keys:
let let
apiStatus = account.apis[api] apiStatus = session.apis[api]
obj = %*{} obj = %*{}
if apiStatus.reset > now.int: if apiStatus.reset > now.int:
@@ -111,92 +111,91 @@ proc getAccountPoolDebug*(): JsonNode =
if "remaining" notin obj: if "remaining" notin obj:
continue continue
accountJson{"apis", $api} = obj sessionJson{"apis", $api} = obj
list[$account.id] = accountJson list[$session.id] = sessionJson
return %list return %list
proc rateLimitError*(): ref RateLimitError = proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited") newException(RateLimitError, "rate limited")
proc noAccountsError*(): ref NoAccountsError = proc noSessionsError*(): ref NoSessionsError =
newException(NoAccountsError, "no accounts available") newException(NoSessionsError, "no sessions available")
proc isLimited(account: GuestAccount; api: Api): bool = proc isLimited(session: Session; api: Api): bool =
if account.isNil: if session.isNil:
return true return true
if account.limited and api != Api.userTweets: if session.limited and api != Api.userTweets:
if (epochTime().int - account.limitedAt) > dayInSeconds: if (epochTime().int - session.limitedAt) > dayInSeconds:
account.limited = false session.limited = false
log "resetting limit: ", account.id log "resetting limit: ", session.id
else: else:
return false return false
if api in account.apis: if api in session.apis:
let limit = account.apis[api] let limit = session.apis[api]
return limit.remaining <= 10 and limit.reset > epochTime().int return limit.remaining <= 10 and limit.reset > epochTime().int
else: else:
return false return false
proc isReady(account: GuestAccount; api: Api): bool = proc isReady(session: Session; api: Api): bool =
not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api)) not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
proc invalidate*(account: var GuestAccount) = proc invalidate*(session: var Session) =
if account.isNil: return if session.isNil: return
log "invalidating: ", account.id log "invalidating: ", session.id
# TODO: This isn't sufficient, but it works for now # TODO: This isn't sufficient, but it works for now
let idx = accountPool.find(account) let idx = sessionPool.find(session)
if idx > -1: accountPool.delete(idx) if idx > -1: sessionPool.delete(idx)
account = nil session = nil
proc release*(account: GuestAccount) = proc release*(session: Session) =
if account.isNil: return if session.isNil: return
dec account.pending dec session.pending
proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = proc getSession*(api: Api): Future[Session] {.async.} =
for i in 0 ..< accountPool.len: for i in 0 ..< sessionPool.len:
if result.isReady(api): break if result.isReady(api): break
result = accountPool.sample() result = sessionPool.sample()
if not result.isNil and result.isReady(api): if not result.isNil and result.isReady(api):
inc result.pending inc result.pending
else: else:
log "no accounts available for API: ", api log "no sessions available for API: ", api
raise noAccountsError() raise noSessionsError()
proc setLimited*(account: GuestAccount; api: Api) = proc setLimited*(session: Session; api: Api) =
account.limited = true session.limited = true
account.limitedAt = epochTime().int session.limitedAt = epochTime().int
log "rate limited by api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) = proc setRateLimit*(session: Session; api: Api; remaining, reset: int) =
# avoid undefined behavior in race conditions # avoid undefined behavior in race conditions
if api in account.apis: if api in session.apis:
let limit = account.apis[api] let limit = session.apis[api]
if limit.reset >= reset and limit.remaining < remaining: if limit.reset >= reset and limit.remaining < remaining:
return return
if limit.reset == reset and limit.remaining >= remaining: if limit.reset == reset and limit.remaining >= remaining:
account.apis[api].remaining = remaining session.apis[api].remaining = remaining
return return
account.apis[api] = RateLimit(remaining: remaining, reset: reset) session.apis[api] = RateLimit(remaining: remaining, reset: reset)
proc initAccountPool*(cfg: Config; path: string) = proc initSessionPool*(cfg: Config; path: string) =
enableLogging = cfg.enableDebug enableLogging = cfg.enableDebug
let jsonlPath = if path.endsWith(".json"): (path & 'l') else: path if path.endsWith(".json"):
echo "[sessions] ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
if fileExists(jsonlPath):
log "Parsing JSONL guest accounts file: ", jsonlPath
for line in jsonlPath.lines:
accountPool.add parseGuestAccount(line)
elif fileExists(path):
log "Parsing JSON guest accounts file: ", path
accountPool = parseGuestAccounts(path)
else:
echo "[accounts] ERROR: ", path, " not found. This file is required to authenticate API requests."
quit 1 quit 1
log "Successfully added ", accountPool.len, " valid accounts." if not fileExists(path):
echo "[sessions] ERROR: ", path, " not found. This file is required to authenticate API requests."
quit 1
log "Parsing JSONL account sessions file: ", path
for line in path.lines:
sessionPool.add parseSession(line)
log "Successfully added ", sessionPool.len, " valid account sessions."

21
src/experimental/parser/guestaccount.nim
View File

@@ -1,21 +0,0 @@
import std/strutils
import jsony
import ../types/guestaccount
from ../../types import GuestAccount
proc toGuestAccount(account: RawAccount): GuestAccount =
let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
result = GuestAccount(
id: parseBiggestInt(id),
oauthToken: account.oauthToken,
oauthSecret: account.oauthTokenSecret
)
proc parseGuestAccount*(raw: string): GuestAccount =
let rawAccount = raw.fromJson(RawAccount)
result = rawAccount.toGuestAccount
proc parseGuestAccounts*(path: string): seq[GuestAccount] =
let rawAccounts = readFile(path).fromJson(seq[RawAccount])
for account in rawAccounts:
result.add account.toGuestAccount

15
src/experimental/parser/session.nim Normal file
View File

@@ -0,0 +1,15 @@
import std/strutils
import jsony
import ../types/session
from ../../types import Session
proc parseSession*(raw: string): Session =
let
session = raw.fromJson(RawSession)
id = session.oauthToken[0 ..< session.oauthToken.find('-')]
result = Session(
id: parseBiggestInt(id),
oauthToken: session.oauthToken,
oauthSecret: session.oauthTokenSecret
)

2
src/experimental/types/guestaccount.nim → src/experimental/types/session.nim
View File

@@ -1,4 +1,4 @@
type type
RawAccount* = object RawSession* = object
oauthToken*: string oauthToken*: string
oauthTokenSecret*: string oauthTokenSecret*: string

8
src/nitter.nim
View File

@@ -19,9 +19,9 @@ let
configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
(cfg, fullCfg) = getConfig(configPath) (cfg, fullCfg) = getConfig(configPath)
accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
initAccountPool(cfg, accountsPath) initSessionPool(cfg, sessionsPath)
if not cfg.enableDebug: if not cfg.enableDebug:
# Silence Jester's query warning # Silence Jester's query warning
@@ -97,10 +97,10 @@ routes:
resp Http429, showError( resp Http429, showError(
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg) &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
error NoAccountsError: error NoSessionsError:
const link = a("another instance", href = instancesUrl) const link = a("another instance", href = instancesUrl)
resp Http429, showError( resp Http429, showError(
&"Instance has no available accounts.<br>Use {link} or try again later.", cfg) &"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
extend rss, "" extend rss, ""
extend status, "" extend status, ""

6
src/routes/debug.nim
View File

@@ -6,8 +6,8 @@ import ".."/[auth, types]
proc createDebugRouter*(cfg: Config) = proc createDebugRouter*(cfg: Config) =
router debug: router debug:
get "/.health": get "/.health":
respJson getAccountPoolHealth() respJson getSessionPoolHealth()
get "/.accounts": get "/.sessions":
cond cfg.enableDebug cond cfg.enableDebug
respJson getAccountPoolDebug() respJson getSessionPoolDebug()

4
src/types.nim
View File

@@ -6,7 +6,7 @@ genPrefsType()
type type
RateLimitError* = object of CatchableError RateLimitError* = object of CatchableError
NoAccountsError* = object of CatchableError NoSessionsError* = object of CatchableError
InternalError* = object of CatchableError InternalError* = object of CatchableError
BadClientError* = object of CatchableError BadClientError* = object of CatchableError
@@ -31,7 +31,7 @@ type
remaining*: int remaining*: int
reset*: int reset*: int
GuestAccount* = ref object Session* = ref object
id*: int64 id*: int64
oauthToken*: string oauthToken*: string
oauthSecret*: string oauthSecret*: string