Salastil/nitter
1
0
Fork 0
mirror of https://github.com/zedeus/nitter.git synced 2025-12-05 19:45:36 -05:00
Code Activity

Add experimental x-client-transaction-id support (#1324)

Browse Source 
* Add experimental x-client-transaction-id support

* Remove broken test
This commit is contained in:
Zed
2025-11-29 01:13:08 +01:00
committed by GitHub
parent dae68b4f13
commit 31d210ca47
13 changed files with 239 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nitter.example.conf
View File

@@ -26,6 +26,7 @@ enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.sessions) enableDebug = false # enable request logs and debug endpoints (/.sessions)
proxy = "" # http/https url, SOCKS proxies are not supported proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = "" proxyAuth = ""
disableTid = false # enable this if cookie-based auth is failing
# Change default preferences here, see src/prefs_impl.nim for a complete list # Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences] [Preferences]

129
src/api.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables import asyncdispatch, httpclient, strutils, sequtils, sugar
import packedjson import packedjson
import types, query, formatters, consts, apiutils, parser import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser import experimental/parser as newParser
@@ -11,95 +11,91 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
if fieldToggles.len > 0: if fieldToggles.len > 0:
result.add ("fieldToggles", fieldToggles) result.add ("fieldToggles", fieldToggles)
proc mediaUrl(id: string; cursor: string): SessionAwareUrl = proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
let return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
cookieVars = userMediaVars % [id, cursor]
oauthVars = restIdVars % [id, cursor] proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
result = SessionAwareUrl( let url = apiUrl(endpoint, variables, fieldToggles)
cookieUrl: graphUserMedia ? genParams(cookieVars), return ApiReq(cookie: url, oauth: url)
oauthUrl: graphUserMediaV2 ? genParams(oauthVars)
proc mediaUrl(id: string; cursor: string): ApiReq =
result = ApiReq(
cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor]),
oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
) )
proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl = proc userTweetsUrl(id: string; cursor: string): ApiReq =
let result = ApiReq(
cookieVars = userTweetsVars % [id, cursor] # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
oauthVars = restIdVars % [id, cursor] oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
result = SessionAwareUrl(
# cookieUrl: graphUserTweets ? genParams(cookieVars, userTweetsFieldToggles),
oauthUrl: graphUserTweetsV2 ? genParams(oauthVars)
) )
# might change this in the future pending testing # might change this in the future pending testing
result.cookieUrl = result.oauthUrl result.cookie = result.oauth
proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl = proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
let let cookieVars = userTweetsAndRepliesVars % [id, cursor]
cookieVars = userTweetsAndRepliesVars % [id, cursor] result = ApiReq(
oauthVars = restIdVars % [id, cursor] cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
result = SessionAwareUrl( oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVars, userTweetsFieldToggles),
oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVars)
) )
proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl = proc tweetDetailUrl(id: string; cursor: string): ApiReq =
let let cookieVars = tweetDetailVars % [id, cursor]
cookieVars = tweetDetailVars % [id, cursor] result = ApiReq(
oauthVars = tweetVars % [id, cursor] cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
result = SessionAwareUrl( oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
cookieUrl: graphTweetDetail ? genParams(cookieVars, tweetDetailFieldToggles),
oauthUrl: graphTweet ? genParams(oauthVars)
) )
proc userUrl(username: string): SessionAwareUrl = proc userUrl(username: string): ApiReq =
let let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username result = ApiReq(
oauthVars = """{"screen_name": "$1"}""" % username cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
result = SessionAwareUrl( oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
cookieUrl: graphUser ? genParams(cookieVars, tweetDetailFieldToggles),
oauthUrl: graphUserV2 ? genParams(oauthVars)
) )
proc getGraphUser*(username: string): Future[User] {.async.} = proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return if username.len == 0: return
let js = await fetchRaw(userUrl(username), Api.userScreenName) let js = await fetchRaw(userUrl(username))
result = parseGraphUser(js) result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} = proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return if id.len == 0 or id.any(c => not c.isDigit): return
let let
url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id) url = apiReq(graphUserById, """{"rest_id": "$1"}""" % id)
js = await fetchRaw(url, Api.userRestId) js = await fetchRaw(url)
result = parseGraphUser(js) result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
js = case kind url = case kind
of TimelineKind.tweets: of TimelineKind.tweets: userTweetsUrl(id, cursor)
await fetch(userTweetsUrl(id, cursor), Api.userTweets) of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
of TimelineKind.replies: of TimelineKind.media: mediaUrl(id, cursor)
await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies) js = await fetch(url)
of TimelineKind.media:
await fetch(mediaUrl(id, cursor), Api.userMedia)
result = parseGraphTimeline(js, after) result = parseGraphTimeline(js, after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
url = graphListTweets ? genParams(restIdVars % [id, cursor]) url = apiReq(graphListTweets, restIdVars % [id, cursor])
result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets js = await fetch(url)
result = parseGraphTimeline(js, after).tweets
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let let
variables = %*{"screenName": name, "listSlug": list} variables = %*{"screenName": name, "listSlug": list}
url = graphListBySlug ? genParams($variables) url = apiReq(graphListBySlug, $variables)
result = parseGraphList(await fetch(url, Api.listBySlug)) js = await fetch(url)
result = parseGraphList(js)
proc getGraphList*(id: string): Future[List] {.async.} = proc getGraphList*(id: string): Future[List] {.async.} =
let let
url = graphListById ? genParams("""{"listId": "$1"}""" % id) url = apiReq(graphListById, """{"listId": "$1"}""" % id)
result = parseGraphList(await fetch(url, Api.list)) js = await fetch(url)
result = parseGraphList(js)
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} = proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return if list.id.len == 0: return
@@ -113,22 +109,23 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
} }
if after.len > 0: if after.len > 0:
variables["cursor"] = % after variables["cursor"] = % after
let url = graphListMembers ? genParams($variables) let
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) url = apiReq(graphListMembers, $variables)
js = await fetchRaw(url)
result = parseGraphListMembers(js, after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
variables = """{"rest_id": "$1"}""" % id url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
params = {"variables": variables, "features": gqlFeatures} js = await fetch(url)
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js) result = parseGraphTweetResult(js)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail) js = await fetch(tweetDetailUrl(id, cursor))
result = parseGraphConversation(js, id) result = parseGraphConversation(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -157,8 +154,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
} }
if after.len > 0: if after.len > 0:
variables["cursor"] = % after variables["cursor"] = % after
let url = graphSearchTimeline ? genParams($variables) let
result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) url = apiReq(graphSearchTimeline, $variables)
js = await fetch(url)
result = parseGraphSearch[Tweets](js, after)
result.query = query result.query = query
proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} = proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -179,13 +178,15 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
variables["cursor"] = % after variables["cursor"] = % after
result.beginning = false result.beginning = false
let url = graphSearchTimeline ? genParams($variables) let
result = parseGraphSearch[User](await fetch(url, Api.search), after) url = apiReq(graphSearchTimeline, $variables)
js = await fetch(url)
result = parseGraphSearch[User](js, after)
result.query = query result.query = query
proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
if id.len == 0: return if id.len == 0: return
let js = await fetch(mediaUrl(id, ""), Api.userMedia) let js = await fetch(mediaUrl(id, ""))
result = parseGraphPhotoRail(js) result = parseGraphPhotoRail(js)
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =

74
src/apiutils.nim
View File

@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1 import jsony, packedjson, zippy, oauth1
import types, auth, consts, parserutils, http_pool import types, auth, consts, parserutils, http_pool, tid
import experimental/types/common import experimental/types/common
const const
@@ -10,7 +10,21 @@ const
rlLimit = "x-rate-limit-limit" rlLimit = "x-rate-limit-limit"
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest} errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
var pool: HttpPool var
pool: HttpPool
disableTid: bool
proc setDisableTid*(disable: bool) =
disableTid = disable
proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
case sessionKind
of oauth:
let o = req.oauth
parseUri("https://api.x.com/graphql") / o.endpoint ? o.params
of cookie:
let c = req.cookie
parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
let let
@@ -32,15 +46,15 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
proc getCookieHeader(authToken, ct0: string): string = proc getCookieHeader(authToken, ct0: string): string =
"auth_token=" & authToken & "; ct0=" & ct0 "auth_token=" & authToken & "; ct0=" & ct0
proc genHeaders*(session: Session, url: string): HttpHeaders = proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
result = newHttpHeaders({ result = newHttpHeaders({
"connection": "keep-alive", "connection": "keep-alive",
"content-type": "application/json", "content-type": "application/json",
"x-twitter-active-user": "yes", "x-twitter-active-user": "yes",
"x-twitter-client-language": "en", "x-twitter-client-language": "en",
"authority": "api.x.com", "origin": "https://x.com",
"accept-encoding": "gzip", "accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.9", "accept-language": "en-US,en;q=0.5",
"accept": "*/*", "accept": "*/*",
"DNT": "1", "DNT": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -48,15 +62,20 @@ proc genHeaders*(session: Session, url: string): HttpHeaders =
case session.kind case session.kind
of SessionKind.oauth: of SessionKind.oauth:
result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret) result["authority"] = "api.x.com"
result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
of SessionKind.cookie: of SessionKind.cookie:
result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
result["x-twitter-auth-type"] = "OAuth2Session" result["x-twitter-auth-type"] = "OAuth2Session"
result["x-csrf-token"] = session.ct0 result["x-csrf-token"] = session.ct0
result["cookie"] = getCookieHeader(session.authToken, session.ct0) result["cookie"] = getCookieHeader(session.authToken, session.ct0)
if disableTid:
result["authorization"] = bearerToken2
else:
result["authorization"] = bearerToken
result["x-client-transaction-id"] = await genTid(url.path)
proc getAndValidateSession*(api: Api): Future[Session] {.async.} = proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
result = await getSession(api) result = await getSession(req)
case result.kind case result.kind
of SessionKind.oauth: of SessionKind.oauth:
if result.oauthToken.len == 0: if result.oauthToken.len == 0:
@@ -73,7 +92,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try: try:
var resp: AsyncResponse var resp: AsyncResponse
pool.use(genHeaders(session, $url)): pool.use(await genHeaders(session, url)):
template getContent = template getContent =
resp = await c.get($url) resp = await c.get($url)
result = await resp.body result = await resp.body
@@ -89,7 +108,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
remaining = parseInt(resp.headers[rlRemaining]) remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset]) reset = parseInt(resp.headers[rlReset])
limit = parseInt(resp.headers[rlLimit]) limit = parseInt(resp.headers[rlLimit])
session.setRateLimit(api, remaining, reset, limit) session.setRateLimit(req, remaining, reset, limit)
if result.len > 0: if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip": if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -98,24 +117,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
if result.startsWith("{\"errors"): if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors) let errors = result.fromJson(Errors)
if errors notin errorsToSkip: if errors notin errorsToSkip:
echo "Fetch error, API: ", api, ", errors: ", errors echo "Fetch error, API: ", url.path, ", errors: ", errors
if errors in {expiredToken, badToken, locked}: if errors in {expiredToken, badToken, locked}:
invalidate(session) invalidate(session)
raise rateLimitError() raise rateLimitError()
elif errors in {rateLimited}: elif errors in {rateLimited}:
# rate limit hit, resets after 24 hours # rate limit hit, resets after 24 hours
setLimited(session, api) setLimited(session, req)
raise rateLimitError() raise rateLimitError()
elif result.startsWith("429 Too Many Requests"): elif result.startsWith("429 Too Many Requests"):
echo "[sessions] 429 error, API: ", api, ", session: ", session.pretty echo "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
session.apis[api].remaining = 0
# rate limit hit, resets after the 15 minute window
raise rateLimitError() raise rateLimitError()
fetchBody fetchBody
if resp.status == $Http400: if resp.status == $Http400:
echo "ERROR 400, ", api, ": ", result echo "ERROR 400, ", url.path, ": ", result
raise newException(InternalError, $url) raise newException(InternalError, $url)
except InternalError as e: except InternalError as e:
raise e raise e
@@ -134,19 +151,16 @@ template retry(bod) =
try: try:
bod bod
except RateLimitError: except RateLimitError:
echo "[sessions] Rate limited, retrying ", api, " request..." echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
bod bod
proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} = proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
retry: retry:
var var
body: string body: string
session = await getAndValidateSession(api) session = await getAndValidateSession(req)
when url is SessionAwareUrl: let url = req.toUrl(session.kind)
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
fetchImpl body: fetchImpl body:
if body.startsWith('{') or body.startsWith('['): if body.startsWith('{') or body.startsWith('['):
@@ -157,19 +171,15 @@ proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
let error = result.getError let error = result.getError
if error != null and error notin errorsToSkip: if error != null and error notin errorsToSkip:
echo "Fetch error, API: ", api, ", error: ", error echo "Fetch error, API: ", url.path, ", error: ", error
if error in {expiredToken, badToken, locked}: if error in {expiredToken, badToken, locked}:
invalidate(session) invalidate(session)
raise rateLimitError() raise rateLimitError()
proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} = proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
retry: retry:
var session = await getAndValidateSession(api) var session = await getAndValidateSession(req)
let url = req.toUrl(session.kind)
when url is SessionAwareUrl:
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
fetchImpl result: fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')): if not (result.startsWith('{') or result.startsWith('[')):

32
src/auth.nim
View File

@@ -1,6 +1,6 @@
#SPDX-License-Identifier: AGPL-3.0-only #SPDX-License-Identifier: AGPL-3.0-only
import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os] import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os]
import types import types, consts
import experimental/parser/session import experimental/parser/session
# max requests at a time per session to avoid race conditions # max requests at a time per session to avoid race conditions
@@ -15,6 +15,11 @@ var
template log(str: varargs[string, `$`]) = template log(str: varargs[string, `$`]) =
echo "[sessions] ", str.join("") echo "[sessions] ", str.join("")
proc endpoint(req: ApiReq; session: Session): string =
case session.kind
of oauth: req.oauth.endpoint
of cookie: req.cookie.endpoint
proc pretty*(session: Session): string = proc pretty*(session: Session): string =
if session.isNil: if session.isNil:
return "<null>" return "<null>"
@@ -122,11 +127,12 @@ proc rateLimitError*(): ref RateLimitError =
proc noSessionsError*(): ref NoSessionsError = proc noSessionsError*(): ref NoSessionsError =
newException(NoSessionsError, "no sessions available") newException(NoSessionsError, "no sessions available")
proc isLimited(session: Session; api: Api): bool = proc isLimited(session: Session; req: ApiReq): bool =
if session.isNil: if session.isNil:
return true return true
if session.limited and api != Api.userTweets: let api = req.endpoint(session)
if session.limited and api != graphUserTweetsV2:
if (epochTime().int - session.limitedAt) > hourInSeconds: if (epochTime().int - session.limitedAt) > hourInSeconds:
session.limited = false session.limited = false
log "resetting limit: ", session.pretty log "resetting limit: ", session.pretty
@@ -140,8 +146,8 @@ proc isLimited(session: Session; api: Api): bool =
else: else:
return false return false
proc isReady(session: Session; api: Api): bool = proc isReady(session: Session; req: ApiReq): bool =
not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api)) not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(req))
proc invalidate*(session: var Session) = proc invalidate*(session: var Session) =
if session.isNil: return if session.isNil: return
@@ -156,24 +162,26 @@ proc release*(session: Session) =
if session.isNil: return if session.isNil: return
dec session.pending dec session.pending
proc getSession*(api: Api): Future[Session] {.async.} = proc getSession*(req: ApiReq): Future[Session] {.async.} =
for i in 0 ..< sessionPool.len: for i in 0 ..< sessionPool.len:
if result.isReady(api): break if result.isReady(req): break
result = sessionPool.sample() result = sessionPool.sample()
if not result.isNil and result.isReady(api): if not result.isNil and result.isReady(req):
inc result.pending inc result.pending
else: else:
log "no sessions available for API: ", api log "no sessions available for API: ", req.cookie.endpoint
raise noSessionsError() raise noSessionsError()
proc setLimited*(session: Session; api: Api) = proc setLimited*(session: Session; req: ApiReq) =
let api = req.endpoint(session)
session.limited = true session.limited = true
session.limitedAt = epochTime().int session.limitedAt = epochTime().int
log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty
proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) = proc setRateLimit*(session: Session; req: ApiReq; remaining, reset, limit: int) =
# avoid undefined behavior in race conditions # avoid undefined behavior in race conditions
let api = req.endpoint(session)
if api in session.apis: if api in session.apis:
let rateLimit = session.apis[api] let rateLimit = session.apis[api]
if rateLimit.reset >= reset and rateLimit.remaining < remaining: if rateLimit.reset >= reset and rateLimit.remaining < remaining:

3
src/config.nim
View File

@@ -40,7 +40,8 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
enableRss: cfg.get("Config", "enableRSS", true), enableRss: cfg.get("Config", "enableRSS", true),
enableDebug: cfg.get("Config", "enableDebug", false), enableDebug: cfg.get("Config", "enableDebug", false),
proxy: cfg.get("Config", "proxy", ""), proxy: cfg.get("Config", "proxy", ""),
proxyAuth: cfg.get("Config", "proxyAuth", "") proxyAuth: cfg.get("Config", "proxyAuth", ""),
disableTid: cfg.get("Config", "disableTid", false)
) )
return (conf, cfg) return (conf, cfg)

40
src/consts.nim
View File

@@ -1,29 +1,29 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import uri, strutils import strutils
const const
consumerKey* = "3nVuSoBZnx6U4vzUxf5w" consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
gql = parseUri("https://api.x.com") / "graphql" graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
graphUser* = gql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
graphUserV2* = gql / "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
graphUserById* = gql / "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery" graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
graphUserTweetsV2* = gql / "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
graphUserTweetsAndRepliesV2* = gql / "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets" graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia" graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
graphUserMediaV2* = gql / "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
graphTweet* = gql / "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline" graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
graphTweetResult* = gql / "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
graphSearchTimeline* = gql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
graphListTweets* = gql / "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
gqlFeatures* = """{ gqlFeatures* = """{
"android_ad_formats_media_component_render_overlay_enabled": false, "android_ad_formats_media_component_render_overlay_enabled": false,

8
src/experimental/parser/tid.nim Normal file
View File

@@ -0,0 +1,8 @@
import jsony
import ../types/tid
export TidPair
proc parseTidPairs*(raw: string): seq[TidPair] =
result = raw.fromJson(seq[TidPair])
if result.len == 0:
raise newException(ValueError, "Parsing pairs failed: " & raw)

4
src/experimental/types/tid.nim Normal file
View File

@@ -0,0 +1,4 @@
type
TidPair* = object
animationKey*: string
verification*: string

3
src/nitter.nim
View File

@@ -6,7 +6,7 @@ from os import getEnv
import jester import jester
import types, config, prefs, formatters, redis_cache, http_pool, auth import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
import views/[general, about] import views/[general, about]
import routes/[ import routes/[
preferences, timeline, status, media, search, rss, list, debug, preferences, timeline, status, media, search, rss, list, debug,
@@ -37,6 +37,7 @@ setHmacKey(cfg.hmacKey)
setProxyEncoding(cfg.base64Media) setProxyEncoding(cfg.base64Media)
setMaxHttpConns(cfg.httpMaxConns) setMaxHttpConns(cfg.httpMaxConns)
setHttpProxy(cfg.proxy, cfg.proxyAuth) setHttpProxy(cfg.proxy, cfg.proxyAuth)
setDisableTid(cfg.disableTid)
initAboutPage(cfg.staticDir) initAboutPage(cfg.staticDir)
waitFor initRedisPool(cfg) waitFor initRedisPool(cfg)

62
src/tid.nim Normal file
View File

@@ -0,0 +1,62 @@
import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
import nimcrypto
import experimental/parser/tid
randomize()
const defaultKeyword = "obfiowerehiring";
const pairsUrl =
"https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
var
cachedPairs: seq[TidPair] = @[]
lastCached = 0
# refresh every hour
ttlSec = 60 * 60
proc getPair(): Future[TidPair] {.async.} =
if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
lastCached = int(epochTime())
let client = newAsyncHttpClient()
defer: client.close()
let resp = await client.get(pairsUrl)
if resp.status == $Http200:
cachedPairs = parseTidPairs(await resp.body)
return sample(cachedPairs)
proc encodeSha256(text: string): array[32, byte] =
let
data = cast[ptr byte](addr text[0])
dataLen = uint(len(text))
digest = sha256.digest(data, dataLen)
return digest.data
proc encodeBase64[T](data: T): string =
return encode(data).replace("=", "")
proc decodeBase64(data: string): seq[byte] =
return cast[seq[byte]](decode(data))
proc genTid*(path: string): Future[string] {.async.} =
let
pair = await getPair()
timeNow = int(epochTime() - 1682924400)
timeNowBytes = @[
byte(timeNow and 0xff),
byte((timeNow shr 8) and 0xff),
byte((timeNow shr 16) and 0xff),
byte((timeNow shr 24) and 0xff)
]
data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
hashBytes = encodeSha256(data)
keyBytes = decodeBase64(pair.verification)
bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
randomNum = byte(rand(256))
tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
return encodeBase64(tid)

29
src/types.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import times, sequtils, options, tables, uri import times, sequtils, options, tables
import prefs_impl import prefs_impl
genPrefsType() genPrefsType()
@@ -13,19 +13,13 @@ type
TimelineKind* {.pure.} = enum TimelineKind* {.pure.} = enum
tweets, replies, media tweets, replies, media
Api* {.pure.} = enum ApiUrl* = object
tweetDetail endpoint*: string
tweetResult params*: seq[(string, string)]
search
list ApiReq* = object
listBySlug oauth*: ApiUrl
listMembers cookie*: ApiUrl
listTweets
userRestId
userScreenName
userTweets
userTweetsAndReplies
userMedia
RateLimit* = object RateLimit* = object
limit*: int limit*: int
@@ -42,7 +36,7 @@ type
pending*: int pending*: int
limited*: bool limited*: bool
limitedAt*: int limitedAt*: int
apis*: Table[Api, RateLimit] apis*: Table[string, RateLimit]
case kind*: SessionKind case kind*: SessionKind
of oauth: of oauth:
oauthToken*: string oauthToken*: string
@@ -51,10 +45,6 @@ type
authToken*: string authToken*: string
ct0*: string ct0*: string
SessionAwareUrl* = object
oauthUrl*: Uri
cookieUrl*: Uri
Error* = enum Error* = enum
null = 0 null = 0
noUserMatches = 17 noUserMatches = 17
@@ -285,6 +275,7 @@ type
enableDebug*: bool enableDebug*: bool
proxy*: string proxy*: string
proxyAuth*: string proxyAuth*: string
disableTid*: bool
rssCacheTime*: int rssCacheTime*: int
listCacheTime*: int listCacheTime*: int

7
tests/test_card.py
View File

@@ -11,12 +11,7 @@ card = [
['voidtarget/status/1094632512926605312', ['voidtarget/status/1094632512926605312',
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True], 'gist.github.com', True]
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
'nim-lang.org', True]
] ]
no_thumb = [ no_thumb = [

2
tools/create_session_browser.py
View File

@@ -94,7 +94,7 @@ async def login_and_get_cookies(username, password, totp_seed=None, headless=Fal
async def main(): async def main():
if len(sys.argv) < 3: if len(sys.argv) < 3:
print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]') print('Usage: python3 create_session_browser.py username password [totp_seed] [--append file.jsonl] [--headless]')
sys.exit(1) sys.exit(1)
username = sys.argv[1] username = sys.argv[1]