mirror of
https://github.com/zedeus/nitter.git
synced 2026-04-09 07:12:10 -04:00
Add experimental x-client-transaction-id support (#1324)
* Add experimental x-client-transaction-id support * Remove broken test
This commit is contained in:
Zed
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
||||
import jsony, packedjson, zippy, oauth1
|
||||
import types, auth, consts, parserutils, http_pool
|
||||
import types, auth, consts, parserutils, http_pool, tid
|
||||
import experimental/types/common
|
||||
|
||||
const
|
||||
@@ -10,7 +10,21 @@ const
|
||||
rlLimit = "x-rate-limit-limit"
|
||||
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
|
||||
|
||||
var pool: HttpPool
|
||||
var
|
||||
pool: HttpPool
|
||||
disableTid: bool
|
||||
|
||||
proc setDisableTid*(disable: bool) =
|
||||
disableTid = disable
|
||||
|
||||
proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
|
||||
case sessionKind
|
||||
of oauth:
|
||||
let o = req.oauth
|
||||
parseUri("https://api.x.com/graphql") / o.endpoint ? o.params
|
||||
of cookie:
|
||||
let c = req.cookie
|
||||
parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
|
||||
|
||||
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
|
||||
let
|
||||
@@ -32,15 +46,15 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
|
||||
proc getCookieHeader(authToken, ct0: string): string =
|
||||
"auth_token=" & authToken & "; ct0=" & ct0
|
||||
|
||||
proc genHeaders*(session: Session, url: string): HttpHeaders =
|
||||
proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
|
||||
result = newHttpHeaders({
|
||||
"connection": "keep-alive",
|
||||
"content-type": "application/json",
|
||||
"x-twitter-active-user": "yes",
|
||||
"x-twitter-client-language": "en",
|
||||
"authority": "api.x.com",
|
||||
"origin": "https://x.com",
|
||||
"accept-encoding": "gzip",
|
||||
"accept-language": "en-US,en;q=0.9",
|
||||
"accept-language": "en-US,en;q=0.5",
|
||||
"accept": "*/*",
|
||||
"DNT": "1",
|
||||
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
@@ -48,15 +62,20 @@ proc genHeaders*(session: Session, url: string): HttpHeaders =
|
||||
|
||||
case session.kind
|
||||
of SessionKind.oauth:
|
||||
result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
|
||||
result["authority"] = "api.x.com"
|
||||
result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
|
||||
of SessionKind.cookie:
|
||||
result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
|
||||
result["x-twitter-auth-type"] = "OAuth2Session"
|
||||
result["x-csrf-token"] = session.ct0
|
||||
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
|
||||
if disableTid:
|
||||
result["authorization"] = bearerToken2
|
||||
else:
|
||||
result["authorization"] = bearerToken
|
||||
result["x-client-transaction-id"] = await genTid(url.path)
|
||||
|
||||
proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
|
||||
result = await getSession(api)
|
||||
proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
|
||||
result = await getSession(req)
|
||||
case result.kind
|
||||
of SessionKind.oauth:
|
||||
if result.oauthToken.len == 0:
|
||||
@@ -73,7 +92,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||
|
||||
try:
|
||||
var resp: AsyncResponse
|
||||
pool.use(genHeaders(session, $url)):
|
||||
pool.use(await genHeaders(session, url)):
|
||||
template getContent =
|
||||
resp = await c.get($url)
|
||||
result = await resp.body
|
||||
@@ -89,7 +108,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
limit = parseInt(resp.headers[rlLimit])
|
||||
session.setRateLimit(api, remaining, reset, limit)
|
||||
session.setRateLimit(req, remaining, reset, limit)
|
||||
|
||||
if result.len > 0:
|
||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||
@@ -98,24 +117,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors notin errorsToSkip:
|
||||
echo "Fetch error, API: ", api, ", errors: ", errors
|
||||
echo "Fetch error, API: ", url.path, ", errors: ", errors
|
||||
if errors in {expiredToken, badToken, locked}:
|
||||
invalidate(session)
|
||||
raise rateLimitError()
|
||||
elif errors in {rateLimited}:
|
||||
# rate limit hit, resets after 24 hours
|
||||
setLimited(session, api)
|
||||
setLimited(session, req)
|
||||
raise rateLimitError()
|
||||
elif result.startsWith("429 Too Many Requests"):
|
||||
echo "[sessions] 429 error, API: ", api, ", session: ", session.pretty
|
||||
session.apis[api].remaining = 0
|
||||
# rate limit hit, resets after the 15 minute window
|
||||
echo "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
|
||||
raise rateLimitError()
|
||||
|
||||
fetchBody
|
||||
|
||||
if resp.status == $Http400:
|
||||
echo "ERROR 400, ", api, ": ", result
|
||||
echo "ERROR 400, ", url.path, ": ", result
|
||||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
@@ -134,19 +151,16 @@ template retry(bod) =
|
||||
try:
|
||||
bod
|
||||
except RateLimitError:
|
||||
echo "[sessions] Rate limited, retrying ", api, " request..."
|
||||
echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
|
||||
bod
|
||||
|
||||
proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
|
||||
proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
|
||||
retry:
|
||||
var
|
||||
body: string
|
||||
session = await getAndValidateSession(api)
|
||||
session = await getAndValidateSession(req)
|
||||
|
||||
when url is SessionAwareUrl:
|
||||
let url = case session.kind
|
||||
of SessionKind.oauth: url.oauthUrl
|
||||
of SessionKind.cookie: url.cookieUrl
|
||||
let url = req.toUrl(session.kind)
|
||||
|
||||
fetchImpl body:
|
||||
if body.startsWith('{') or body.startsWith('['):
|
||||
@@ -157,19 +171,15 @@ proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
|
||||
|
||||
let error = result.getError
|
||||
if error != null and error notin errorsToSkip:
|
||||
echo "Fetch error, API: ", api, ", error: ", error
|
||||
echo "Fetch error, API: ", url.path, ", error: ", error
|
||||
if error in {expiredToken, badToken, locked}:
|
||||
invalidate(session)
|
||||
raise rateLimitError()
|
||||
|
||||
proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
|
||||
proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
|
||||
retry:
|
||||
var session = await getAndValidateSession(api)
|
||||
|
||||
when url is SessionAwareUrl:
|
||||
let url = case session.kind
|
||||
of SessionKind.oauth: url.oauthUrl
|
||||
of SessionKind.cookie: url.cookieUrl
|
||||
var session = await getAndValidateSession(req)
|
||||
let url = req.toUrl(session.kind)
|
||||
|
||||
fetchImpl result:
|
||||
if not (result.startsWith('{') or result.startsWith('[')):
|
||||
|
||||
Reference in New Issue
Block a user