Zed
233 lines
7.7 KiB
Nim
233 lines
7.7 KiB
Nim
# SPDX-License-Identifier: AGPL-3.0-only
|
|
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
|
import jsony, packedjson, zippy, oauth/oauth1
|
|
import types, auth, consts, parserutils, http_pool, tid
|
|
import experimental/types/common
|
|
|
|
const
|
|
rlRemaining = "x-rate-limit-remaining"
|
|
rlReset = "x-rate-limit-reset"
|
|
rlLimit = "x-rate-limit-limit"
|
|
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
|
|
|
|
var
|
|
pool: HttpPool
|
|
disableTid: bool
|
|
apiProxy: string
|
|
maxRetries: int
|
|
retryDelayMs: int
|
|
|
|
proc setDisableTid*(disable: bool) =
|
|
disableTid = disable
|
|
|
|
proc setMaxRetries*(n: int) =
|
|
maxRetries = n
|
|
|
|
proc setRetryDelayMs*(ms: int) =
|
|
retryDelayMs = ms
|
|
|
|
proc setApiProxy*(url: string) =
|
|
apiProxy = ""
|
|
if url.len > 0:
|
|
apiProxy = url.strip(chars={'/'}) & "/"
|
|
if "http" notin apiProxy:
|
|
apiProxy = "http://" & apiProxy
|
|
|
|
proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
|
|
let url = case sessionKind
|
|
of oauth: req.oauth
|
|
of cookie: req.cookie
|
|
let base = case sessionKind
|
|
of oauth: "https://api.x.com"
|
|
of cookie: "https://x.com/i/api"
|
|
let prefix = if url.endpoint.startsWith("1.1/"): "" else: "graphql/"
|
|
parseUri(base) / (prefix & url.endpoint) ? url.params
|
|
|
|
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
|
|
let
|
|
encodedUrl = url.replace(",", "%2C").replace("+", "%20")
|
|
params = OAuth1Parameters(
|
|
consumerKey: consumerKey,
|
|
signatureMethod: "HMAC-SHA1",
|
|
timestamp: $int(round(epochTime())),
|
|
nonce: "0",
|
|
isIncludeVersionToHeader: true,
|
|
token: oauthToken
|
|
)
|
|
signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
|
|
|
|
params.signature = percentEncode(signature)
|
|
|
|
return getOauth1RequestHeader(params)["authorization"]
|
|
|
|
proc getCookieHeader(authToken, ct0: string): string =
|
|
"auth_token=" & authToken & "; ct0=" & ct0
|
|
|
|
proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
|
|
result = newHttpHeaders({
|
|
"accept": "*/*",
|
|
"accept-encoding": "gzip",
|
|
"accept-language": "en-US,en;q=0.9",
|
|
"connection": "keep-alive",
|
|
"content-type": "application/json",
|
|
"origin": "https://x.com",
|
|
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
|
|
"x-twitter-active-user": "yes",
|
|
"x-twitter-client-language": "en",
|
|
"priority": "u=1, i"
|
|
})
|
|
|
|
case session.kind
|
|
of SessionKind.oauth:
|
|
result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
|
|
of SessionKind.cookie:
|
|
result["x-twitter-auth-type"] = "OAuth2Session"
|
|
result["x-csrf-token"] = session.ct0
|
|
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
|
|
result["referer"] = "https://x.com/"
|
|
result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
|
|
result["sec-ch-ua-mobile"] = "?0"
|
|
result["sec-ch-ua-platform"] = "Windows"
|
|
result["sec-fetch-dest"] = "empty"
|
|
result["sec-fetch-mode"] = "cors"
|
|
result["sec-fetch-site"] = "same-origin"
|
|
if disableTid or "/1.1/" in url.path:
|
|
result["authorization"] = bearerToken2
|
|
else:
|
|
result["authorization"] = bearerToken
|
|
result["x-client-transaction-id"] = await genTid(url.path)
|
|
|
|
proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
|
|
result = await getSession(req)
|
|
case result.kind
|
|
of SessionKind.oauth:
|
|
if result.oauthToken.len == 0:
|
|
echo "[sessions] Empty oauth token, session: ", result.pretty
|
|
raise rateLimitError()
|
|
of SessionKind.cookie:
|
|
if result.authToken.len == 0 or result.ct0.len == 0:
|
|
echo "[sessions] Empty cookie credentials, session: ", result.pretty
|
|
raise rateLimitError()
|
|
|
|
template fetchImpl(result, fetchBody) {.dirty.} =
|
|
once:
|
|
pool = HttpPool()
|
|
|
|
try:
|
|
var resp: AsyncResponse
|
|
let headers = await genHeaders(session, url)
|
|
|
|
pool.use(headers):
|
|
template getContent =
|
|
# TODO: this is a temporary simple implementation
|
|
if apiProxy.len > 0 and "/1.1/" notin url.path:
|
|
resp = await c.get(($url).replace("https://", apiProxy))
|
|
else:
|
|
resp = await c.get($url)
|
|
result = await resp.body
|
|
|
|
getContent()
|
|
|
|
if resp.status == $Http503:
|
|
badClient = true
|
|
raise newException(BadClientError, "Bad client")
|
|
|
|
if resp.status == $Http404 and result.len == 0:
|
|
echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
|
|
raise rateLimitError()
|
|
|
|
if resp.headers.hasKey(rlRemaining):
|
|
let
|
|
remaining = parseInt(resp.headers[rlRemaining])
|
|
reset = parseInt(resp.headers[rlReset])
|
|
limit = parseInt(resp.headers[rlLimit])
|
|
session.setRateLimit(req, remaining, reset, limit)
|
|
|
|
if result.len > 0:
|
|
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
|
result = uncompress(result, dfGzip)
|
|
|
|
if result.startsWith("{\"errors"):
|
|
let errors = result.fromJson(Errors)
|
|
if errors notin errorsToSkip:
|
|
echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
|
|
if errors in {expiredToken, badToken, locked}:
|
|
invalidate(session)
|
|
raise rateLimitError()
|
|
elif errors in {rateLimited}:
|
|
# rate limit hit, resets after 24 hours
|
|
setLimited(session, req)
|
|
raise rateLimitError()
|
|
elif result.startsWith("429 Too Many Requests"):
|
|
echo "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
|
|
raise rateLimitError()
|
|
|
|
fetchBody
|
|
|
|
if resp.status == $Http400:
|
|
echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
|
|
raise newException(InternalError, $url)
|
|
except InternalError as e:
|
|
raise e
|
|
except BadClientError as e:
|
|
raise e
|
|
except OSError as e:
|
|
raise e
|
|
except Exception as e:
|
|
let s = session.pretty
|
|
echo "error: ", e.name, ", msg: ", e.msg, ", session: ", s, ", url: ", url
|
|
raise rateLimitError()
|
|
finally:
|
|
release(session)
|
|
|
|
template retry(bod) {.dirty.} =
|
|
var session: Session
|
|
for i in 0 ..< maxRetries:
|
|
try:
|
|
session = nil
|
|
bod
|
|
break
|
|
except RateLimitError:
|
|
let api = if session.isNil: req.cookie.endpoint
|
|
else: req.endpoint(session)
|
|
if session.isNil:
|
|
echo "[sessions] Rate limited, retrying ", api,
|
|
" request (", i, "/", maxRetries, ")..."
|
|
else:
|
|
echo "[sessions] Rate limited, retrying ", api,
|
|
" request (", i, "/", maxRetries, ")..., session: ", session.pretty
|
|
session = nil
|
|
if retryDelayMs > 0:
|
|
await sleepAsync(retryDelayMs)
|
|
|
|
proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
|
|
retry:
|
|
var body: string
|
|
session = await getAndValidateSession(req)
|
|
|
|
let url = req.toUrl(session.kind)
|
|
|
|
fetchImpl body:
|
|
if body.startsWith('{') or body.startsWith('['):
|
|
result = parseJson(body)
|
|
else:
|
|
echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
|
|
result = newJNull()
|
|
|
|
let error = result.getError
|
|
if error != null and error notin errorsToSkip:
|
|
echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
|
|
if error in {expiredToken, badToken, locked}:
|
|
invalidate(session)
|
|
raise rateLimitError()
|
|
|
|
proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
|
|
retry:
|
|
session = await getAndValidateSession(req)
|
|
let url = req.toUrl(session.kind)
|
|
|
|
fetchImpl result:
|
|
if not (result.startsWith('{') or result.startsWith('[')):
|
|
echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
|
|
result.setLen(0)
|