diff --git a/tests/test_queue_client.py b/tests/test_queue_client.py index ea5cb85..d0698ba 100644 --- a/tests/test_queue_client.py +++ b/tests/test_queue_client.py @@ -159,7 +159,7 @@ async def test_queue_client_passes_effective_proxy_to_xclid(pool_mock: AccountsP def calc(self, *args, **kwargs): return "mocked-clid" - async def fake_get(cls, username, proxy=None, fresh=False): + async def fake_get(cls, username, proxy=None, cookies=None, fresh=False): seen["username"] = username seen["proxy"] = proxy seen["fresh"] = fresh @@ -497,3 +497,45 @@ async def test_404_retries_exhaust_and_abort(client_fixture: CF): assert rep is None await client.__aexit__(None, None, None) + + +async def test_queue_client_passes_account_cookies_to_xclid(pool_mock: AccountsPool, monkeypatch): + mock = MockClient() + seen = {} + + class FakeXClIdGen: + def calc(self, *args, **kwargs): + return "mocked-clid" + + async def fake_get(cls, username, proxy=None, cookies=None, fresh=False): + seen["username"] = username + seen["cookies"] = cookies + seen["fresh"] = fresh + return FakeXClIdGen() + + monkeypatch.setattr(Account, "make_client", lambda self, proxy=None: mock) + monkeypatch.setattr(XClIdGenStore, "get", classmethod(fake_get)) + + await pool_mock.add_account( + "user1", + "pass1", + "email1", + "email_pass1", + cookies='{"auth_token": "abc", "ct0": "def"}', + ) + await pool_mock.set_active("user1", True) + + client = QueueClient(pool_mock, "SearchTimeline") + await client.__aenter__() + + mock.add_response(json={"ok": True}) + rep = await client.get(URL) + + assert rep is not None + assert seen == { + "username": "user1", + "cookies": {"auth_token": "abc", "ct0": "def"}, + "fresh": False, + } + + await client.__aexit__(None, None, None) diff --git a/tests/test_xclid.py b/tests/test_xclid.py index bcef66e..758bc94 100644 --- a/tests/test_xclid.py +++ b/tests/test_xclid.py @@ -9,13 +9,14 @@ class FakeClient: self.closed = True -async def test_xclid_create_passes_proxy_to_client(monkeypatch): +async def test_xclid_create_passes_proxy_and_cookies_to_client(monkeypatch): seen = {} fake_client = FakeClient() - def fake_make_client(*, headers=None, proxy=None): + def fake_make_client(*, headers=None, proxy=None, cookies=None): seen["headers"] = headers seen["proxy"] = proxy + seen["cookies"] = cookies return fake_client async def fake_get_tw_page_text(url, clt): @@ -31,13 +32,39 @@ async def test_xclid_create_passes_proxy_to_client(monkeypatch): monkeypatch.setattr(xclid, "get_tw_page_text", fake_get_tw_page_text) monkeypatch.setattr(xclid, "load_keys", fake_load_keys) - gen = await xclid.XClIdGen.create(proxy="http://127.0.0.1:7897") + proxy = "http://127.0.0.1:7897" + cookies = {"auth_token": "abc", "ct0": "def"} + gen = await xclid.XClIdGen.create(proxy=proxy, cookies=cookies) assert gen.vk_bytes == [1, 2, 3] assert gen.anim_key == "anim-key" assert seen["headers"] == {"user-agent": "@chrome"} - assert seen["proxy"] == "http://127.0.0.1:7897" + assert seen["proxy"] == proxy + assert seen["cookies"] == cookies assert seen["url"] == "https://x.com/tesla" assert seen["client"] is fake_client assert seen["load_client"] is fake_client assert fake_client.closed is True + + +async def test_xclid_create_without_proxy_or_cookies(monkeypatch): + seen = {} + fake_client = FakeClient() + + def fake_make_client(*, headers=None, proxy=None, cookies=None): + seen["proxy"] = proxy + seen["cookies"] = cookies + return fake_client + + async def fake_get_tw_page_text(url, clt): + return "" + + async def fake_load_keys(soup, clt): + return [1, 2, 3], "anim-key" + + monkeypatch.setattr(xclid, "_make_http_client", fake_make_client) + monkeypatch.setattr(xclid, "get_tw_page_text", fake_get_tw_page_text) + monkeypatch.setattr(xclid, "load_keys", fake_load_keys) + + await xclid.XClIdGen.create() + assert seen == {"proxy": None, "cookies": None} diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index 1958d12..380eedb 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -25,7 +25,13 @@ class XClIdGenStore: items: dict[tuple[str, str | None], XClIdGen] = {} # (username, proxy) -> XClIdGen @classmethod - async def get(cls, username: str, proxy: str | None = None, fresh=False) -> XClIdGen: + async def get( + cls, + username: str, + proxy: str | None = None, + cookies: dict[str, str] | None = None, + fresh=False, + ) -> XClIdGen: key = (username, proxy) if key in cls.items and not fresh: return cls.items[key] @@ -33,7 +39,7 @@ class XClIdGenStore: tries = 0 while tries < 3: try: - clid_gen = await XClIdGen.create(proxy=proxy) + clid_gen = await XClIdGen.create(proxy=proxy, cookies=cookies) cls.items[key] = clid_gen return clid_gen except Exception as e: @@ -65,7 +71,12 @@ class Ctx: tries = 0 while tries < 3: - gen = await XClIdGenStore.get(self.acc.username, proxy=self.proxy, fresh=tries > 0) + gen = await XClIdGenStore.get( + self.acc.username, + proxy=self.proxy, + cookies=self.acc.cookies, + fresh=tries > 0, + ) hdr = {"x-client-transaction-id": gen.calc(method, path)} rep = await self.clt.request(method, url, params=params, headers=hdr) if rep.status_code != 404: diff --git a/twscrape/xclid.py b/twscrape/xclid.py index bd61346..13cc4ba 100644 --- a/twscrape/xclid.py +++ b/twscrape/xclid.py @@ -2,8 +2,10 @@ import asyncio import base64 import hashlib import math +import os import random import re +import sys import time from urllib.parse import urljoin @@ -11,10 +13,11 @@ import bs4 from .http import HttpClient from .http import make_client as _make_http_client +from .utils import parse_cookies -def _make_client(proxy: str | None = None) -> HttpClient: - return _make_http_client(headers={"user-agent": "@chrome"}, proxy=proxy) +def _make_client(proxy: str | None = None, cookies: dict[str, str] | None = None) -> HttpClient: + return _make_http_client(headers={"user-agent": "@chrome"}, proxy=proxy, cookies=cookies) async def get_tw_page_text(url: str, clt: HttpClient): @@ -315,8 +318,13 @@ async def load_keys(soup: bs4.BeautifulSoup, clt: HttpClient) -> tuple[list[int] class XClIdGen: @staticmethod - async def create(proxy: str | None = None) -> "XClIdGen": - clt = _make_client(proxy=proxy) + async def create( + proxy: str | None = None, cookies: dict[str, str] | None = None + ) -> "XClIdGen": + # X serves a different/legacy web build to authenticated vs anonymous + # sessions. Only authenticated sessions reliably contain the indices + # this parser depends on (see INDICES_FILE_RE). + clt = _make_client(proxy=proxy, cookies=cookies) try: text = await get_tw_page_text("https://x.com/tesla", clt) soup = bs4.BeautifulSoup(text, "html.parser") @@ -348,7 +356,12 @@ class XClIdGen: async def main(): - clt = _make_client() + cookies_raw = os.getenv("TWS_COOKIES") + cookies = parse_cookies(cookies_raw) if cookies_raw else None + if not cookies: + print("Warning: TWS_COOKIES not set — anonymous fetch will likely fail.", file=sys.stderr) + + clt = _make_client(cookies=cookies) try: text = await get_tw_page_text("https://x.com/elonmusk", clt) soup = bs4.BeautifulSoup(text, "html.parser")