fix: use account cookies for XClId asset requests (#321)

Co-authored-by: vladkens <[email protected]>

tests/test_queue_client.py

@@ -159,7 +159,7 @@ async def test_queue_client_passes_effective_proxy_to_xclid(pool_mock: AccountsP
         def calc(self, *args, **kwargs):
             return "mocked-clid"
 
-    async def fake_get(cls, username, proxy=None, fresh=False):
+    async def fake_get(cls, username, proxy=None, cookies=None, fresh=False):
         seen["username"] = username
         seen["proxy"] = proxy
         seen["fresh"] = fresh
@@ -497,3 +497,45 @@ async def test_404_retries_exhaust_and_abort(client_fixture: CF):
     assert rep is None
 
     await client.__aexit__(None, None, None)
+
+
+async def test_queue_client_passes_account_cookies_to_xclid(pool_mock: AccountsPool, monkeypatch):
+    mock = MockClient()
+    seen = {}
+
+    class FakeXClIdGen:
+        def calc(self, *args, **kwargs):
+            return "mocked-clid"
+
+    async def fake_get(cls, username, proxy=None, cookies=None, fresh=False):
+        seen["username"] = username
+        seen["cookies"] = cookies
+        seen["fresh"] = fresh
+        return FakeXClIdGen()
+
+    monkeypatch.setattr(Account, "make_client", lambda self, proxy=None: mock)
+    monkeypatch.setattr(XClIdGenStore, "get", classmethod(fake_get))
+
+    await pool_mock.add_account(
+        "user1",
+        "pass1",
+        "email1",
+        "email_pass1",
+        cookies='{"auth_token": "abc", "ct0": "def"}',
+    )
+    await pool_mock.set_active("user1", True)
+
+    client = QueueClient(pool_mock, "SearchTimeline")
+    await client.__aenter__()
+
+    mock.add_response(json={"ok": True})
+    rep = await client.get(URL)
+
+    assert rep is not None
+    assert seen == {
+        "username": "user1",
+        "cookies": {"auth_token": "abc", "ct0": "def"},
+        "fresh": False,
+    }
+
+    await client.__aexit__(None, None, None)

tests/test_xclid.py

@@ -9,13 +9,14 @@ class FakeClient:
         self.closed = True
 
 
-async def test_xclid_create_passes_proxy_to_client(monkeypatch):
+async def test_xclid_create_passes_proxy_and_cookies_to_client(monkeypatch):
     seen = {}
     fake_client = FakeClient()
 
-    def fake_make_client(*, headers=None, proxy=None):
+    def fake_make_client(*, headers=None, proxy=None, cookies=None):
         seen["headers"] = headers
         seen["proxy"] = proxy
+        seen["cookies"] = cookies
         return fake_client
 
     async def fake_get_tw_page_text(url, clt):
@@ -31,13 +32,39 @@ async def test_xclid_create_passes_proxy_to_client(monkeypatch):
     monkeypatch.setattr(xclid, "get_tw_page_text", fake_get_tw_page_text)
     monkeypatch.setattr(xclid, "load_keys", fake_load_keys)
 
-    gen = await xclid.XClIdGen.create(proxy="http://127.0.0.1:7897")
+    proxy = "http://127.0.0.1:7897"
+    cookies = {"auth_token": "abc", "ct0": "def"}
+    gen = await xclid.XClIdGen.create(proxy=proxy, cookies=cookies)
 
     assert gen.vk_bytes == [1, 2, 3]
     assert gen.anim_key == "anim-key"
     assert seen["headers"] == {"user-agent": "@chrome"}
-    assert seen["proxy"] == "http://127.0.0.1:7897"
+    assert seen["proxy"] == proxy
+    assert seen["cookies"] == cookies
     assert seen["url"] == "https://x.com/tesla"
     assert seen["client"] is fake_client
     assert seen["load_client"] is fake_client
     assert fake_client.closed is True
+
+
+async def test_xclid_create_without_proxy_or_cookies(monkeypatch):
+    seen = {}
+    fake_client = FakeClient()
+
+    def fake_make_client(*, headers=None, proxy=None, cookies=None):
+        seen["proxy"] = proxy
+        seen["cookies"] = cookies
+        return fake_client
+
+    async def fake_get_tw_page_text(url, clt):
+        return "<html></html>"
+
+    async def fake_load_keys(soup, clt):
+        return [1, 2, 3], "anim-key"
+
+    monkeypatch.setattr(xclid, "_make_http_client", fake_make_client)
+    monkeypatch.setattr(xclid, "get_tw_page_text", fake_get_tw_page_text)
+    monkeypatch.setattr(xclid, "load_keys", fake_load_keys)
+
+    await xclid.XClIdGen.create()
+    assert seen == {"proxy": None, "cookies": None}

twscrape/queue_client.py

@@ -25,7 +25,13 @@ class XClIdGenStore:
     items: dict[tuple[str, str | None], XClIdGen] = {}  # (username, proxy) -> XClIdGen
 
     @classmethod
-    async def get(cls, username: str, proxy: str | None = None, fresh=False) -> XClIdGen:
+    async def get(
+        cls,
+        username: str,
+        proxy: str | None = None,
+        cookies: dict[str, str] | None = None,
+        fresh=False,
+    ) -> XClIdGen:
         key = (username, proxy)
         if key in cls.items and not fresh:
             return cls.items[key]
@@ -33,7 +39,7 @@ class XClIdGenStore:
         tries = 0
         while tries < 3:
             try:
-                clid_gen = await XClIdGen.create(proxy=proxy)
+                clid_gen = await XClIdGen.create(proxy=proxy, cookies=cookies)
                 cls.items[key] = clid_gen
                 return clid_gen
             except Exception as e:
@@ -65,7 +71,12 @@ class Ctx:
 
         tries = 0
         while tries < 3:
-            gen = await XClIdGenStore.get(self.acc.username, proxy=self.proxy, fresh=tries > 0)
+            gen = await XClIdGenStore.get(
+                self.acc.username,
+                proxy=self.proxy,
+                cookies=self.acc.cookies,
+                fresh=tries > 0,
+            )
             hdr = {"x-client-transaction-id": gen.calc(method, path)}
             rep = await self.clt.request(method, url, params=params, headers=hdr)
             if rep.status_code != 404:

twscrape/xclid.py

@@ -2,8 +2,10 @@ import asyncio
 import base64
 import hashlib
 import math
+import os
 import random
 import re
+import sys
 import time
 from urllib.parse import urljoin
 
@@ -11,10 +13,11 @@ import bs4
 
 from .http import HttpClient
 from .http import make_client as _make_http_client
+from .utils import parse_cookies
 
 
-def _make_client(proxy: str | None = None) -> HttpClient:
-    return _make_http_client(headers={"user-agent": "@chrome"}, proxy=proxy)
+def _make_client(proxy: str | None = None, cookies: dict[str, str] | None = None) -> HttpClient:
+    return _make_http_client(headers={"user-agent": "@chrome"}, proxy=proxy, cookies=cookies)
 
 
 async def get_tw_page_text(url: str, clt: HttpClient):
@@ -315,8 +318,13 @@ async def load_keys(soup: bs4.BeautifulSoup, clt: HttpClient) -> tuple[list[int]
 
 class XClIdGen:
     @staticmethod
-    async def create(proxy: str | None = None) -> "XClIdGen":
-        clt = _make_client(proxy=proxy)
+    async def create(
+        proxy: str | None = None, cookies: dict[str, str] | None = None
+    ) -> "XClIdGen":
+        # X serves a different/legacy web build to authenticated vs anonymous
+        # sessions. Only authenticated sessions reliably contain the indices
+        # this parser depends on (see INDICES_FILE_RE).
+        clt = _make_client(proxy=proxy, cookies=cookies)
         try:
             text = await get_tw_page_text("https://x.com/tesla", clt)
             soup = bs4.BeautifulSoup(text, "html.parser")
@@ -348,7 +356,12 @@ class XClIdGen:
 
 
 async def main():
-    clt = _make_client()
+    cookies_raw = os.getenv("TWS_COOKIES")
+    cookies = parse_cookies(cookies_raw) if cookies_raw else None
+    if not cookies:
+        print("Warning: TWS_COOKIES not set — anonymous fetch will likely fail.", file=sys.stderr)
+
+    clt = _make_client(cookies=cookies)
     try:
         text = await get_tw_page_text("https://x.com/elonmusk", clt)
         soup = bs4.BeautifulSoup(text, "html.parser")