diff --git a/tests/test_pagination.py b/tests/test_pagination.py index d41e385..61296df 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -42,6 +42,24 @@ def make_promo_page(cursor: str | None = "next_cursor"): return {"data": {"entries": entries}} +def make_content_page(entry_id: str, cursor: str): + return { + "data": { + "entries": [ + {"entryId": entry_id, "content": {"itemContent": {}}}, + { + "entryId": f"cursor-{cursor}", + "content": { + "__typename": "TimelineTimelineCursor", + "cursorType": "Bottom", + "value": cursor, + }, + }, + ] + } + } + + async def _make_api(): pool = AccountsPool() await pool.add_account("u1", "p1", "e1", "ep1") @@ -121,3 +139,43 @@ async def test_followers_stops_after_too_many_consecutive_empty_pages(monkeypatc assert len(users) == 0 assert idx < 10, f"too many requests ({idx}); should have stopped after a few empty pages" + + +async def test_gql_items_stops_on_repeated_cursor(monkeypatch): + page = make_content_page("user-1", "same-cursor") + calls = 0 + + async def mock_get(self, url, params=None): + nonlocal calls + calls += 1 + return FakeRep(page) + + monkeypatch.setattr(QueueClient, "get", mock_get) + api = await _make_api() + + reps = [x async for x in api._gql_items("hash/Followers", {})] + + assert len(reps) == 1 + assert calls == 2 + + +async def test_search_stops_on_repeated_page_with_new_cursor(monkeypatch): + pages = [ + make_content_page("tweet-1", "cursor-1"), + make_content_page("tweet-1", "cursor-2"), + ] + calls = 0 + + async def mock_get(self, url, params=None): + nonlocal calls + page = pages[min(calls, len(pages) - 1)] + calls += 1 + return FakeRep(page) + + monkeypatch.setattr(QueueClient, "get", mock_get) + api = await _make_api() + + reps = [x async for x in api._gql_items("hash/SearchTimeline", {})] + + assert len(reps) == 1 + assert calls == 2 diff --git a/twscrape/api.py b/twscrape/api.py index 4861f5b..9dc4842 100644 --- a/twscrape/api.py +++ b/twscrape/api.py @@ -142,6 +142,25 @@ class API: return rep if is_res else None, new_total, is_cur and not is_lim + def _is_stalled(self, q: str, res: list, cur: str | None, seen: set[tuple[str, ...]]): + keys: list[tuple[str, ...]] = [("cursor", cur)] if cur is not None else [] + if q == "SearchTimeline": + entry_ids = tuple( + str(x.get("entryId")) + for x in res + if isinstance(x, dict) and x.get("entryId") is not None + ) + if entry_ids: + keys.append(("entries", *entry_ids)) + + if any(x in seen for x in keys): + return True + + # Not sure whether A → B → A should count as a stall, so only compare the last page. + seen.clear() # Comment this out to detect repeats across all pages. + seen.update(keys) + return False + def _get_cursor(self, obj: dict, cursor_type="Bottom") -> str | None: # standard timeline cursor: {cursorType: "Bottom", value: "..."} # fallback: community endpoints use slice_info.next_cursor (plain string) @@ -171,6 +190,7 @@ class API: queue, cur, cnt, active = op.split("/")[-1], None, 0, True kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})} empty_pages = 0 + seen: set[tuple[str, ...]] = set() async with QueueClient(self.pool, queue, self.debug, proxy=self.proxy) as client: while active: @@ -190,6 +210,10 @@ class API: els = self._gql_entries(obj) cur = self._get_cursor(obj, cursor_type) + if self._is_stalled(queue, els, cur, seen): + logger.warning(f"{queue} pagination stalled, stopping") + return + rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit) if rep is None: # cursor exists → data may follow after empty/filtered pages (e.g. promo)