fix: stop pagination on repeated cursors and pages

tests/test_pagination.py

@@ -42,6 +42,24 @@ def make_promo_page(cursor: str | None = "next_cursor"):
     return {"data": {"entries": entries}}
 
 
+def make_content_page(entry_id: str, cursor: str):
+    return {
+        "data": {
+            "entries": [
+                {"entryId": entry_id, "content": {"itemContent": {}}},
+                {
+                    "entryId": f"cursor-{cursor}",
+                    "content": {
+                        "__typename": "TimelineTimelineCursor",
+                        "cursorType": "Bottom",
+                        "value": cursor,
+                    },
+                },
+            ]
+        }
+    }
+
+
 async def _make_api():
     pool = AccountsPool()
     await pool.add_account("u1", "p1", "e1", "ep1")
@@ -121,3 +139,43 @@ async def test_followers_stops_after_too_many_consecutive_empty_pages(monkeypatc
 
     assert len(users) == 0
     assert idx < 10, f"too many requests ({idx}); should have stopped after a few empty pages"
+
+
+async def test_gql_items_stops_on_repeated_cursor(monkeypatch):
+    page = make_content_page("user-1", "same-cursor")
+    calls = 0
+
+    async def mock_get(self, url, params=None):
+        nonlocal calls
+        calls += 1
+        return FakeRep(page)
+
+    monkeypatch.setattr(QueueClient, "get", mock_get)
+    api = await _make_api()
+
+    reps = [x async for x in api._gql_items("hash/Followers", {})]
+
+    assert len(reps) == 1
+    assert calls == 2
+
+
+async def test_search_stops_on_repeated_page_with_new_cursor(monkeypatch):
+    pages = [
+        make_content_page("tweet-1", "cursor-1"),
+        make_content_page("tweet-1", "cursor-2"),
+    ]
+    calls = 0
+
+    async def mock_get(self, url, params=None):
+        nonlocal calls
+        page = pages[min(calls, len(pages) - 1)]
+        calls += 1
+        return FakeRep(page)
+
+    monkeypatch.setattr(QueueClient, "get", mock_get)
+    api = await _make_api()
+
+    reps = [x async for x in api._gql_items("hash/SearchTimeline", {})]
+
+    assert len(reps) == 1
+    assert calls == 2

twscrape/api.py

@@ -142,6 +142,25 @@ class API:
 
         return rep if is_res else None, new_total, is_cur and not is_lim
 
+    def _is_stalled(self, q: str, res: list, cur: str | None, seen: set[tuple[str, ...]]):
+        keys: list[tuple[str, ...]] = [("cursor", cur)] if cur is not None else []
+        if q == "SearchTimeline":
+            entry_ids = tuple(
+                str(x.get("entryId"))
+                for x in res
+                if isinstance(x, dict) and x.get("entryId") is not None
+            )
+            if entry_ids:
+                keys.append(("entries", *entry_ids))
+
+        if any(x in seen for x in keys):
+            return True
+
+        # Not sure whether A → B → A should count as a stall, so only compare the last page.
+        seen.clear()  # Comment this out to detect repeats across all pages.
+        seen.update(keys)
+        return False
+
     def _get_cursor(self, obj: dict, cursor_type="Bottom") -> str | None:
         # standard timeline cursor: {cursorType: "Bottom", value: "..."}
         # fallback: community endpoints use slice_info.next_cursor (plain string)
@@ -171,6 +190,7 @@ class API:
         queue, cur, cnt, active = op.split("/")[-1], None, 0, True
         kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})}
         empty_pages = 0
+        seen: set[tuple[str, ...]] = set()
 
         async with QueueClient(self.pool, queue, self.debug, proxy=self.proxy) as client:
             while active:
@@ -190,6 +210,10 @@ class API:
                 els = self._gql_entries(obj)
                 cur = self._get_cursor(obj, cursor_type)
 
+                if self._is_stalled(queue, els, cur, seen):
+                    logger.warning(f"{queue} pagination stalled, stopping")
+                    return
+
                 rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit)
                 if rep is None:
                     # cursor exists → data may follow after empty/filtered pages (e.g. promo)