Add cashtag filter option

Fix broken 2.2.x build
Add /<acc>/about tests
2026-06-05 21:19:17 -04:00 · 2026-06-06 00:14:28 +02:00 · 2026-06-05 23:54:39 +02:00 · 2026-06-05 22:26:46 +02:00 · 2026-06-05 22:26:41 +02:00 · 2026-06-05 22:26:38 +02:00
18 changed files with 285 additions and 195 deletions
@@ -33,9 +33,9 @@ jobs:
        uses: actions/cache@v5
        with:
          path: ~/.nimble
-          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
+          key: ${{ matrix.nim }}-nimble-v3-${{ hashFiles('*.nimble') }}
          restore-keys: |
-            ${{ matrix.nim }}-nimble-v2-
+            ${{ matrix.nim }}-nimble-v3-

      - name: Setup Nim
        uses: jiro4989/setup-nim-action@v2
@@ -104,9 +104,9 @@ jobs:
        uses: actions/cache@v5
        with:
          path: ~/.nimble
-          key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
+          key: 2.2.x-nimble-v3-${{ hashFiles('*.nimble') }}
          restore-keys: |
-            2.2.x-nimble-v2-
+            2.2.x-nimble-v3-

      - name: Setup Nim
        uses: jiro4989/setup-nim-action@v2
@@ -18,7 +18,7 @@ requires "nimcrypto#a079df9"
 requires "markdown#158efe3"
 requires "packedjson#9e6fbb6"
 requires "supersnappy#6c94198"
-requires "redpool#8b7c1db"
+requires "https://github.com/zedeus/redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#ca5989a"
 requires "flatty#e668085"
@@ -25,27 +25,27 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
  )

 proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  result = ApiReq(
-    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  )
-  # might change this in the future pending testing
-  result.cookie = result.oauth
+  return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # result = ApiReq(
+  #   cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+  #   oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # )

 proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-  result = ApiReq(
-    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  )
+  return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
+  #let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+  #   oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
+  # )

 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = tweetDetailVars % [id, cursor]
-  result = ApiReq(
-    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
-    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-  )
+  return apiReq(graphTweet, tweetVars % [id, cursor])
+  # let cookieVars = tweetDetailVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+  #   oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  # )

 proc userUrl(username: string): ApiReq =
  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
@@ -184,13 +184,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
  var
    variables = %*{
      "rawQuery": q,
-      "query_source": "typedQuery",
      "count": 20,
+      "querySource": "typed_query",
      "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
    }
+
  if after.len > 0 and maxId.len == 0:
    variables["cursor"] = % after
  let
@@ -212,12 +212,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
  var
    variables = %*{
      "rawQuery": query.text,
-      "query_source": "typedQuery",
      "count": 20,
+      "querySource": "typed_query",
      "product": "People",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
    }
  if after.len > 0:
    variables["cursor"] = % after
@@ -84,12 +84,13 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
    result["x-twitter-auth-type"] = "OAuth2Session"
    result["x-csrf-token"] = session.ct0
    result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    result["referer"] = "https://x.com/"
    result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
    result["sec-ch-ua-mobile"] = "?0"
    result["sec-ch-ua-platform"] = "Windows"
    result["sec-fetch-dest"] = "empty"
    result["sec-fetch-mode"] = "cors"
-    result["sec-fetch-site"] = "same-site"
+    result["sec-fetch-site"] = "same-origin"
    if disableTid or "/1.1/" in url.path:
      result["authorization"] = bearerToken2
    else:
@@ -114,7 +115,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =

  try:
    var resp: AsyncResponse
-    pool.use(await genHeaders(session, url)):
+    let headers = await genHeaders(session, url)
+
+    pool.use(headers):
      template getContent =
        # TODO: this is a temporary simple implementation
        if apiProxy.len > 0 and "/1.1/" notin url.path:
@@ -130,7 +133,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
        raise newException(BadClientError, "Bad client")

      if resp.status == $Http404 and result.len == 0:
-        echo "[sessions] transient 404 (empty body), retrying: ", url.path
+        echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
        raise rateLimitError()

    if resp.headers.hasKey(rlRemaining):
@@ -147,7 +150,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
      if result.startsWith("{\"errors"):
        let errors = result.fromJson(Errors)
        if errors notin errorsToSkip:
-          echo "Fetch error, API: ", url.path, ", errors: ", errors
+          echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
          if errors in {expiredToken, badToken, locked}:
            invalidate(session)
            raise rateLimitError()
@@ -162,7 +165,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
    fetchBody

    if resp.status == $Http400:
-      echo "ERROR 400, ", url.path, ": ", result
+      echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
      raise newException(InternalError, $url)
  except InternalError as e:
    raise e
@@ -177,22 +180,30 @@ template fetchImpl(result, fetchBody) {.dirty.} =
  finally:
    release(session)

-template retry(bod) =
+template retry(bod) {.dirty.} =
+  var session: Session
  for i in 0 ..< maxRetries:
    try:
+      session = nil
      bod
      break
    except RateLimitError:
-      echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
-           " request (", i, "/", maxRetries, ")..."
+      let api = if session.isNil: req.cookie.endpoint
+                else: req.endpoint(session)
+      if session.isNil:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..."
+      else:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..., session: ", session.pretty
+      session = nil
      if retryDelayMs > 0:
        await sleepAsync(retryDelayMs)

 proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
  retry:
-    var 
-      body: string
-      session = await getAndValidateSession(req)
+    var body: string
+    session = await getAndValidateSession(req)

    let url = req.toUrl(session.kind)

@@ -200,22 +211,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
      if body.startsWith('{') or body.startsWith('['):
        result = parseJson(body)
      else:
-        echo resp.status, ": ", body, " --- url: ", url
+        echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
        result = newJNull()

      let error = result.getError
      if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", url.path, ", error: ", error
+        echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
        if error in {expiredToken, badToken, locked}:
          invalidate(session)
          raise rateLimitError()

 proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
  retry:
-    var session = await getAndValidateSession(req)
+    session = await getAndValidateSession(req)
    let url = req.toUrl(session.kind)

    fetchImpl result:
      if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
+        echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
        result.setLen(0)
@@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
 template log(str: varargs[string, `$`]) =
  echo "[sessions] ", str.join("")

-proc endpoint(req: ApiReq; session: Session): string =
+proc endpoint*(req: ApiReq; session: Session): string =
  case session.kind
  of oauth: req.oauth.endpoint
  of cookie: req.cookie.endpoint
@@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
    oldest = now.int64
    newest = 0'i64
    average = 0'i64
+    oauthTotal, cookieTotal = 0
+    oauthLimited, cookieLimited = 0

  for session in sessionPool:
    let created = snowflakeToEpoch(session.id)
@@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
      oldest = created
    average += created

+    case session.kind
+    of oauth: inc oauthTotal
+    of cookie: inc cookieTotal
+
    if session.limited:
      limited.incl session.id
+      case session.kind
+      of oauth: inc oauthLimited
+      of cookie: inc cookieLimited

    for api in session.apis.keys:
      let
@@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
    "sessions": %*{
      "total": sessionPool.len,
      "limited": limited.card,
+      "oauth": %*{"total": oauthTotal, "limited": oauthLimited},
+      "cookie": %*{"total": cookieTotal, "limited": cookieLimited},
      "oldest": $fromUnix(oldest),
      "newest": $fromUnix(newest),
      "average": $fromUnix(average)
@@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =

  for session in sessionPool:
    let sessionJson = %*{
+      "kind": $session.kind,
      "apis": newJObject(),
      "pending": session.pending,
    }
@@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
  if not result.isNil and result.isReady(req):
    inc result.pending
  else:
-    log "no sessions available for API: ", req.cookie.endpoint
+    if result.isNil:
+      log "no sessions available for API: ", req.cookie.endpoint
+    else:
+      log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
    raise noSessionsError()

 proc setLimited*(session: Session; req: ApiReq) =
@@ -7,109 +7,70 @@ const
  bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
  bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"

-  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
-  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
-  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
+  graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
+  graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
+  graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
+  graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
+  graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
+  graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
+  graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
+  graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
+  graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
+  graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
+  graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
+  graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"

-  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
+  graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
+  graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
+  graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
+  graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
+  graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
+
+  graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
  restLiveStream* = "1.1/live_video_stream/status/"

  gqlFeatures* = """{
-  "android_ad_formats_media_component_render_overlay_enabled": false,
-  "android_graphql_skip_api_media_color_palette": false,
-  "android_professional_link_spotlight_display_enabled": false,
-  "articles_api_enabled": false,
-  "articles_preview_enabled": true,
-  "blue_business_profile_image_shape_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": true,
-  "commerce_android_shop_module_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
-  "creator_subscriptions_quote_tweet_preview_enabled": false,
-  "creator_subscriptions_subscription_count_enabled": false,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "grok_android_analyze_trend_fetch_enabled": false,
-  "grok_translations_community_note_auto_translation_is_enabled": false,
-  "grok_translations_community_note_translation_is_enabled": false,
-  "grok_translations_post_auto_translation_is_enabled": false,
-  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
-  "hidden_profile_likes_enabled": false,
-  "highlights_tweets_tab_ui_enabled": false,
-  "immersive_video_status_linkable_timestamps": false,
-  "interactive_text_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "mobile_app_spotlight_module_enabled": false,
-  "payments_enabled": false,
-  "post_ctas_fetch_enabled": true,
-  "premium_content_api_read_enabled": false,
+  "rweb_video_screen_enabled": false,
+  "rweb_cashtags_enabled": true,
  "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "profile_label_improvements_pcf_label_in_profile_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "responsive_web_enhance_cards_enabled": false,
-  "responsive_web_graphql_exclude_directive_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "rweb_tipjar_consumption_enabled": false,
+  "verified_phone_label_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
  "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_grok_analysis_button_from_backend": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "c9s_tweet_anatomy_moderator_badge_enabled": true,
  "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
  "responsive_web_grok_analyze_post_followups_enabled": true,
+  "rweb_cashtags_composer_attachment_enabled": true,
+  "responsive_web_jetfuel_frame": true,
+  "responsive_web_grok_share_attachment_enabled": true,
  "responsive_web_grok_annotations_enabled": true,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "articles_preview_enabled": true,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "rweb_conversational_replies_downvote_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "view_counts_everywhere_api_enabled": true,
+  "longform_notetweets_consumption_enabled": true,
+  "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "content_disclosure_indicator_enabled": true,
+  "content_disclosure_ai_generated_indicator_enabled": true,
+  "responsive_web_grok_show_grok_translated_post": true,
+  "responsive_web_grok_analysis_button_from_backend": true,
+  "post_ctas_fetch_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
+  "standardized_nudges_misinfo": true,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
  "responsive_web_grok_image_annotation_enabled": true,
  "responsive_web_grok_imagine_annotation_enabled": true,
-  "responsive_web_grok_share_attachment_enabled": true,
-  "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_jetfuel_frame": true,
-  "responsive_web_media_download_video_enabled": false,
-  "responsive_web_profile_redirect_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_twitter_article_notes_tab_enabled": false,
-  "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-  "rweb_lists_timeline_redesign_enabled": true,
-  "rweb_tipjar_consumption_enabled": true,
-  "rweb_video_screen_enabled": false,
-  "rweb_video_timestamps_enabled": false,
-  "spaces_2022_h2_clipping": true,
-  "spaces_2022_h2_spaces_communities": true,
-  "standardized_nudges_misinfo": true,
-  "subscriptions_feature_can_gift_premium": false,
-  "subscriptions_verification_info_enabled": true,
-  "subscriptions_verification_info_is_identity_verified_enabled": false,
-  "subscriptions_verification_info_reason_enabled": true,
-  "subscriptions_verification_info_verified_since_enabled": true,
-  "super_follow_badge_privacy_enabled": false,
-  "super_follow_exclusive_tweet_notifications_enabled": false,
-  "super_follow_tweet_api_enabled": false,
-  "super_follow_user_api_enabled": false,
-  "tweet_awards_web_tipping_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "tweetypie_unmention_optimization_enabled": false,
-  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-  "unified_cards_destination_url_params_enabled": false,
-  "verified_phone_label_enabled": false,
-  "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": true,
-  "hidden_profile_subscriptions_enabled": false
+  "responsive_web_grok_community_note_auto_translation_is_enabled": true,
+  "responsive_web_enhance_cards_enabled": false
 }""".replace(" ", "").replace("\n", "")

  tweetVars* = """{
@@ -4,7 +4,7 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard

-proc parseGraphTweet(js: JsonNode): Tweet
+proc parseGraphTweet*(js: JsonNode): Tweet

 proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
  try: parseEnum[VerifiedType](s)
@@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
 proc parseGraphUser(js: JsonNode): User =
  var user = js{"user_result", "result"}
  if user.isNull:
-    user = ? js{"user_results", "result"}
+    user = js{"user_results", "result"}

  if user.isNull:
-    if js{"core"}.notNull and js{"legacy"}.notNull:
+    if js{"core"}.notNull:
      user = js
    else:
      return
@@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User =

  # fallback to support UserMedia/recent GraphQL updates
  if result.username.len == 0:
+    result.id = user{"rest_id"}.getStr
    result.username = user{"core", "screen_name"}.getStr
    result.fullname = user{"core", "name"}.getStr
    result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@@ -261,6 +262,20 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
            durationMs: mediaInfo{"duration_millis"}.getInt,
            variants: parseVideoVariants(mediaInfo{"variants"})
          ))
+
+          # Parse source user for video attribution
+          with sourceUser, mediaEntity{"source_user_results", "result"}:
+            if result.attribution.isNone:
+              let
+                expanded = mediaEntity{"expanded_url"}.getStr
+                pathStart = expanded.find('/', expanded.find("://") + 3)
+              if pathStart >= 0:
+                result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "")
+              result.attribution = some(User(
+                id: sourceUser{"rest_id"}.getStr,
+                fullname: sourceUser{"core", "name"}.getStr,
+                userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+              ))
        of "ApiGif":
          parsedMedia.addMedia(Gif(
            url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@@ -428,17 +443,17 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
  # graphql
  with rt, js{"retweeted_status_result", "result"}:
    # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt:
+    if "legacy" in rt or "rest_id" in rt:
      result.retweet = some parseGraphTweet(rt)
      return

  with reposts, js{"repostedStatusResults"}:
    with rt, reposts{"result"}:
-      if "legacy" in rt:
+      if "legacy" in rt or "rest_id" in rt:
        result.retweet = some parseGraphTweet(rt)
        return

-  if jsCard.kind != JNull and jsCard{"rest_id"}.getStr.allCharsInSet({'0'..'9'}):
+  if jsCard.kind != JNull:
    let name = jsCard{"name"}.getStr
    if "poll" in name:
      if "image" in name:
@@ -449,7 +464,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
      result.poll = some parsePoll(jsCard)
    elif name == "amplify":
      result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-    else:
+    elif name.len > 0 and jsCard{"binding_values"}.notNull:
      result.card = some parseCard(jsCard, js{"entities", "urls"})

  result.expandTweetEntities(js)
@@ -469,7 +484,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
      result.text.removeSuffix(" Learn more.")
      result.available = false

-proc parseGraphTweet(js: JsonNode): Tweet =
+proc parseGraphTweet*(js: JsonNode): Tweet =
  if js.kind == JNull:
    return Tweet()

@@ -498,17 +513,13 @@ proc parseGraphTweet(js: JsonNode): Tweet =
      if bindingArray.kind == JArray:
        var bindingObj: seq[(string, JsonNode)]
        for item in bindingArray:
-          if item{"key"}.getStr == "_omit_link_":
-            jsCard = newJNull()
-            break
          bindingObj.add((item{"key"}.getStr, item{"value"}))
-        if jsCard.kind != JNull:
-          # Create a new card object with flattened structure
-          jsCard = %*{
-            "name": legacyCard{"name"},
-            "url": legacyCard{"url"},
-            "binding_values": %bindingObj
-          }
+        # Create a new card object with flattened structure
+        jsCard = %*{
+          "name": legacyCard{"name"},
+          "url": legacyCard{"url"},
+          "binding_values": %bindingObj
+        }

  var replyId = 0
  with restId, js{"reply_to_results", "rest_id"}:
@@ -530,7 +541,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
      )
    )

-    if jsCard.kind != JNull and jsCard{"rest_id"}.getStr.allCharsInSet({'0'..'9'}):
+    if jsCard.kind != JNull:
      let name = jsCard{"name"}.getStr
      if "poll" in name:
        if "image" in name:
@@ -541,10 +552,21 @@ proc parseGraphTweet(js: JsonNode): Tweet =
        result.poll = some parsePoll(jsCard)
      elif name == "amplify":
        result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-      else:
+      elif name.len > 0 and jsCard{"binding_values"}.notNull:
        result.card = some parseCard(jsCard, js{"url_entities"})

    result.expandTweetEntitiesV2(js)
+
+    # Strip video source URL from text (for videos from other tweets)
+    with mediaEntities, js{"media_entities"}:
+      for m in mediaEntities:
+        if "source_status_id_str" in m:
+          let mediaUrl = m{"url"}.getStr
+          if mediaUrl.len > 0:
+            let idx = result.text.rfind(mediaUrl)
+            if idx >= 0:
+              result.text = result.text[0 ..< idx].strip()
+            break
  else:
    result = parseTweet(js{"legacy"}, jsCard, replyId)
    result.id = js{"rest_id"}.getId
@@ -563,6 +585,12 @@ proc parseGraphTweet(js: JsonNode): Tweet =

  parseMediaEntities(js, result)

+  # Handle retweets - check both legacy and top-level paths
+  with reposts, js{"legacy", "repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt or "rest_id" in rt:
+        result.retweet = some parseGraphTweet(rt)
+
  with quoted, js{"quoted_status_result", "result"}:
    result.quote = some(parseGraphTweet(quoted))

@@ -8,7 +8,7 @@ const
    "media", "images", "twimg", "videos",
    "native_video", "consumer_video", "spaces",
    "links", "news", "quote", "mentions",
-    "replies", "retweets", "nativeretweets"
+    "replies", "retweets", "nativeretweets", "cashtags"
  ]

  emptyQuery* = "include:nativeretweets"
@@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) =
      redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash"))

    get "/opensearch":
-      let url = getUrlPrefix(cfg) & "/search?f=tweets&q="
-      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
+      let
+        url = getUrlPrefix(cfg) & "/search?f=tweets&q="
+        headers = {"Content-Type": "application/opensearchdescription+xml"}
+      resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url)
@@ -264,6 +264,7 @@ type
    stats*: TweetStats
    retweet*: Option[Tweet]
    attribution*: Option[User]
+    attributionLink*: string
    mediaTags*: seq[User]
    quote*: Option[Tweet]
    card*: Option[Card]
@@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
        text cfg.title

    meta(name="viewport", content="width=device-width, initial-scale=1.0")
+    meta(name="referrer", content="same-origin")
    meta(name="theme-color", content="#1F1F1F")
    meta(property="og:type", content=ogType)
    meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))
@@ -15,7 +15,8 @@ const toggles = {
  "links": "Links",
  "images": "Images",
  "quote": "Quotes",
-  "spaces": "Spaces"
+  "spaces": "Spaces",
+  "cashtags": "Cashtags"
 }.toOrderedTable

 proc renderSearch*(): VNode =
@@ -239,8 +239,9 @@ proc renderReply(tweet: Tweet): VNode =
      if i > 0: text " "
      a(href=("/" & u)): text "@" & u

-proc renderAttribution(user: User; prefs: Prefs): VNode =
-  buildHtml(a(class="attribution", href=("/" & user.username))):
+proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode =
+  let href = if link.len > 0: link else: "/" & user.username
+  buildHtml(a(class="attribution", href=href)):
    renderMiniAvatar(user, prefs)
    strong: text user.fullname
    verifiedIcon(user)
@@ -386,7 +387,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
        verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)

      if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get(), prefs)
+        renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink)

      if tweet.card.isSome and tweet.card.get().kind != hidden:
        renderCard(tweet.card.get(), prefs, path)
@@ -0,0 +1,76 @@
+from base import BaseTestCase, Profile
+from parameterized import parameterized
+
+
+class AboutAccount(object):
+    header = '.about-account-header'
+    name = '.about-account-name'
+    body = '.about-account-body'
+    row = '.about-account-row'
+    label = '.about-account-label'
+    value = '.about-account-value'
+
+
+# (username, expected_labels)
+# Each label is checked for presence in the page text
+about_data = [
+    ['jack', ['Date joined', 'Account based in', 'Connected via']],
+    ['NASA', ['Date joined']],
+    ['elonmusk', ['Date joined']],
+]
+
+about_verified = [
+    ['jack', 'Verified', 'Since '],
+]
+
+about_affiliate = [
+    ['jack', 'An affiliate of', 'Square'],
+    ['elonmusk', 'An affiliate of', 'X'],
+]
+
+
+class AboutAccountTest(BaseTestCase):
+    @parameterized.expand(about_data)
+    def test_about_page_has_labels(self, username, expected_labels):
+        """About page shows expected info labels"""
+        self.open_nitter(f'{username}/about')
+        self.assert_element_visible(AboutAccount.header)
+        self.assert_element_visible(AboutAccount.body)
+        for label in expected_labels:
+            self.assert_text(label, AboutAccount.body)
+
+    @parameterized.expand(about_verified)
+    def test_about_verified(self, username, label, value_prefix):
+        """About page shows verification info for verified accounts"""
+        self.open_nitter(f'{username}/about')
+        self.assert_text(label, AboutAccount.body)
+        self.assert_text(value_prefix, AboutAccount.body)
+
+    @parameterized.expand(about_affiliate)
+    def test_about_affiliate(self, username, label, affiliate):
+        """About page shows affiliate info"""
+        self.open_nitter(f'{username}/about')
+        self.assert_text(label, AboutAccount.body)
+        self.assert_text(f'@{affiliate}', AboutAccount.body)
+
+    def test_about_page_title(self):
+        """Title contains account name"""
+        self.open_nitter('jack/about')
+        self.assert_text('jack', AboutAccount.name)
+
+    def test_about_join_date(self):
+        """About page always shows join date"""
+        self.open_nitter('jack/about')
+        self.assert_text('Date joined', AboutAccount.body)
+        self.assert_text('March 2006', AboutAccount.body)
+
+    def test_about_invalid_user(self):
+        """About page for non-existent user shows error"""
+        self.open_nitter('thisprofiledoesntexist/about')
+        self.assert_text('User "thisprofiledoesntexist" not found')
+
+    def test_joindate_links_to_about(self):
+        """Join date on profile page links to about page"""
+        self.open_nitter('jack')
+        link = self.find_element(Profile.joinDate + ' a')
+        self.assertIn('/jack/about', link.get_attribute('href'))
@@ -11,18 +11,18 @@ card = [
    ['voidtarget/status/1094632512926605312',
     'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
     'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True]
+     'gist.github.com', True],
+
+    ['NASA/status/2061872347477418301',
+     'Nancy Grace Roman Space Telescope Mission - NASA Science',
+     'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.',
+     'science.nasa.gov', True]
 ]

 no_thumb = [
-    ['FluentAI/status/1116417904831029248',
-     'LinkedIn',
-     'This link will take you to a page that’s not on LinkedIn',
-     'lnkd.in'],
-
    ['Thom_Wolf/status/1122466524860702729',
-     'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
-     '',
+     'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model',
+     'PyTorch original implementation of Cross-lingual Language Model Pretraining.',
     'github.com'],

    ['brent_p/status/1088857328680488961',
@@ -37,14 +37,9 @@ no_thumb = [
 ]

 playable = [
-    ['nim_lang/status/1118234460904919042',
-     'Nim development blog 2019-03',
-     'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...',
-     'youtube.com'],
-
-    ['nim_lang/status/1121090879823986688',
-     'Nim - First natively compiled language w/ hot code-reloading at...',
-     '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...',
+    ['NASA/status/2047048645845897398',
+     'NASA\'s Artemis II News Conference with Moon Astronauts',
+     'Live from NASA\'s Johnson Space Center in Houston',
     'youtube.com']
 ]

@@ -72,7 +67,7 @@ class CardTest(BaseTestCase):
        if len(description) > 0:
            self.assert_text(description, c.description)

-    @parameterized.expand(playable)
+    @parameterized.expand(playable, skip_on_empty=True)
    def test_card_playable(self, tweet, title, description, destination):
        self.open_nitter(tweet)
        c = Card(Conversation.main + " ")
@@ -8,7 +8,7 @@ thread = [
        [],
        "Based",
        ["Crystal", "Julia"],
-        [["yeah,"]],
+        [["For", "Then"], ["yeah,"]],
    ],
    ["octonion/status/975254452625002496", ["Based"], "Crystal", ["Julia"], []],
    ["octonion/status/975256058384887808", ["Based", "Crystal"], "Julia", [], []],
@@ -71,8 +71,8 @@ emoji = [
 ]

 retweet = [
-    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
+    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test',
+     'Testing. 1234.']
 ]


@@ -120,7 +120,7 @@ class TweetTest(BaseTestCase):
            link = self.find_link_text(f'@{un}')
            self.assertIn(f'/{un}', link.get_property('href'))

-    @parameterized.expand(retweet)
+    @parameterized.expand(retweet, skip_on_empty=True)
    def test_retweet(self, index, url, retweet_by, fullname, username, text):
        self.open_nitter(url)
        tweet = get_timeline_tweet(index)
@@ -28,14 +28,14 @@ video_m3u8 = [
 ]

 gallery = [
-    # ['mobile_test/status/451108446603980803', [
-    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
-    # ]],
+    ['mobile_test/status/451108446603980803', [
+        ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+    ]],

-    # ['mobile_test/status/471539824713691137', [
-    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
-    #     ['Bos--IqIQAAav23']
-    # ]],
+    ['mobile_test/status/471539824713691137', [
+        ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+        ['Bos--IqIQAAav23']
+    ]],

    ['mobile_test/status/469530783384743936', [
        ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],
Author	SHA1	Message	Date
Salastil	3562b0708d	Add cashtag filter option Docker / tests (push) Has been cancelled Details Docker / build-docker-amd64 (push) Has been cancelled Details Docker / build-docker-arm64 (push) Has been cancelled Details	2026-06-05 21:19:17 -04:00
Zed	a86be15f85	Fix broken 2.2.x build	2026-06-06 00:14:28 +02:00
Zed	c956f7c373	Add /<acc>/about tests	2026-06-05 23:54:39 +02:00
Zed	e4e6dd13e6	Update API endpoints	2026-06-05 22:26:46 +02:00
Zed	083d65a8cf	Update tests	2026-06-05 22:26:41 +02:00
Zed	1d57f1f432	Add video attribution link support	2026-06-05 22:26:38 +02:00
Zed	d5ff410c5d	Add same-origin referrer policy Fixes #1346	2026-06-02 23:59:00 +02:00
Ian Brown	5a4faa0367	Fix OpenSearch response crash (#1400 )	2026-06-02 22:31:40 +02:00
Zed	82099de55b	Include session.kind in all debug output Fixes #1330	2026-06-02 17:21:20 +02:00