Regex fixes, nre is not compiling

.github/workflows/run-tests.yml

@@ -33,9 +33,9 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ~/.nimble
-          key: ${{ matrix.nim }}-nimble-v3-${{ hashFiles('*.nimble') }}
+          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            ${{ matrix.nim }}-nimble-v3-
+            ${{ matrix.nim }}-nimble-v2-
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2
@@ -104,9 +104,9 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ~/.nimble
-          key: 2.2.x-nimble-v3-${{ hashFiles('*.nimble') }}
+          key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            2.2.x-nimble-v3-
+            2.2.x-nimble-v2-
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2

nitter.nimble

@@ -18,7 +18,7 @@ requires "nimcrypto#a079df9"
 requires "markdown#158efe3"
 requires "packedjson#9e6fbb6"
 requires "supersnappy#6c94198"
-requires "https://github.com/zedeus/redpool#8b7c1db"
+requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#ca5989a"
 requires "flatty#e668085"

src/api.nim

@@ -25,27 +25,27 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
   )
 
 proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  # result = ApiReq(
-  #   cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-  #   oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  # )
+  result = ApiReq(
+    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  )
+  # might change this in the future pending testing
+  result.cookie = result.oauth
 
 proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  #let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-  # result = ApiReq(
-  #   cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-  #   oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  # )
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
+  )
 
 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  return apiReq(graphTweet, tweetVars % [id, cursor])
-  # let cookieVars = tweetDetailVars % [id, cursor]
-  # result = ApiReq(
-  #   cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-  #   oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-  # )
+  let cookieVars = tweetDetailVars % [id, cursor]
+  result = ApiReq(
+    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  )
 
 proc userUrl(username: string): ApiReq =
   let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
@@ -184,13 +184,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
+      "query_source": "typedQuery",
       "count": 20,
-      "querySource": "typed_query",
       "product": "Latest",
-      "withGrokTranslatedBio":true,
-      "withQuickPromoteEligibilityTweetFields":false
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
     }
-
   if after.len > 0 and maxId.len == 0:
     variables["cursor"] = % after
   let
@@ -212,11 +212,12 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
+      "query_source": "typedQuery",
       "count": 20,
-      "querySource": "typed_query",
       "product": "People",
-      "withGrokTranslatedBio":true,
-      "withQuickPromoteEligibilityTweetFields":false
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
     }
   if after.len > 0:
     variables["cursor"] = % after

src/apiutils.nim

@@ -84,13 +84,12 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
-    result["referer"] = "https://x.com/"
     result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
     result["sec-ch-ua-mobile"] = "?0"
     result["sec-ch-ua-platform"] = "Windows"
     result["sec-fetch-dest"] = "empty"
     result["sec-fetch-mode"] = "cors"
-    result["sec-fetch-site"] = "same-origin"
+    result["sec-fetch-site"] = "same-site"
     if disableTid or "/1.1/" in url.path:
       result["authorization"] = bearerToken2
     else:
@@ -115,9 +114,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    let headers = await genHeaders(session, url)
-
-    pool.use(headers):
+    pool.use(await genHeaders(session, url)):
       template getContent =
         # TODO: this is a temporary simple implementation
         if apiProxy.len > 0 and "/1.1/" notin url.path:
@@ -133,7 +130,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         raise newException(BadClientError, "Bad client")
 
       if resp.status == $Http404 and result.len == 0:
-        echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
+        echo "[sessions] transient 404 (empty body), retrying: ", url.path
         raise rateLimitError()
 
     if resp.headers.hasKey(rlRemaining):
@@ -150,7 +147,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
+          echo "Fetch error, API: ", url.path, ", errors: ", errors
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
@@ -165,7 +162,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     fetchBody
 
     if resp.status == $Http400:
-      echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
+      echo "ERROR 400, ", url.path, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -180,30 +177,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   finally:
     release(session)
 
-template retry(bod) {.dirty.} =
-  var session: Session
+template retry(bod) =
   for i in 0 ..< maxRetries:
     try:
-      session = nil
       bod
       break
     except RateLimitError:
-      let api = if session.isNil: req.cookie.endpoint
-                else: req.endpoint(session)
-      if session.isNil:
-        echo "[sessions] Rate limited, retrying ", api,
-             " request (", i, "/", maxRetries, ")..."
-      else:
-        echo "[sessions] Rate limited, retrying ", api,
-             " request (", i, "/", maxRetries, ")..., session: ", session.pretty
-      session = nil
+      echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
+           " request (", i, "/", maxRetries, ")..."
       if retryDelayMs > 0:
         await sleepAsync(retryDelayMs)
 
 proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
-    var body: string
-    session = await getAndValidateSession(req)
+    var 
+      body: string
+      session = await getAndValidateSession(req)
 
     let url = req.toUrl(session.kind)
 
@@ -211,22 +200,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
-        echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
+        echo resp.status, ": ", body, " --- url: ", url
         result = newJNull()
 
       let error = result.getError
       if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
+        echo "Fetch error, API: ", url.path, ", error: ", error
         if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
 proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    session = await getAndValidateSession(req)
+    var session = await getAndValidateSession(req)
     let url = req.toUrl(session.kind)
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
+        echo resp.status, ": ", result, " --- url: ", url
         result.setLen(0)

src/auth.nim

@@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
 template log(str: varargs[string, `$`]) =
   echo "[sessions] ", str.join("")
 
-proc endpoint*(req: ApiReq; session: Session): string =
+proc endpoint(req: ApiReq; session: Session): string =
   case session.kind
   of oauth: req.oauth.endpoint
   of cookie: req.cookie.endpoint
@@ -50,8 +50,6 @@ proc getSessionPoolHealth*(): JsonNode =
     oldest = now.int64
     newest = 0'i64
     average = 0'i64
-    oauthTotal, cookieTotal = 0
-    oauthLimited, cookieLimited = 0
 
   for session in sessionPool:
     let created = snowflakeToEpoch(session.id)
@@ -61,15 +59,8 @@ proc getSessionPoolHealth*(): JsonNode =
       oldest = created
     average += created
 
-    case session.kind
-    of oauth: inc oauthTotal
-    of cookie: inc cookieTotal
-
     if session.limited:
       limited.incl session.id
-      case session.kind
-      of oauth: inc oauthLimited
-      of cookie: inc cookieLimited
 
     for api in session.apis.keys:
       let
@@ -93,8 +84,6 @@ proc getSessionPoolHealth*(): JsonNode =
     "sessions": %*{
       "total": sessionPool.len,
       "limited": limited.card,
-      "oauth": %*{"total": oauthTotal, "limited": oauthLimited},
-      "cookie": %*{"total": cookieTotal, "limited": cookieLimited},
       "oldest": $fromUnix(oldest),
       "newest": $fromUnix(newest),
       "average": $fromUnix(average)
@@ -111,7 +100,6 @@ proc getSessionPoolDebug*(): JsonNode =
 
   for session in sessionPool:
     let sessionJson = %*{
-      "kind": $session.kind,
       "apis": newJObject(),
       "pending": session.pending,
     }
@@ -185,10 +173,7 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
   if not result.isNil and result.isReady(req):
     inc result.pending
   else:
-    if result.isNil:
-      log "no sessions available for API: ", req.cookie.endpoint
-    else:
-      log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
+    log "no sessions available for API: ", req.cookie.endpoint
     raise noSessionsError()
 
 proc setLimited*(session: Session; req: ApiReq) =

src/consts.nim

@@ -7,70 +7,109 @@ const
   bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
   bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
-  graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
-  graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
-  graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
-  graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
-  graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
-  graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
-  graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
-  graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
-  graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
-  graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
-  graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"
+  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
+  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
+  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
+  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
+  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
+  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
+  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
+  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
+  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
+  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
+  graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
+  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
+  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
+  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
+  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
+  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
+  graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
 
-  graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
-  graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
-  graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
-  graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
-  graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
-
-  graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
+  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
   restLiveStream* = "1.1/live_video_stream/status/"
 
   gqlFeatures* = """{
-  "rweb_video_screen_enabled": false,
-  "rweb_cashtags_enabled": true,
-  "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "responsive_web_profile_redirect_enabled": false,
-  "rweb_tipjar_consumption_enabled": false,
-  "verified_phone_label_enabled": false,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-  "premium_content_api_read_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
+  "android_ad_formats_media_component_render_overlay_enabled": false,
+  "android_graphql_skip_api_media_color_palette": false,
+  "android_professional_link_spotlight_display_enabled": false,
+  "articles_api_enabled": false,
+  "articles_preview_enabled": true,
+  "blue_business_profile_image_shape_enabled": false,
   "c9s_tweet_anatomy_moderator_badge_enabled": true,
+  "commerce_android_shop_module_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "creator_subscriptions_quote_tweet_preview_enabled": false,
+  "creator_subscriptions_subscription_count_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "grok_android_analyze_trend_fetch_enabled": false,
+  "grok_translations_community_note_auto_translation_is_enabled": false,
+  "grok_translations_community_note_translation_is_enabled": false,
+  "grok_translations_post_auto_translation_is_enabled": false,
+  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
+  "hidden_profile_likes_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "immersive_video_status_linkable_timestamps": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_inline_media_enabled": true,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": true,
+  "mobile_app_spotlight_module_enabled": false,
+  "payments_enabled": false,
+  "post_ctas_fetch_enabled": true,
+  "premium_content_api_read_enabled": false,
+  "profile_label_improvements_pcf_label_in_post_enabled": true,
+  "profile_label_improvements_pcf_label_in_profile_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": true,
+  "responsive_web_grok_analysis_button_from_backend": true,
   "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
   "responsive_web_grok_analyze_post_followups_enabled": true,
-  "rweb_cashtags_composer_attachment_enabled": true,
-  "responsive_web_jetfuel_frame": true,
-  "responsive_web_grok_share_attachment_enabled": true,
   "responsive_web_grok_annotations_enabled": true,
-  "articles_preview_enabled": true,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "rweb_conversational_replies_downvote_enabled": false,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "view_counts_everywhere_api_enabled": true,
-  "longform_notetweets_consumption_enabled": true,
-  "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "content_disclosure_indicator_enabled": true,
-  "content_disclosure_ai_generated_indicator_enabled": true,
-  "responsive_web_grok_show_grok_translated_post": true,
-  "responsive_web_grok_analysis_button_from_backend": true,
-  "post_ctas_fetch_enabled": true,
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "standardized_nudges_misinfo": true,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "longform_notetweets_inline_media_enabled": false,
+  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
   "responsive_web_grok_image_annotation_enabled": true,
   "responsive_web_grok_imagine_annotation_enabled": true,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": true,
-  "responsive_web_enhance_cards_enabled": false
+  "responsive_web_grok_share_attachment_enabled": true,
+  "responsive_web_grok_show_grok_translated_post": false,
+  "responsive_web_jetfuel_frame": true,
+  "responsive_web_media_download_video_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "rweb_lists_timeline_redesign_enabled": true,
+  "rweb_tipjar_consumption_enabled": true,
+  "rweb_video_screen_enabled": false,
+  "rweb_video_timestamps_enabled": false,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": true,
+  "subscriptions_feature_can_gift_premium": false,
+  "subscriptions_verification_info_enabled": true,
+  "subscriptions_verification_info_is_identity_verified_enabled": false,
+  "subscriptions_verification_info_reason_enabled": true,
+  "subscriptions_verification_info_verified_since_enabled": true,
+  "super_follow_badge_privacy_enabled": false,
+  "super_follow_exclusive_tweet_notifications_enabled": false,
+  "super_follow_tweet_api_enabled": false,
+  "super_follow_user_api_enabled": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+  "tweetypie_unmention_optimization_enabled": false,
+  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
+  "unified_cards_destination_url_params_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": true,
+  "hidden_profile_subscriptions_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVars* = """{

src/experimental/parser/user.nim

@@ -1,4 +1,4 @@
-import std/[algorithm, unicode, re, strutils, strformat, options, nre]
+import std/[algorithm, unicode, re, strutils, strformat, options]
 import jsony
 import utils, slices
 import ../types/user as userType
@@ -8,7 +8,7 @@ let
   unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = nre.re"""(*U)(^|[^\w-_.?])([#＃$])([\w_]*+)(?!</a>|">|#)"""
+  htRegex = re.re"(^|[^a-zA-Z0-9_-_.?])([#＃$])([a-zA-Z0-9_]+)"
   htReplace = "$1<a href=\"/search?f=tweets&q=%23$3\">$2$3</a>"
 
 proc expandUserEntities(user: var User; raw: RawUser) =
@@ -29,7 +29,7 @@ proc expandUserEntities(user: var User; raw: RawUser) =
 
   user.bio = orig.replacedWith(replacements, 0 .. orig.len)
                  .replacef(unRegex, unReplace)
-                 .replace(htRegex, htReplace)
+                 .replacef(htRegex, htReplace)
 
 proc getBanner(user: RawUser): string =
   if user.profileBannerUrl.len > 0:

src/parser.nim

@@ -4,7 +4,7 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet*(js: JsonNode): Tweet
+proc parseGraphTweet(js: JsonNode): Tweet
 
 proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
   try: parseEnum[VerifiedType](s)
@@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
 proc parseGraphUser(js: JsonNode): User =
   var user = js{"user_result", "result"}
   if user.isNull:
-    user = js{"user_results", "result"}
+    user = ? js{"user_results", "result"}
 
   if user.isNull:
-    if js{"core"}.notNull:
+    if js{"core"}.notNull and js{"legacy"}.notNull:
       user = js
     else:
       return
@@ -61,7 +61,6 @@ proc parseGraphUser(js: JsonNode): User =
 
   # fallback to support UserMedia/recent GraphQL updates
   if result.username.len == 0:
-    result.id = user{"rest_id"}.getStr
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@@ -262,20 +261,6 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
             durationMs: mediaInfo{"duration_millis"}.getInt,
             variants: parseVideoVariants(mediaInfo{"variants"})
           ))
-
-          # Parse source user for video attribution
-          with sourceUser, mediaEntity{"source_user_results", "result"}:
-            if result.attribution.isNone:
-              let
-                expanded = mediaEntity{"expanded_url"}.getStr
-                pathStart = expanded.find('/', expanded.find("://") + 3)
-              if pathStart >= 0:
-                result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "")
-              result.attribution = some(User(
-                id: sourceUser{"rest_id"}.getStr,
-                fullname: sourceUser{"core", "name"}.getStr,
-                userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
-              ))
         of "ApiGif":
           parsedMedia.addMedia(Gif(
             url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@@ -443,13 +428,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
   # graphql
   with rt, js{"retweeted_status_result", "result"}:
     # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt or "rest_id" in rt:
+    if "legacy" in rt:
       result.retweet = some parseGraphTweet(rt)
       return
 
   with reposts, js{"repostedStatusResults"}:
     with rt, reposts{"result"}:
-      if "legacy" in rt or "rest_id" in rt:
+      if "legacy" in rt:
         result.retweet = some parseGraphTweet(rt)
         return
 
@@ -464,7 +449,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.poll = some parsePoll(jsCard)
     elif name == "amplify":
       result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-    elif name.len > 0 and jsCard{"binding_values"}.notNull:
+    else:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
@@ -484,7 +469,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet*(js: JsonNode): Tweet =
+proc parseGraphTweet(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
@@ -552,21 +537,10 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
         result.poll = some parsePoll(jsCard)
       elif name == "amplify":
         result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-      elif name.len > 0 and jsCard{"binding_values"}.notNull:
+      else:
         result.card = some parseCard(jsCard, js{"url_entities"})
 
     result.expandTweetEntitiesV2(js)
-
-    # Strip video source URL from text (for videos from other tweets)
-    with mediaEntities, js{"media_entities"}:
-      for m in mediaEntities:
-        if "source_status_id_str" in m:
-          let mediaUrl = m{"url"}.getStr
-          if mediaUrl.len > 0:
-            let idx = result.text.rfind(mediaUrl)
-            if idx >= 0:
-              result.text = result.text[0 ..< idx].strip()
-            break
   else:
     result = parseTweet(js{"legacy"}, jsCard, replyId)
     result.id = js{"rest_id"}.getId
@@ -585,12 +559,6 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
 
   parseMediaEntities(js, result)
 
-  # Handle retweets - check both legacy and top-level paths
-  with reposts, js{"legacy", "repostedStatusResults"}:
-    with rt, reposts{"result"}:
-      if "legacy" in rt or "rest_id" in rt:
-        result.retweet = some parseGraphTweet(rt)
-
   with quoted, js{"quoted_status_result", "result"}:
     result.quote = some(parseGraphTweet(quoted))
 

src/parserutils.nim

@@ -16,7 +16,7 @@ let
   unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = re"(^|[^\w-_./?])([#$]|＃)([\w_]+)"
+  htRegex = re"(^|[^a-zA-Z0-9_-_.?])([#$]|＃)([a-zA-Z0-9_]+)"
   htReplace = "$1<a href=\"/search?f=tweets&q=%23$3\">$2$3</a>"
 
 type

src/routes/search.nim

@@ -46,7 +46,6 @@ proc createSearchRouter*(cfg: Config) =
       redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
-      let
-        url = getUrlPrefix(cfg) & "/search?f=tweets&q="
-        headers = {"Content-Type": "application/opensearchdescription+xml"}
-      resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url)
+      let url = getUrlPrefix(cfg) & "/search?f=tweets&q="
+      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/types.nim

@@ -264,7 +264,6 @@ type
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
-    attributionLink*: string
     mediaTags*: seq[User]
     quote*: Option[Tweet]
     card*: Option[Card]

src/views/general.nim

@@ -84,7 +84,6 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
         text cfg.title
 
     meta(name="viewport", content="width=device-width, initial-scale=1.0")
-    meta(name="referrer", content="same-origin")
     meta(name="theme-color", content="#1F1F1F")
     meta(property="og:type", content=ogType)
     meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))

src/views/tweet.nim

@@ -239,9 +239,8 @@ proc renderReply(tweet: Tweet): VNode =
       if i > 0: text " "
       a(href=("/" & u)): text "@" & u
 
-proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode =
-  let href = if link.len > 0: link else: "/" & user.username
-  buildHtml(a(class="attribution", href=href)):
+proc renderAttribution(user: User; prefs: Prefs): VNode =
+  buildHtml(a(class="attribution", href=("/" & user.username))):
     renderMiniAvatar(user, prefs)
     strong: text user.fullname
     verifiedIcon(user)
@@ -387,7 +386,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
 
       if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink)
+        renderAttribution(tweet.attribution.get(), prefs)
 
       if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)

tests/test_about_account.py

@@ -1,76 +0,0 @@
-from base import BaseTestCase, Profile
-from parameterized import parameterized
-
-
-class AboutAccount(object):
-    header = '.about-account-header'
-    name = '.about-account-name'
-    body = '.about-account-body'
-    row = '.about-account-row'
-    label = '.about-account-label'
-    value = '.about-account-value'
-
-
-# (username, expected_labels)
-# Each label is checked for presence in the page text
-about_data = [
-    ['jack', ['Date joined', 'Account based in', 'Connected via']],
-    ['NASA', ['Date joined']],
-    ['elonmusk', ['Date joined']],
-]
-
-about_verified = [
-    ['jack', 'Verified', 'Since '],
-]
-
-about_affiliate = [
-    ['jack', 'An affiliate of', 'Square'],
-    ['elonmusk', 'An affiliate of', 'X'],
-]
-
-
-class AboutAccountTest(BaseTestCase):
-    @parameterized.expand(about_data)
-    def test_about_page_has_labels(self, username, expected_labels):
-        """About page shows expected info labels"""
-        self.open_nitter(f'{username}/about')
-        self.assert_element_visible(AboutAccount.header)
-        self.assert_element_visible(AboutAccount.body)
-        for label in expected_labels:
-            self.assert_text(label, AboutAccount.body)
-
-    @parameterized.expand(about_verified)
-    def test_about_verified(self, username, label, value_prefix):
-        """About page shows verification info for verified accounts"""
-        self.open_nitter(f'{username}/about')
-        self.assert_text(label, AboutAccount.body)
-        self.assert_text(value_prefix, AboutAccount.body)
-
-    @parameterized.expand(about_affiliate)
-    def test_about_affiliate(self, username, label, affiliate):
-        """About page shows affiliate info"""
-        self.open_nitter(f'{username}/about')
-        self.assert_text(label, AboutAccount.body)
-        self.assert_text(f'@{affiliate}', AboutAccount.body)
-
-    def test_about_page_title(self):
-        """Title contains account name"""
-        self.open_nitter('jack/about')
-        self.assert_text('jack', AboutAccount.name)
-
-    def test_about_join_date(self):
-        """About page always shows join date"""
-        self.open_nitter('jack/about')
-        self.assert_text('Date joined', AboutAccount.body)
-        self.assert_text('March 2006', AboutAccount.body)
-
-    def test_about_invalid_user(self):
-        """About page for non-existent user shows error"""
-        self.open_nitter('thisprofiledoesntexist/about')
-        self.assert_text('User "thisprofiledoesntexist" not found')
-
-    def test_joindate_links_to_about(self):
-        """Join date on profile page links to about page"""
-        self.open_nitter('jack')
-        link = self.find_element(Profile.joinDate + ' a')
-        self.assertIn('/jack/about', link.get_attribute('href'))

tests/test_card.py

@@ -11,18 +11,18 @@ card = [
     ['voidtarget/status/1094632512926605312',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True],
-
-    ['NASA/status/2061872347477418301',
-     'Nancy Grace Roman Space Telescope Mission - NASA Science',
-     'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.',
-     'science.nasa.gov', True]
+     'gist.github.com', True]
 ]
 
 no_thumb = [
+    ['FluentAI/status/1116417904831029248',
+     'LinkedIn',
+     'This link will take you to a page that’s not on LinkedIn',
+     'lnkd.in'],
+
     ['Thom_Wolf/status/1122466524860702729',
-     'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model',
-     'PyTorch original implementation of Cross-lingual Language Model Pretraining.',
+     'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
+     '',
      'github.com'],
 
     ['brent_p/status/1088857328680488961',
@@ -37,9 +37,14 @@ no_thumb = [
 ]
 
 playable = [
-    ['NASA/status/2047048645845897398',
-     'NASA\'s Artemis II News Conference with Moon Astronauts',
-     'Live from NASA\'s Johnson Space Center in Houston',
+    ['nim_lang/status/1118234460904919042',
+     'Nim development blog 2019-03',
+     'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...',
+     'youtube.com'],
+
+    ['nim_lang/status/1121090879823986688',
+     'Nim - First natively compiled language w/ hot code-reloading at...',
+     '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...',
      'youtube.com']
 ]
 
@@ -67,7 +72,7 @@ class CardTest(BaseTestCase):
         if len(description) > 0:
             self.assert_text(description, c.description)
 
-    @parameterized.expand(playable, skip_on_empty=True)
+    @parameterized.expand(playable)
     def test_card_playable(self, tweet, title, description, destination):
         self.open_nitter(tweet)
         c = Card(Conversation.main + " ")

tests/test_thread.py

@@ -8,7 +8,7 @@ thread = [
         [],
         "Based",
         ["Crystal", "Julia"],
-        [["For", "Then"], ["yeah,"]],
+        [["yeah,"]],
     ],
     ["octonion/status/975254452625002496", ["Based"], "Crystal", ["Julia"], []],
     ["octonion/status/975256058384887808", ["Based", "Crystal"], "Julia", [], []],

tests/test_tweet.py

@@ -71,8 +71,8 @@ emoji = [
 ]
 
 retweet = [
-    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test',
-     'Testing. 1234.']
+    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
+    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
 ]
 
 
@@ -120,7 +120,7 @@ class TweetTest(BaseTestCase):
             link = self.find_link_text(f'@{un}')
             self.assertIn(f'/{un}', link.get_property('href'))
 
-    @parameterized.expand(retweet, skip_on_empty=True)
+    @parameterized.expand(retweet)
     def test_retweet(self, index, url, retweet_by, fullname, username, text):
         self.open_nitter(url)
         tweet = get_timeline_tweet(index)

tests/test_tweet_media.py

@@ -28,14 +28,14 @@ video_m3u8 = [
 ]
 
 gallery = [
-    ['mobile_test/status/451108446603980803', [
-        ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
-    ]],
+    # ['mobile_test/status/451108446603980803', [
+    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+    # ]],
 
-    ['mobile_test/status/471539824713691137', [
-        ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
-        ['Bos--IqIQAAav23']
-    ]],
+    # ['mobile_test/status/471539824713691137', [
+    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+    #     ['Bos--IqIQAAav23']
+    # ]],
 
     ['mobile_test/status/469530783384743936', [
         ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],