Add user-agent to guest_token request (#1359)

Co-authored-by: yav <796176@protonmail.com>

nitter.example.conf

@@ -26,7 +26,9 @@ enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.sessions)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
+apiProxy = "" # nitter-proxy host, e.g. localhost:7000
 disableTid = false # enable this if cookie-based auth is failing
+maxConcurrentReqs = 2 # max requests at a time per session to avoid race conditions
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/api.nim

@@ -42,7 +42,8 @@ proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
   let cookieVars = tweetDetailVars % [id, cursor]
   result = ApiReq(
-    cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
     oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
   )
 

src/apiutils.nim

@@ -13,10 +13,17 @@ const
 var 
   pool: HttpPool
   disableTid: bool
+  apiProxy: string
 
 proc setDisableTid*(disable: bool) =
   disableTid = disable
 
+proc setApiProxy*(url: string) =
+  if url.len > 0:
+    apiProxy = url.strip(chars={'/'}) & "/"
+    if "http" notin apiProxy:
+      apiProxy = "http://" & apiProxy
+
 proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
   case sessionKind
   of oauth:  
@@ -48,26 +55,31 @@ proc getCookieHeader(authToken, ct0: string): string =
 
 proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
+    "accept": "*/*",
+    "accept-encoding": "gzip",
+    "accept-language": "en-US,en;q=0.9",
     "connection": "keep-alive",
     "content-type": "application/json",
+    "origin": "https://x.com",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
     "x-twitter-active-user": "yes",
     "x-twitter-client-language": "en",
-    "origin": "https://x.com",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "accept": "*/*",
-    "DNT": "1",
-    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+    "priority": "u=1, i"
   })
 
   case session.kind
   of SessionKind.oauth:
-    result["authority"] = "api.x.com"
     result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
   of SessionKind.cookie:
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
+    result["sec-ch-ua-mobile"] = "?0"
+    result["sec-ch-ua-platform"] = "Windows"
+    result["sec-fetch-dest"] = "empty"
+    result["sec-fetch-mode"] = "cors"
+    result["sec-fetch-site"] = "same-site"
     if disableTid:
       result["authorization"] = bearerToken2
     else:
@@ -94,7 +106,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     var resp: AsyncResponse
     pool.use(await genHeaders(session, url)):
       template getContent =
-        resp = await c.get($url)
+        # TODO: this is a temporary simple implementation
+        if apiProxy.len > 0:
+          resp = await c.get(($url).replace("https://", apiProxy))
+        else:
+          resp = await c.get($url)
         result = await resp.body
 
       getContent()

src/auth.nim

@@ -3,14 +3,17 @@ import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os
 import types, consts
 import experimental/parser/session
 
-# max requests at a time per session to avoid race conditions
-const
-  maxConcurrentReqs = 2
-  hourInSeconds = 60 * 60
+const hourInSeconds = 60 * 60
 
 var
   sessionPool: seq[Session]
   enableLogging = false
+  # max requests at a time per session to avoid race conditions
+  maxConcurrentReqs = 2
+
+proc setMaxConcurrentReqs*(reqs: int) =
+  if reqs > 0:
+    maxConcurrentReqs = reqs
 
 template log(str: varargs[string, `$`]) =
   echo "[sessions] ", str.join("")

src/config.nim

@@ -41,7 +41,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
     proxyAuth: cfg.get("Config", "proxyAuth", ""),
-    disableTid: cfg.get("Config", "disableTid", false)
+    apiProxy: cfg.get("Config", "apiProxy", ""),
+    disableTid: cfg.get("Config", "disableTid", false),
+    maxConcurrentReqs: cfg.get("Config", "maxConcurrentReqs", 2)
   )
 
   return (conf, cfg)

src/nitter.nim

@@ -37,7 +37,9 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setApiProxy(cfg.apiProxy)
 setDisableTid(cfg.disableTid)
+setMaxConcurrentReqs(cfg.maxConcurrentReqs)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)

src/routes/media.nim

@@ -52,10 +52,10 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
         ""
 
     let headers = newHttpHeaders({
-      "Content-Type": res.headers["content-type", 0],
-      "Content-Length": contentLength,
-      "Cache-Control": maxAge,
-      "ETag": hashed
+      "content-type": res.headers["content-type", 0],
+      "content-length": contentLength,
+      "cache-control": maxAge,
+      "etag": hashed
     })
 
     respond(request, headers)

src/routes/timeline.nim

@@ -114,6 +114,7 @@ proc createTimelineRouter*(cfg: Config) =
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
+      cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','})
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()

src/types.nim

@@ -275,7 +275,9 @@ type
     enableDebug*: bool
     proxy*: string
     proxyAuth*: string
+    apiProxy*: string
     disableTid*: bool
+    maxConcurrentReqs*: int
 
     rssCacheTime*: int
     listCacheTime*: int

src/views/profile.nim

@@ -26,6 +26,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
 
       tdiv(class="profile-card-tabs-name"):
         linkUser(user, class="profile-card-fullname")
+        verifiedIcon(user)
         linkUser(user, class="profile-card-username")
 
     tdiv(class="profile-card-extra"):

src/views/renderutils.nim

@@ -42,7 +42,6 @@ proc linkUser*(user: User, class=""): VNode =
   buildHtml(a(href=href, class=class, title=nameText)):
     text nameText
     if isName:
-      verifiedIcon(user)
       if user.protected:
         text " "
         icon "lock", title="Protected account"

src/views/rss.nimf

@@ -2,6 +2,9 @@
 ## SPDX-License-Identifier: AGPL-3.0-only
 #import strutils, xmltree, strformat, options, unicode
 #import ../types, ../utils, ../formatters, ../prefs
+## Snowflake ID cutoff for RSS GUID format transition
+## Corresponds to approximately December 14, 2025 UTC
+#const guidCutoff = 2000000000000000000'i64
 #
 #proc getTitle(tweet: Tweet; retweet: string): string =
 #if tweet.pinned: result = "Pinned: "
@@ -101,12 +104,17 @@ ${renderRssTweet(quoteTweet, cfg)}
 #    if link in links: continue
 #    end if
 #    links.add link
+#    let useGlobalGuid = tweet.id >= guidCutoff
       <item>
         <title>${getTitle(tweet, retweet)}</title>
         <dc:creator>@${tweet.user.username}</dc:creator>
         <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
         <pubDate>${getRfc822Time(tweet)}</pubDate>
+#if useGlobalGuid:
+        <guid isPermaLink="false">${tweet.id}</guid>
+#else:
         <guid>${urlPrefix & link}</guid>
+#end if
         <link>${urlPrefix & link}</link>
       </item>
 #  end for

src/views/status.nim

@@ -28,14 +28,19 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
     if thread.hasMore:
       renderMoreReplies(thread)
 
-proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
+proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string; tweet: Tweet = nil): VNode =
   buildHtml(tdiv(class="replies", id="r")):
+    var hasReplies = false
+    var replyCount = 0
     for thread in replies.content:
       if thread.content.len == 0: continue
+      hasReplies = true
+      replyCount += thread.content.len
       renderReplyThread(thread, prefs, path)
 
-    if replies.bottom.len > 0:
-      renderMore(Query(), replies.bottom, focus="#r")
+    if hasReplies and replies.bottom.len > 0:
+      if tweet == nil or not replies.beginning or replyCount < tweet.stats.replies:
+        renderMore(Query(), replies.bottom, focus="#r")
 
 proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode =
   let hasAfter = conv.after.content.len > 0
@@ -70,6 +75,6 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
       if not conv.replies.beginning:
         renderNewer(Query(), getLink(conv.tweet), focus="#r")
       if conv.replies.content.len > 0 or conv.replies.bottom.len > 0:
-        renderReplies(conv.replies, prefs, path)
+        renderReplies(conv.replies, prefs, path, conv.tweet)
 
     renderToTop(focus="#m")

src/views/timeline.nim

@@ -66,6 +66,7 @@ proc renderUser(user: User; prefs: Prefs): VNode =
         tdiv(class="tweet-name-row"):
           tdiv(class="fullname-and-username"):
             linkUser(user, class="fullname")
+            verifiedIcon(user)
         linkUser(user, class="username")
 
       tdiv(class="tweet-content media-body", dir="auto"):

src/views/tweet.nim

@@ -31,6 +31,7 @@ proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VN
       tdiv(class="tweet-name-row"):
         tdiv(class="fullname-and-username"):
           linkUser(tweet.user, class="fullname")
+          verifiedIcon(tweet.user)
           linkUser(tweet.user, class="username")
 
         span(class="tweet-date"):
@@ -235,6 +236,7 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
       tdiv(class="fullname-and-username"):
         renderMiniAvatar(quote.user, prefs)
         linkUser(quote.user, class="fullname")
+        verifiedIcon(quote.user)
         linkUser(quote.user, class="username")
 
       span(class="tweet-date"):

tests/test_profile.py

@@ -15,7 +15,19 @@ protected = [
     ['Poop', 'Randy', 'Social media fanatic.']
 ]
 
-invalid = [['thisprofiledoesntexist'], ['%']]
+invalid = [['thisprofiledoesntexist']]
+
+malformed = [
+    ['${userId}'],
+    ['$%7BuserId%7D'],  # URL encoded version
+    ['%'],  # Percent sign is invalid
+    ['user@name'],
+    ['user.name'],
+    ['user-name'],
+    ['user$name'],
+    ['user{name}'],
+    ['user name'],  # space
+]
 
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
@@ -65,6 +77,13 @@ class ProfileTest(BaseTestCase):
         self.open_nitter(username)
         self.assert_text(f'User "{username}" not found')
 
+    @parameterized.expand(malformed)
+    def test_malformed_username(self, username):
+        """Test that malformed usernames (with invalid characters) return 404"""
+        self.open_nitter(username)
+        # Malformed usernames should return 404 page not found, not try to fetch from Twitter
+        self.assert_text('Page not found')
+
     def test_suspended(self):
         self.open_nitter('suspendme')
         self.assert_text('User "suspendme" has been suspended')

tools/get_session.py

@@ -21,7 +21,10 @@ def auth(username, password, otp_secret):
 
     guest_token = requests.post(
         "https://api.twitter.com/1.1/guest/activate.json",
-        headers={'Authorization': bearer_token}
+        headers={
+            'Authorization': bearer_token,
+             "User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9"
+        }
     ).json().get('guest_token')
 
     if not guest_token: